From 571e257db12eaa6cdd47811f5663ac1003e32b1b Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 10 Mar 2012 12:26:37 +0100 Subject: [PATCH 0001/1752] Redis 2.6 branch obtained from unstable removing all the cluster related code. --- src/Makefile | 4 +- src/cluster.c | 1762 ------------------------------------------------- src/config.c | 7 - src/db.c | 43 -- src/migrate.c | 190 ++++++ src/pubsub.c | 1 - src/redis.c | 53 +- src/redis.h | 148 +---- 8 files changed, 193 insertions(+), 2015 deletions(-) delete mode 100644 src/cluster.c create mode 100644 src/migrate.c diff --git a/src/Makefile b/src/Makefile index cca785cbc2f..8677b60e7e2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -73,7 +73,7 @@ QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); endif -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o cluster.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o +OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o @@ -101,8 +101,6 @@ aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h bio.o: bio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h bio.h -cluster.o: cluster.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ diff --git a/src/cluster.c b/src/cluster.c deleted file mode 100644 index f76e8ff5cf8..00000000000 --- a/src/cluster.c +++ /dev/null @@ -1,1762 +0,0 @@ -#include "redis.h" - -#include -#include -#include - -void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask); -void clusterSendPing(clusterLink *link, int type); -void clusterSendFail(char *nodename); -void clusterUpdateState(void); -int clusterNodeGetSlotBit(clusterNode *n, int slot); -sds clusterGenNodesDescription(void); -clusterNode *clusterLookupNode(char *name); -int clusterNodeAddSlave(clusterNode *master, clusterNode *slave); -int clusterAddSlot(clusterNode *n, int slot); - -/* ----------------------------------------------------------------------------- - * Initialization - * -------------------------------------------------------------------------- */ - -int clusterLoadConfig(char *filename) { - FILE *fp = fopen(filename,"r"); - char *line; - int maxline, j; - - if (fp == NULL) return REDIS_ERR; - - /* Parse the file. Note that single liens of the cluster config file can - * be really long as they include all the hash slots of the node. - * This means in the worst possible case REDIS_CLUSTER_SLOTS/2 integers. - * To simplify we allocate 1024+REDIS_CLUSTER_SLOTS*16 bytes per line. */ - maxline = 1024+REDIS_CLUSTER_SLOTS*16; - line = zmalloc(maxline); - while(fgets(line,maxline,fp) != NULL) { - int argc; - sds *argv = sdssplitargs(line,&argc); - clusterNode *n, *master; - char *p, *s; - - /* Create this node if it does not exist */ - n = clusterLookupNode(argv[0]); - if (!n) { - n = createClusterNode(argv[0],0); - clusterAddNode(n); - } - /* Address and port */ - if ((p = strchr(argv[1],':')) == NULL) goto fmterr; - *p = '\0'; - memcpy(n->ip,argv[1],strlen(argv[1])+1); - n->port = atoi(p+1); - - /* Parse flags */ - p = s = argv[2]; - while(p) { - p = strchr(s,','); - if (p) *p = '\0'; - if (!strcasecmp(s,"myself")) { - redisAssert(server.cluster.myself == NULL); - server.cluster.myself = n; - n->flags |= REDIS_NODE_MYSELF; - } else if (!strcasecmp(s,"master")) { - n->flags |= REDIS_NODE_MASTER; - } else if (!strcasecmp(s,"slave")) { - n->flags |= REDIS_NODE_SLAVE; - } else if (!strcasecmp(s,"fail?")) { - n->flags |= REDIS_NODE_PFAIL; - } else if (!strcasecmp(s,"fail")) { - n->flags |= REDIS_NODE_FAIL; - } else if (!strcasecmp(s,"handshake")) { - n->flags |= REDIS_NODE_HANDSHAKE; - } else if (!strcasecmp(s,"noaddr")) { - n->flags |= REDIS_NODE_NOADDR; - } else if (!strcasecmp(s,"noflags")) { - /* nothing to do */ - } else { - redisPanic("Unknown flag in redis cluster config file"); - } - if (p) s = p+1; - } - - /* Get master if any. Set the master and populate master's - * slave list. */ - if (argv[3][0] != '-') { - master = clusterLookupNode(argv[3]); - if (!master) { - master = createClusterNode(argv[3],0); - clusterAddNode(master); - } - n->slaveof = master; - clusterNodeAddSlave(master,n); - } - - /* Set ping sent / pong received timestamps */ - if (atoi(argv[4])) n->ping_sent = time(NULL); - if (atoi(argv[5])) n->pong_received = time(NULL); - - /* Populate hash slots served by this instance. */ - for (j = 7; j < argc; j++) { - int start, stop; - - if (argv[j][0] == '[') { - /* Here we handle migrating / importing slots */ - int slot; - char direction; - clusterNode *cn; - - p = strchr(argv[j],'-'); - redisAssert(p != NULL); - *p = '\0'; - direction = p[1]; /* Either '>' or '<' */ - slot = atoi(argv[j]+1); - p += 3; - cn = clusterLookupNode(p); - if (!cn) { - cn = createClusterNode(p,0); - clusterAddNode(cn); - } - if (direction == '>') { - server.cluster.migrating_slots_to[slot] = cn; - } else { - server.cluster.importing_slots_from[slot] = cn; - } - continue; - } else if ((p = strchr(argv[j],'-')) != NULL) { - *p = '\0'; - start = atoi(argv[j]); - stop = atoi(p+1); - } else { - start = stop = atoi(argv[j]); - } - while(start <= stop) clusterAddSlot(n, start++); - } - - sdssplitargs_free(argv,argc); - } - zfree(line); - fclose(fp); - - /* Config sanity check */ - redisAssert(server.cluster.myself != NULL); - redisLog(REDIS_NOTICE,"Node configuration loaded, I'm %.40s", - server.cluster.myself->name); - clusterUpdateState(); - return REDIS_OK; - -fmterr: - redisLog(REDIS_WARNING,"Unrecovarable error: corrupted cluster config file."); - fclose(fp); - exit(1); -} - -/* Cluster node configuration is exactly the same as CLUSTER NODES output. - * - * This function writes the node config and returns 0, on error -1 - * is returned. */ -int clusterSaveConfig(void) { - sds ci = clusterGenNodesDescription(); - int fd; - - if ((fd = open(server.cluster.configfile,O_WRONLY|O_CREAT|O_TRUNC,0644)) - == -1) goto err; - if (write(fd,ci,sdslen(ci)) != (ssize_t)sdslen(ci)) goto err; - close(fd); - sdsfree(ci); - return 0; - -err: - sdsfree(ci); - return -1; -} - -void clusterSaveConfigOrDie(void) { - if (clusterSaveConfig() == -1) { - redisLog(REDIS_WARNING,"Fatal: can't update cluster config file."); - exit(1); - } -} - -void clusterInit(void) { - int saveconf = 0; - - server.cluster.myself = NULL; - server.cluster.state = REDIS_CLUSTER_FAIL; - server.cluster.nodes = dictCreate(&clusterNodesDictType,NULL); - server.cluster.node_timeout = 15; - memset(server.cluster.migrating_slots_to,0, - sizeof(server.cluster.migrating_slots_to)); - memset(server.cluster.importing_slots_from,0, - sizeof(server.cluster.importing_slots_from)); - memset(server.cluster.slots,0, - sizeof(server.cluster.slots)); - if (clusterLoadConfig(server.cluster.configfile) == REDIS_ERR) { - /* No configuration found. We will just use the random name provided - * by the createClusterNode() function. */ - server.cluster.myself = createClusterNode(NULL,REDIS_NODE_MYSELF); - redisLog(REDIS_NOTICE,"No cluster configuration found, I'm %.40s", - server.cluster.myself->name); - clusterAddNode(server.cluster.myself); - saveconf = 1; - } - if (saveconf) clusterSaveConfigOrDie(); - /* We need a listening TCP port for our cluster messaging needs */ - server.cfd = anetTcpServer(server.neterr, - server.port+REDIS_CLUSTER_PORT_INCR, server.bindaddr); - if (server.cfd == -1) { - redisLog(REDIS_WARNING, "Opening cluster TCP port: %s", server.neterr); - exit(1); - } - if (aeCreateFileEvent(server.el, server.cfd, AE_READABLE, - clusterAcceptHandler, NULL) == AE_ERR) oom("creating file event"); - server.cluster.slots_to_keys = zslCreate(); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER communication link - * -------------------------------------------------------------------------- */ - -clusterLink *createClusterLink(clusterNode *node) { - clusterLink *link = zmalloc(sizeof(*link)); - link->sndbuf = sdsempty(); - link->rcvbuf = sdsempty(); - link->node = node; - link->fd = -1; - return link; -} - -/* Free a cluster link, but does not free the associated node of course. - * Just this function will make sure that the original node associated - * with this link will have the 'link' field set to NULL. */ -void freeClusterLink(clusterLink *link) { - if (link->fd != -1) { - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); - aeDeleteFileEvent(server.el, link->fd, AE_READABLE); - } - sdsfree(link->sndbuf); - sdsfree(link->rcvbuf); - if (link->node) - link->node->link = NULL; - close(link->fd); - zfree(link); -} - -void clusterAcceptHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - int cport, cfd; - char cip[128]; - clusterLink *link; - REDIS_NOTUSED(el); - REDIS_NOTUSED(mask); - REDIS_NOTUSED(privdata); - - cfd = anetTcpAccept(server.neterr, fd, cip, &cport); - if (cfd == AE_ERR) { - redisLog(REDIS_VERBOSE,"Accepting cluster node: %s", server.neterr); - return; - } - redisLog(REDIS_VERBOSE,"Accepted cluster node %s:%d", cip, cport); - /* We need to create a temporary node in order to read the incoming - * packet in a valid contest. This node will be released once we - * read the packet and reply. */ - link = createClusterLink(NULL); - link->fd = cfd; - aeCreateFileEvent(server.el,cfd,AE_READABLE,clusterReadHandler,link); -} - -/* ----------------------------------------------------------------------------- - * Key space handling - * -------------------------------------------------------------------------- */ - -/* We have 4096 hash slots. The hash slot of a given key is obtained - * as the least significant 12 bits of the crc16 of the key. */ -unsigned int keyHashSlot(char *key, int keylen) { - return crc16(key,keylen) & 0x0FFF; -} - -/* ----------------------------------------------------------------------------- - * CLUSTER node API - * -------------------------------------------------------------------------- */ - -/* Create a new cluster node, with the specified flags. - * If "nodename" is NULL this is considered a first handshake and a random - * node name is assigned to this node (it will be fixed later when we'll - * receive the first pong). - * - * The node is created and returned to the user, but it is not automatically - * added to the nodes hash table. */ -clusterNode *createClusterNode(char *nodename, int flags) { - clusterNode *node = zmalloc(sizeof(*node)); - - if (nodename) - memcpy(node->name, nodename, REDIS_CLUSTER_NAMELEN); - else - getRandomHexChars(node->name, REDIS_CLUSTER_NAMELEN); - node->flags = flags; - memset(node->slots,0,sizeof(node->slots)); - node->numslaves = 0; - node->slaves = NULL; - node->slaveof = NULL; - node->ping_sent = node->pong_received = 0; - node->configdigest = NULL; - node->configdigest_ts = 0; - node->link = NULL; - return node; -} - -int clusterNodeRemoveSlave(clusterNode *master, clusterNode *slave) { - int j; - - for (j = 0; j < master->numslaves; j++) { - if (master->slaves[j] == slave) { - memmove(master->slaves+j,master->slaves+(j+1), - (master->numslaves-1)-j); - master->numslaves--; - return REDIS_OK; - } - } - return REDIS_ERR; -} - -int clusterNodeAddSlave(clusterNode *master, clusterNode *slave) { - int j; - - /* If it's already a slave, don't add it again. */ - for (j = 0; j < master->numslaves; j++) - if (master->slaves[j] == slave) return REDIS_ERR; - master->slaves = zrealloc(master->slaves, - sizeof(clusterNode*)*(master->numslaves+1)); - master->slaves[master->numslaves] = slave; - master->numslaves++; - return REDIS_OK; -} - -void clusterNodeResetSlaves(clusterNode *n) { - zfree(n->slaves); - n->numslaves = 0; -} - -void freeClusterNode(clusterNode *n) { - sds nodename; - - nodename = sdsnewlen(n->name, REDIS_CLUSTER_NAMELEN); - redisAssert(dictDelete(server.cluster.nodes,nodename) == DICT_OK); - sdsfree(nodename); - if (n->slaveof) clusterNodeRemoveSlave(n->slaveof, n); - if (n->link) freeClusterLink(n->link); - zfree(n); -} - -/* Add a node to the nodes hash table */ -int clusterAddNode(clusterNode *node) { - int retval; - - retval = dictAdd(server.cluster.nodes, - sdsnewlen(node->name,REDIS_CLUSTER_NAMELEN), node); - return (retval == DICT_OK) ? REDIS_OK : REDIS_ERR; -} - -/* Node lookup by name */ -clusterNode *clusterLookupNode(char *name) { - sds s = sdsnewlen(name, REDIS_CLUSTER_NAMELEN); - struct dictEntry *de; - - de = dictFind(server.cluster.nodes,s); - sdsfree(s); - if (de == NULL) return NULL; - return dictGetVal(de); -} - -/* This is only used after the handshake. When we connect a given IP/PORT - * as a result of CLUSTER MEET we don't have the node name yet, so we - * pick a random one, and will fix it when we receive the PONG request using - * this function. */ -void clusterRenameNode(clusterNode *node, char *newname) { - int retval; - sds s = sdsnewlen(node->name, REDIS_CLUSTER_NAMELEN); - - redisLog(REDIS_DEBUG,"Renaming node %.40s into %.40s", - node->name, newname); - retval = dictDelete(server.cluster.nodes, s); - sdsfree(s); - redisAssert(retval == DICT_OK); - memcpy(node->name, newname, REDIS_CLUSTER_NAMELEN); - clusterAddNode(node); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER messages exchange - PING/PONG and gossip - * -------------------------------------------------------------------------- */ - -/* Process the gossip section of PING or PONG packets. - * Note that this function assumes that the packet is already sanity-checked - * by the caller, not in the content of the gossip section, but in the - * length. */ -void clusterProcessGossipSection(clusterMsg *hdr, clusterLink *link) { - uint16_t count = ntohs(hdr->count); - clusterMsgDataGossip *g = (clusterMsgDataGossip*) hdr->data.ping.gossip; - clusterNode *sender = link->node ? link->node : clusterLookupNode(hdr->sender); - - while(count--) { - sds ci = sdsempty(); - uint16_t flags = ntohs(g->flags); - clusterNode *node; - - if (flags == 0) ci = sdscat(ci,"noflags,"); - if (flags & REDIS_NODE_MYSELF) ci = sdscat(ci,"myself,"); - if (flags & REDIS_NODE_MASTER) ci = sdscat(ci,"master,"); - if (flags & REDIS_NODE_SLAVE) ci = sdscat(ci,"slave,"); - if (flags & REDIS_NODE_PFAIL) ci = sdscat(ci,"fail?,"); - if (flags & REDIS_NODE_FAIL) ci = sdscat(ci,"fail,"); - if (flags & REDIS_NODE_HANDSHAKE) ci = sdscat(ci,"handshake,"); - if (flags & REDIS_NODE_NOADDR) ci = sdscat(ci,"noaddr,"); - if (ci[sdslen(ci)-1] == ',') ci[sdslen(ci)-1] = ' '; - - redisLog(REDIS_DEBUG,"GOSSIP %.40s %s:%d %s", - g->nodename, - g->ip, - ntohs(g->port), - ci); - sdsfree(ci); - - /* Update our state accordingly to the gossip sections */ - node = clusterLookupNode(g->nodename); - if (node != NULL) { - /* We already know this node. Let's start updating the last - * time PONG figure if it is newer than our figure. - * Note that it's not a problem if we have a PING already - * in progress against this node. */ - if (node->pong_received < (signed) ntohl(g->pong_received)) { - redisLog(REDIS_DEBUG,"Node pong_received updated by gossip"); - node->pong_received = ntohl(g->pong_received); - } - /* Mark this node as FAILED if we think it is possibly failing - * and another node also thinks it's failing. */ - if (node->flags & REDIS_NODE_PFAIL && - (flags & (REDIS_NODE_FAIL|REDIS_NODE_PFAIL))) - { - redisLog(REDIS_NOTICE,"Received a PFAIL acknowledge from node %.40s, marking node %.40s as FAIL!", hdr->sender, node->name); - node->flags &= ~REDIS_NODE_PFAIL; - node->flags |= REDIS_NODE_FAIL; - /* Broadcast the failing node name to everybody */ - clusterSendFail(node->name); - clusterUpdateState(); - clusterSaveConfigOrDie(); - } - } else { - /* If it's not in NOADDR state and we don't have it, we - * start an handshake process against this IP/PORT pairs. - * - * Note that we require that the sender of this gossip message - * is a well known node in our cluster, otherwise we risk - * joining another cluster. */ - if (sender && !(flags & REDIS_NODE_NOADDR)) { - clusterNode *newnode; - - redisLog(REDIS_DEBUG,"Adding the new node"); - newnode = createClusterNode(NULL,REDIS_NODE_HANDSHAKE); - memcpy(newnode->ip,g->ip,sizeof(g->ip)); - newnode->port = ntohs(g->port); - clusterAddNode(newnode); - } - } - - /* Next node */ - g++; - } -} - -/* IP -> string conversion. 'buf' is supposed to at least be 16 bytes. */ -void nodeIp2String(char *buf, clusterLink *link) { - struct sockaddr_in sa; - socklen_t salen = sizeof(sa); - - if (getpeername(link->fd, (struct sockaddr*) &sa, &salen) == -1) - redisPanic("getpeername() failed."); - strncpy(buf,inet_ntoa(sa.sin_addr),sizeof(link->node->ip)); -} - - -/* Update the node address to the IP address that can be extracted - * from link->fd, and at the specified port. */ -void nodeUpdateAddress(clusterNode *node, clusterLink *link, int port) { - /* TODO */ -} - -/* When this function is called, there is a packet to process starting - * at node->rcvbuf. Releasing the buffer is up to the caller, so this - * function should just handle the higher level stuff of processing the - * packet, modifying the cluster state if needed. - * - * The function returns 1 if the link is still valid after the packet - * was processed, otherwise 0 if the link was freed since the packet - * processing lead to some inconsistency error (for instance a PONG - * received from the wrong sender ID). */ -int clusterProcessPacket(clusterLink *link) { - clusterMsg *hdr = (clusterMsg*) link->rcvbuf; - uint32_t totlen = ntohl(hdr->totlen); - uint16_t type = ntohs(hdr->type); - clusterNode *sender; - - redisLog(REDIS_DEBUG,"--- Processing packet of type %d, %lu bytes", - type, (unsigned long) totlen); - - /* Perform sanity checks */ - if (totlen < 8) return 1; - if (totlen > sdslen(link->rcvbuf)) return 1; - if (type == CLUSTERMSG_TYPE_PING || type == CLUSTERMSG_TYPE_PONG || - type == CLUSTERMSG_TYPE_MEET) - { - uint16_t count = ntohs(hdr->count); - uint32_t explen; /* expected length of this packet */ - - explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - explen += (sizeof(clusterMsgDataGossip)*count); - if (totlen != explen) return 1; - } - if (type == CLUSTERMSG_TYPE_FAIL) { - uint32_t explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - - explen += sizeof(clusterMsgDataFail); - if (totlen != explen) return 1; - } - if (type == CLUSTERMSG_TYPE_PUBLISH) { - uint32_t explen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - - explen += sizeof(clusterMsgDataPublish) + - ntohl(hdr->data.publish.msg.channel_len) + - ntohl(hdr->data.publish.msg.message_len); - if (totlen != explen) return 1; - } - - /* Ready to process the packet. Dispatch by type. */ - sender = clusterLookupNode(hdr->sender); - if (type == CLUSTERMSG_TYPE_PING || type == CLUSTERMSG_TYPE_MEET) { - int update_config = 0; - redisLog(REDIS_DEBUG,"Ping packet received: %p", link->node); - - /* Add this node if it is new for us and the msg type is MEET. - * In this stage we don't try to add the node with the right - * flags, slaveof pointer, and so forth, as this details will be - * resolved when we'll receive PONGs from the server. */ - if (!sender && type == CLUSTERMSG_TYPE_MEET) { - clusterNode *node; - - node = createClusterNode(NULL,REDIS_NODE_HANDSHAKE); - nodeIp2String(node->ip,link); - node->port = ntohs(hdr->port); - clusterAddNode(node); - update_config = 1; - } - - /* Get info from the gossip section */ - clusterProcessGossipSection(hdr,link); - - /* Anyway reply with a PONG */ - clusterSendPing(link,CLUSTERMSG_TYPE_PONG); - - /* Update config if needed */ - if (update_config) clusterSaveConfigOrDie(); - } else if (type == CLUSTERMSG_TYPE_PONG) { - int update_state = 0; - int update_config = 0; - - redisLog(REDIS_DEBUG,"Pong packet received: %p", link->node); - if (link->node) { - if (link->node->flags & REDIS_NODE_HANDSHAKE) { - /* If we already have this node, try to change the - * IP/port of the node with the new one. */ - if (sender) { - redisLog(REDIS_WARNING, - "Handshake error: we already know node %.40s, updating the address if needed.", sender->name); - nodeUpdateAddress(sender,link,ntohs(hdr->port)); - freeClusterNode(link->node); /* will free the link too */ - return 0; - } - - /* First thing to do is replacing the random name with the - * right node name if this was an handshake stage. */ - clusterRenameNode(link->node, hdr->sender); - redisLog(REDIS_DEBUG,"Handshake with node %.40s completed.", - link->node->name); - link->node->flags &= ~REDIS_NODE_HANDSHAKE; - update_config = 1; - } else if (memcmp(link->node->name,hdr->sender, - REDIS_CLUSTER_NAMELEN) != 0) - { - /* If the reply has a non matching node ID we - * disconnect this node and set it as not having an associated - * address. */ - redisLog(REDIS_DEBUG,"PONG contains mismatching sender ID"); - link->node->flags |= REDIS_NODE_NOADDR; - freeClusterLink(link); - update_config = 1; - /* FIXME: remove this node if we already have it. - * - * If we already have it but the IP is different, use - * the new one if the old node is in FAIL, PFAIL, or NOADDR - * status... */ - return 0; - } - } - /* Update our info about the node */ - if (link->node) link->node->pong_received = time(NULL); - - /* Update master/slave info */ - if (sender) { - if (!memcmp(hdr->slaveof,REDIS_NODE_NULL_NAME, - sizeof(hdr->slaveof))) - { - sender->flags &= ~REDIS_NODE_SLAVE; - sender->flags |= REDIS_NODE_MASTER; - sender->slaveof = NULL; - } else { - clusterNode *master = clusterLookupNode(hdr->slaveof); - - sender->flags &= ~REDIS_NODE_MASTER; - sender->flags |= REDIS_NODE_SLAVE; - if (sender->numslaves) clusterNodeResetSlaves(sender); - if (master) clusterNodeAddSlave(master,sender); - } - } - - /* Update our info about served slots if this new node is serving - * slots that are not served from our point of view. */ - if (sender && sender->flags & REDIS_NODE_MASTER) { - int newslots, j; - - newslots = - memcmp(sender->slots,hdr->myslots,sizeof(hdr->myslots)) != 0; - memcpy(sender->slots,hdr->myslots,sizeof(hdr->myslots)); - if (newslots) { - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - if (clusterNodeGetSlotBit(sender,j)) { - if (server.cluster.slots[j] == sender) continue; - if (server.cluster.slots[j] == NULL || - server.cluster.slots[j]->flags & REDIS_NODE_FAIL) - { - server.cluster.slots[j] = sender; - update_state = update_config = 1; - } - } - } - } - } - - /* Get info from the gossip section */ - clusterProcessGossipSection(hdr,link); - - /* Update the cluster state if needed */ - if (update_state) clusterUpdateState(); - if (update_config) clusterSaveConfigOrDie(); - } else if (type == CLUSTERMSG_TYPE_FAIL && sender) { - clusterNode *failing; - - failing = clusterLookupNode(hdr->data.fail.about.nodename); - if (failing && !(failing->flags & (REDIS_NODE_FAIL|REDIS_NODE_MYSELF))) - { - redisLog(REDIS_NOTICE, - "FAIL message received from %.40s about %.40s", - hdr->sender, hdr->data.fail.about.nodename); - failing->flags |= REDIS_NODE_FAIL; - failing->flags &= ~REDIS_NODE_PFAIL; - clusterUpdateState(); - clusterSaveConfigOrDie(); - } - } else if (type == CLUSTERMSG_TYPE_PUBLISH) { - robj *channel, *message; - uint32_t channel_len, message_len; - - /* Don't bother creating useless objects if there are no Pub/Sub subscribers. */ - if (dictSize(server.pubsub_channels) || listLength(server.pubsub_patterns)) { - channel_len = ntohl(hdr->data.publish.msg.channel_len); - message_len = ntohl(hdr->data.publish.msg.message_len); - channel = createStringObject( - (char*)hdr->data.publish.msg.bulk_data,channel_len); - message = createStringObject( - (char*)hdr->data.publish.msg.bulk_data+channel_len, message_len); - pubsubPublishMessage(channel,message); - decrRefCount(channel); - decrRefCount(message); - } - } else { - redisLog(REDIS_WARNING,"Received unknown packet type: %d", type); - } - return 1; -} - -/* This function is called when we detect the link with this node is lost. - We set the node as no longer connected. The Cluster Cron will detect - this connection and will try to get it connected again. - - Instead if the node is a temporary node used to accept a query, we - completely free the node on error. */ -void handleLinkIOError(clusterLink *link) { - freeClusterLink(link); -} - -/* Send data. This is handled using a trivial send buffer that gets - * consumed by write(). We don't try to optimize this for speed too much - * as this is a very low traffic channel. */ -void clusterWriteHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - clusterLink *link = (clusterLink*) privdata; - ssize_t nwritten; - REDIS_NOTUSED(el); - REDIS_NOTUSED(mask); - - nwritten = write(fd, link->sndbuf, sdslen(link->sndbuf)); - if (nwritten <= 0) { - redisLog(REDIS_NOTICE,"I/O error writing to node link: %s", - strerror(errno)); - handleLinkIOError(link); - return; - } - link->sndbuf = sdsrange(link->sndbuf,nwritten,-1); - if (sdslen(link->sndbuf) == 0) - aeDeleteFileEvent(server.el, link->fd, AE_WRITABLE); -} - -/* Read data. Try to read the first field of the header first to check the - * full length of the packet. When a whole packet is in memory this function - * will call the function to process the packet. And so forth. */ -void clusterReadHandler(aeEventLoop *el, int fd, void *privdata, int mask) { - char buf[1024]; - ssize_t nread; - clusterMsg *hdr; - clusterLink *link = (clusterLink*) privdata; - int readlen; - REDIS_NOTUSED(el); - REDIS_NOTUSED(mask); - -again: - if (sdslen(link->rcvbuf) >= 4) { - hdr = (clusterMsg*) link->rcvbuf; - readlen = ntohl(hdr->totlen) - sdslen(link->rcvbuf); - } else { - readlen = 4 - sdslen(link->rcvbuf); - } - - nread = read(fd,buf,readlen); - if (nread == -1 && errno == EAGAIN) return; /* Just no data */ - - if (nread <= 0) { - /* I/O error... */ - redisLog(REDIS_NOTICE,"I/O error reading from node link: %s", - (nread == 0) ? "connection closed" : strerror(errno)); - handleLinkIOError(link); - return; - } else { - /* Read data and recast the pointer to the new buffer. */ - link->rcvbuf = sdscatlen(link->rcvbuf,buf,nread); - hdr = (clusterMsg*) link->rcvbuf; - } - - /* Total length obtained? read the payload now instead of burning - * cycles waiting for a new event to fire. */ - if (sdslen(link->rcvbuf) == 4) goto again; - - /* Whole packet in memory? We can process it. */ - if (sdslen(link->rcvbuf) == ntohl(hdr->totlen)) { - if (clusterProcessPacket(link)) { - sdsfree(link->rcvbuf); - link->rcvbuf = sdsempty(); - } - } -} - -/* Put stuff into the send buffer. */ -void clusterSendMessage(clusterLink *link, unsigned char *msg, size_t msglen) { - if (sdslen(link->sndbuf) == 0 && msglen != 0) - aeCreateFileEvent(server.el,link->fd,AE_WRITABLE, - clusterWriteHandler,link); - - link->sndbuf = sdscatlen(link->sndbuf, msg, msglen); -} - -/* Send a message to all the nodes with a reliable link */ -void clusterBroadcastMessage(void *buf, size_t len) { - dictIterator *di; - dictEntry *de; - - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - - if (!node->link) continue; - if (node->flags & (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR)) continue; - clusterSendMessage(node->link,buf,len); - } - dictReleaseIterator(di); -} - -/* Build the message header */ -void clusterBuildMessageHdr(clusterMsg *hdr, int type) { - int totlen = 0; - - memset(hdr,0,sizeof(*hdr)); - hdr->type = htons(type); - memcpy(hdr->sender,server.cluster.myself->name,REDIS_CLUSTER_NAMELEN); - memcpy(hdr->myslots,server.cluster.myself->slots, - sizeof(hdr->myslots)); - memset(hdr->slaveof,0,REDIS_CLUSTER_NAMELEN); - if (server.cluster.myself->slaveof != NULL) { - memcpy(hdr->slaveof,server.cluster.myself->slaveof->name, - REDIS_CLUSTER_NAMELEN); - } - hdr->port = htons(server.port); - hdr->state = server.cluster.state; - memset(hdr->configdigest,0,32); /* FIXME: set config digest */ - - if (type == CLUSTERMSG_TYPE_FAIL) { - totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - totlen += sizeof(clusterMsgDataFail); - } - hdr->totlen = htonl(totlen); - /* For PING, PONG, and MEET, fixing the totlen field is up to the caller */ -} - -/* Send a PING or PONG packet to the specified node, making sure to add enough - * gossip informations. */ -void clusterSendPing(clusterLink *link, int type) { - unsigned char buf[1024]; - clusterMsg *hdr = (clusterMsg*) buf; - int gossipcount = 0, totlen; - /* freshnodes is the number of nodes we can still use to populate the - * gossip section of the ping packet. Basically we start with the nodes - * we have in memory minus two (ourself and the node we are sending the - * message to). Every time we add a node we decrement the counter, so when - * it will drop to <= zero we know there is no more gossip info we can - * send. */ - int freshnodes = dictSize(server.cluster.nodes)-2; - - if (link->node && type == CLUSTERMSG_TYPE_PING) - link->node->ping_sent = time(NULL); - clusterBuildMessageHdr(hdr,type); - - /* Populate the gossip fields */ - while(freshnodes > 0 && gossipcount < 3) { - struct dictEntry *de = dictGetRandomKey(server.cluster.nodes); - clusterNode *this = dictGetVal(de); - clusterMsgDataGossip *gossip; - int j; - - /* Not interesting to gossip about ourself. - * Nor to send gossip info about HANDSHAKE state nodes (zero info). */ - if (this == server.cluster.myself || - this->flags & REDIS_NODE_HANDSHAKE) { - freshnodes--; /* otherwise we may loop forever. */ - continue; - } - - /* Check if we already added this node */ - for (j = 0; j < gossipcount; j++) { - if (memcmp(hdr->data.ping.gossip[j].nodename,this->name, - REDIS_CLUSTER_NAMELEN) == 0) break; - } - if (j != gossipcount) continue; - - /* Add it */ - freshnodes--; - gossip = &(hdr->data.ping.gossip[gossipcount]); - memcpy(gossip->nodename,this->name,REDIS_CLUSTER_NAMELEN); - gossip->ping_sent = htonl(this->ping_sent); - gossip->pong_received = htonl(this->pong_received); - memcpy(gossip->ip,this->ip,sizeof(this->ip)); - gossip->port = htons(this->port); - gossip->flags = htons(this->flags); - gossipcount++; - } - totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - totlen += (sizeof(clusterMsgDataGossip)*gossipcount); - hdr->count = htons(gossipcount); - hdr->totlen = htonl(totlen); - clusterSendMessage(link,buf,totlen); -} - -/* Send a PUBLISH message. - * - * If link is NULL, then the message is broadcasted to the whole cluster. */ -void clusterSendPublish(clusterLink *link, robj *channel, robj *message) { - unsigned char buf[4096], *payload; - clusterMsg *hdr = (clusterMsg*) buf; - uint32_t totlen; - uint32_t channel_len, message_len; - - channel = getDecodedObject(channel); - message = getDecodedObject(message); - channel_len = sdslen(channel->ptr); - message_len = sdslen(message->ptr); - - clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_PUBLISH); - totlen = sizeof(clusterMsg)-sizeof(union clusterMsgData); - totlen += sizeof(clusterMsgDataPublish) + channel_len + message_len; - - hdr->data.publish.msg.channel_len = htonl(channel_len); - hdr->data.publish.msg.message_len = htonl(message_len); - hdr->totlen = htonl(totlen); - - /* Try to use the local buffer if possible */ - if (totlen < sizeof(buf)) { - payload = buf; - } else { - payload = zmalloc(totlen); - hdr = (clusterMsg*) payload; - memcpy(payload,hdr,sizeof(hdr)); - } - memcpy(hdr->data.publish.msg.bulk_data,channel->ptr,sdslen(channel->ptr)); - memcpy(hdr->data.publish.msg.bulk_data+sdslen(channel->ptr), - message->ptr,sdslen(message->ptr)); - - if (link) - clusterSendMessage(link,payload,totlen); - else - clusterBroadcastMessage(payload,totlen); - - decrRefCount(channel); - decrRefCount(message); - if (payload != buf) zfree(payload); -} - -/* Send a FAIL message to all the nodes we are able to contact. - * The FAIL message is sent when we detect that a node is failing - * (REDIS_NODE_PFAIL) and we also receive a gossip confirmation of this: - * we switch the node state to REDIS_NODE_FAIL and ask all the other - * nodes to do the same ASAP. */ -void clusterSendFail(char *nodename) { - unsigned char buf[1024]; - clusterMsg *hdr = (clusterMsg*) buf; - - clusterBuildMessageHdr(hdr,CLUSTERMSG_TYPE_FAIL); - memcpy(hdr->data.fail.about.nodename,nodename,REDIS_CLUSTER_NAMELEN); - clusterBroadcastMessage(buf,ntohl(hdr->totlen)); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER Pub/Sub support - * - * For now we do very little, just propagating PUBLISH messages across the whole - * cluster. In the future we'll try to get smarter and avoiding propagating those - * messages to hosts without receives for a given channel. - * -------------------------------------------------------------------------- */ -void clusterPropagatePublish(robj *channel, robj *message) { - clusterSendPublish(NULL, channel, message); -} - -/* ----------------------------------------------------------------------------- - * CLUSTER cron job - * -------------------------------------------------------------------------- */ - -/* This is executed 1 time every second */ -void clusterCron(void) { - dictIterator *di; - dictEntry *de; - int j; - time_t min_ping_sent = 0; - clusterNode *min_ping_node = NULL; - - /* Check if we have disconnected nodes and reestablish the connection. */ - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - - if (node->flags & (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR)) continue; - if (node->link == NULL) { - int fd; - clusterLink *link; - - fd = anetTcpNonBlockConnect(server.neterr, node->ip, - node->port+REDIS_CLUSTER_PORT_INCR); - if (fd == -1) continue; - link = createClusterLink(node); - link->fd = fd; - node->link = link; - aeCreateFileEvent(server.el,link->fd,AE_READABLE,clusterReadHandler,link); - /* If the node is flagged as MEET, we send a MEET message instead - * of a PING one, to force the receiver to add us in its node - * table. */ - clusterSendPing(link, node->flags & REDIS_NODE_MEET ? - CLUSTERMSG_TYPE_MEET : CLUSTERMSG_TYPE_PING); - /* We can clear the flag after the first packet is sent. - * If we'll never receive a PONG, we'll never send new packets - * to this node. Instead after the PONG is received and we - * are no longer in meet/handshake status, we want to send - * normal PING packets. */ - node->flags &= ~REDIS_NODE_MEET; - - redisLog(REDIS_NOTICE,"Connecting with Node %.40s at %s:%d", node->name, node->ip, node->port+REDIS_CLUSTER_PORT_INCR); - } - } - dictReleaseIterator(di); - - /* Ping some random node. Check a few random nodes and ping the one with - * the oldest ping_sent time */ - for (j = 0; j < 5; j++) { - de = dictGetRandomKey(server.cluster.nodes); - clusterNode *this = dictGetVal(de); - - if (this->link == NULL) continue; - if (this->flags & (REDIS_NODE_MYSELF|REDIS_NODE_HANDSHAKE)) continue; - if (min_ping_node == NULL || min_ping_sent > this->ping_sent) { - min_ping_node = this; - min_ping_sent = this->ping_sent; - } - } - if (min_ping_node) { - redisLog(REDIS_DEBUG,"Pinging node %40s", min_ping_node->name); - clusterSendPing(min_ping_node->link, CLUSTERMSG_TYPE_PING); - } - - /* Iterate nodes to check if we need to flag something as failing */ - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - int delay; - - if (node->flags & - (REDIS_NODE_MYSELF|REDIS_NODE_NOADDR|REDIS_NODE_HANDSHAKE)) - continue; - /* Check only if we already sent a ping and did not received - * a reply yet. */ - if (node->ping_sent == 0 || - node->ping_sent <= node->pong_received) continue; - - delay = time(NULL) - node->pong_received; - if (delay < server.cluster.node_timeout) { - /* The PFAIL condition can be reversed without external - * help if it is not transitive (that is, if it does not - * turn into a FAIL state). - * - * The FAIL condition is also reversible if there are no slaves - * for this host, so no slave election should be in progress. - * - * TODO: consider all the implications of resurrecting a - * FAIL node. */ - if (node->flags & REDIS_NODE_PFAIL) { - node->flags &= ~REDIS_NODE_PFAIL; - } else if (node->flags & REDIS_NODE_FAIL && !node->numslaves) { - node->flags &= ~REDIS_NODE_FAIL; - clusterUpdateState(); - } - } else { - /* Timeout reached. Set the noad se possibly failing if it is - * not already in this state. */ - if (!(node->flags & (REDIS_NODE_PFAIL|REDIS_NODE_FAIL))) { - redisLog(REDIS_DEBUG,"*** NODE %.40s possibly failing", - node->name); - node->flags |= REDIS_NODE_PFAIL; - } - } - } - dictReleaseIterator(di); -} - -/* ----------------------------------------------------------------------------- - * Slots management - * -------------------------------------------------------------------------- */ - -/* Set the slot bit and return the old value. */ -int clusterNodeSetSlotBit(clusterNode *n, int slot) { - off_t byte = slot/8; - int bit = slot&7; - int old = (n->slots[byte] & (1<slots[byte] |= 1<slots[byte] & (1<slots[byte] &= ~(1<slots[byte] & (1<flags & (REDIS_NODE_FAIL)) - { - ok = 0; - break; - } - } - if (ok) { - if (server.cluster.state == REDIS_CLUSTER_NEEDHELP) { - server.cluster.state = REDIS_CLUSTER_NEEDHELP; - } else { - server.cluster.state = REDIS_CLUSTER_OK; - } - } else { - server.cluster.state = REDIS_CLUSTER_FAIL; - } -} - -/* ----------------------------------------------------------------------------- - * CLUSTER command - * -------------------------------------------------------------------------- */ - -sds clusterGenNodesDescription(void) { - sds ci = sdsempty(); - dictIterator *di; - dictEntry *de; - int j, start; - - di = dictGetIterator(server.cluster.nodes); - while((de = dictNext(di)) != NULL) { - clusterNode *node = dictGetVal(de); - - /* Node coordinates */ - ci = sdscatprintf(ci,"%.40s %s:%d ", - node->name, - node->ip, - node->port); - - /* Flags */ - if (node->flags == 0) ci = sdscat(ci,"noflags,"); - if (node->flags & REDIS_NODE_MYSELF) ci = sdscat(ci,"myself,"); - if (node->flags & REDIS_NODE_MASTER) ci = sdscat(ci,"master,"); - if (node->flags & REDIS_NODE_SLAVE) ci = sdscat(ci,"slave,"); - if (node->flags & REDIS_NODE_PFAIL) ci = sdscat(ci,"fail?,"); - if (node->flags & REDIS_NODE_FAIL) ci = sdscat(ci,"fail,"); - if (node->flags & REDIS_NODE_HANDSHAKE) ci =sdscat(ci,"handshake,"); - if (node->flags & REDIS_NODE_NOADDR) ci = sdscat(ci,"noaddr,"); - if (ci[sdslen(ci)-1] == ',') ci[sdslen(ci)-1] = ' '; - - /* Slave of... or just "-" */ - if (node->slaveof) - ci = sdscatprintf(ci,"%.40s ",node->slaveof->name); - else - ci = sdscatprintf(ci,"- "); - - /* Latency from the POV of this node, link status */ - ci = sdscatprintf(ci,"%ld %ld %s", - (long) node->ping_sent, - (long) node->pong_received, - (node->link || node->flags & REDIS_NODE_MYSELF) ? - "connected" : "disconnected"); - - /* Slots served by this instance */ - start = -1; - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - int bit; - - if ((bit = clusterNodeGetSlotBit(node,j)) != 0) { - if (start == -1) start = j; - } - if (start != -1 && (!bit || j == REDIS_CLUSTER_SLOTS-1)) { - if (j == REDIS_CLUSTER_SLOTS-1) j++; - - if (start == j-1) { - ci = sdscatprintf(ci," %d",start); - } else { - ci = sdscatprintf(ci," %d-%d",start,j-1); - } - start = -1; - } - } - - /* Just for MYSELF node we also dump info about slots that - * we are migrating to other instances or importing from other - * instances. */ - if (node->flags & REDIS_NODE_MYSELF) { - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - if (server.cluster.migrating_slots_to[j]) { - ci = sdscatprintf(ci," [%d->-%.40s]",j, - server.cluster.migrating_slots_to[j]->name); - } else if (server.cluster.importing_slots_from[j]) { - ci = sdscatprintf(ci," [%d-<-%.40s]",j, - server.cluster.importing_slots_from[j]->name); - } - } - } - ci = sdscatlen(ci,"\n",1); - } - dictReleaseIterator(di); - return ci; -} - -int getSlotOrReply(redisClient *c, robj *o) { - long long slot; - - if (getLongLongFromObject(o,&slot) != REDIS_OK || - slot < 0 || slot > REDIS_CLUSTER_SLOTS) - { - addReplyError(c,"Invalid or out of range slot"); - return -1; - } - return (int) slot; -} - -void clusterCommand(redisClient *c) { - if (server.cluster_enabled == 0) { - addReplyError(c,"This instance has cluster support disabled"); - return; - } - - if (!strcasecmp(c->argv[1]->ptr,"meet") && c->argc == 4) { - clusterNode *n; - struct sockaddr_in sa; - long port; - - /* Perform sanity checks on IP/port */ - if (inet_aton(c->argv[2]->ptr,&sa.sin_addr) == 0) { - addReplyError(c,"Invalid IP address in MEET"); - return; - } - if (getLongFromObjectOrReply(c, c->argv[3], &port, NULL) != REDIS_OK || - port < 0 || port > (65535-REDIS_CLUSTER_PORT_INCR)) - { - addReplyError(c,"Invalid TCP port specified"); - return; - } - - /* Finally add the node to the cluster with a random name, this - * will get fixed in the first handshake (ping/pong). */ - n = createClusterNode(NULL,REDIS_NODE_HANDSHAKE|REDIS_NODE_MEET); - strncpy(n->ip,inet_ntoa(sa.sin_addr),sizeof(n->ip)); - n->port = port; - clusterAddNode(n); - addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"nodes") && c->argc == 2) { - robj *o; - sds ci = clusterGenNodesDescription(); - - o = createObject(REDIS_STRING,ci); - addReplyBulk(c,o); - decrRefCount(o); - } else if ((!strcasecmp(c->argv[1]->ptr,"addslots") || - !strcasecmp(c->argv[1]->ptr,"delslots")) && c->argc >= 3) - { - /* CLUSTER ADDSLOTS [slot] ... */ - /* CLUSTER DELSLOTS [slot] ... */ - int j, slot; - unsigned char *slots = zmalloc(REDIS_CLUSTER_SLOTS); - int del = !strcasecmp(c->argv[1]->ptr,"delslots"); - - memset(slots,0,REDIS_CLUSTER_SLOTS); - /* Check that all the arguments are parsable and that all the - * slots are not already busy. */ - for (j = 2; j < c->argc; j++) { - if ((slot = getSlotOrReply(c,c->argv[j])) == -1) { - zfree(slots); - return; - } - if (del && server.cluster.slots[slot] == NULL) { - addReplyErrorFormat(c,"Slot %d is already unassigned", slot); - zfree(slots); - return; - } else if (!del && server.cluster.slots[slot]) { - addReplyErrorFormat(c,"Slot %d is already busy", slot); - zfree(slots); - return; - } - if (slots[slot]++ == 1) { - addReplyErrorFormat(c,"Slot %d specified multiple times", - (int)slot); - zfree(slots); - return; - } - } - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - if (slots[j]) { - int retval; - - /* If this slot was set as importing we can clear this - * state as now we are the real owner of the slot. */ - if (server.cluster.importing_slots_from[j]) - server.cluster.importing_slots_from[j] = NULL; - - retval = del ? clusterDelSlot(j) : - clusterAddSlot(server.cluster.myself,j); - redisAssertWithInfo(c,NULL,retval == REDIS_OK); - } - } - zfree(slots); - clusterUpdateState(); - clusterSaveConfigOrDie(); - addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"setslot") && c->argc >= 4) { - /* SETSLOT 10 MIGRATING */ - /* SETSLOT 10 IMPORTING */ - /* SETSLOT 10 STABLE */ - /* SETSLOT 10 NODE */ - int slot; - clusterNode *n; - - if ((slot = getSlotOrReply(c,c->argv[2])) == -1) return; - - if (!strcasecmp(c->argv[3]->ptr,"migrating") && c->argc == 5) { - if (server.cluster.slots[slot] != server.cluster.myself) { - addReplyErrorFormat(c,"I'm not the owner of hash slot %u",slot); - return; - } - if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) { - addReplyErrorFormat(c,"I don't know about node %s", - (char*)c->argv[4]->ptr); - return; - } - server.cluster.migrating_slots_to[slot] = n; - } else if (!strcasecmp(c->argv[3]->ptr,"importing") && c->argc == 5) { - if (server.cluster.slots[slot] == server.cluster.myself) { - addReplyErrorFormat(c, - "I'm already the owner of hash slot %u",slot); - return; - } - if ((n = clusterLookupNode(c->argv[4]->ptr)) == NULL) { - addReplyErrorFormat(c,"I don't know about node %s", - (char*)c->argv[3]->ptr); - return; - } - server.cluster.importing_slots_from[slot] = n; - } else if (!strcasecmp(c->argv[3]->ptr,"stable") && c->argc == 4) { - /* CLUSTER SETSLOT STABLE */ - server.cluster.importing_slots_from[slot] = NULL; - server.cluster.migrating_slots_to[slot] = NULL; - } else if (!strcasecmp(c->argv[3]->ptr,"node") && c->argc == 5) { - /* CLUSTER SETSLOT NODE */ - clusterNode *n = clusterLookupNode(c->argv[4]->ptr); - - if (!n) addReplyErrorFormat(c,"Unknown node %s", - (char*)c->argv[4]->ptr); - /* If this hash slot was served by 'myself' before to switch - * make sure there are no longer local keys for this hash slot. */ - if (server.cluster.slots[slot] == server.cluster.myself && - n != server.cluster.myself) - { - int numkeys; - robj **keys; - - keys = zmalloc(sizeof(robj*)*1); - numkeys = GetKeysInSlot(slot, keys, 1); - zfree(keys); - if (numkeys != 0) { - addReplyErrorFormat(c, "Can't assign hashslot %d to a different node while I still hold keys for this hash slot.", slot); - return; - } - } - /* If this node was the slot owner and the slot was marked as - * migrating, assigning the slot to another node will clear - * the migratig status. */ - if (server.cluster.slots[slot] == server.cluster.myself && - server.cluster.migrating_slots_to[slot]) - server.cluster.migrating_slots_to[slot] = NULL; - - /* If this node was importing this slot, assigning the slot to - * itself also clears the importing status. */ - if (n == server.cluster.myself && server.cluster.importing_slots_from[slot]) - server.cluster.importing_slots_from[slot] = NULL; - - clusterDelSlot(slot); - clusterAddSlot(n,slot); - } else { - addReplyError(c,"Invalid CLUSTER SETSLOT action or number of arguments"); - return; - } - clusterSaveConfigOrDie(); - addReply(c,shared.ok); - } else if (!strcasecmp(c->argv[1]->ptr,"info") && c->argc == 2) { - char *statestr[] = {"ok","fail","needhelp"}; - int slots_assigned = 0, slots_ok = 0, slots_pfail = 0, slots_fail = 0; - int j; - - for (j = 0; j < REDIS_CLUSTER_SLOTS; j++) { - clusterNode *n = server.cluster.slots[j]; - - if (n == NULL) continue; - slots_assigned++; - if (n->flags & REDIS_NODE_FAIL) { - slots_fail++; - } else if (n->flags & REDIS_NODE_PFAIL) { - slots_pfail++; - } else { - slots_ok++; - } - } - - sds info = sdscatprintf(sdsempty(), - "cluster_state:%s\r\n" - "cluster_slots_assigned:%d\r\n" - "cluster_slots_ok:%d\r\n" - "cluster_slots_pfail:%d\r\n" - "cluster_slots_fail:%d\r\n" - "cluster_known_nodes:%lu\r\n" - , statestr[server.cluster.state], - slots_assigned, - slots_ok, - slots_pfail, - slots_fail, - dictSize(server.cluster.nodes) - ); - addReplySds(c,sdscatprintf(sdsempty(),"$%lu\r\n", - (unsigned long)sdslen(info))); - addReplySds(c,info); - addReply(c,shared.crlf); - } else if (!strcasecmp(c->argv[1]->ptr,"keyslot") && c->argc == 3) { - sds key = c->argv[2]->ptr; - - addReplyLongLong(c,keyHashSlot(key,sdslen(key))); - } else if (!strcasecmp(c->argv[1]->ptr,"getkeysinslot") && c->argc == 4) { - long long maxkeys, slot; - unsigned int numkeys, j; - robj **keys; - - if (getLongLongFromObjectOrReply(c,c->argv[2],&slot,NULL) != REDIS_OK) - return; - if (getLongLongFromObjectOrReply(c,c->argv[3],&maxkeys,NULL) != REDIS_OK) - return; - if (slot < 0 || slot >= REDIS_CLUSTER_SLOTS || maxkeys < 0 || - maxkeys > 1024*1024) { - addReplyError(c,"Invalid slot or number of keys"); - return; - } - - keys = zmalloc(sizeof(robj*)*maxkeys); - numkeys = GetKeysInSlot(slot, keys, maxkeys); - addReplyMultiBulkLen(c,numkeys); - for (j = 0; j < numkeys; j++) addReplyBulk(c,keys[j]); - zfree(keys); - } else { - addReplyError(c,"Wrong CLUSTER subcommand or number of arguments"); - } -} - -/* ----------------------------------------------------------------------------- - * RESTORE and MIGRATE commands - * -------------------------------------------------------------------------- */ - -/* RESTORE key ttl serialized-value */ -void restoreCommand(redisClient *c) { - long ttl; - rio payload; - int type; - robj *obj; - - /* Make sure this key does not already exist here... */ - if (lookupKeyWrite(c->db,c->argv[1]) != NULL) { - addReplyError(c,"Target key name is busy."); - return; - } - - /* Check if the TTL value makes sense */ - if (getLongFromObjectOrReply(c,c->argv[2],&ttl,NULL) != REDIS_OK) { - return; - } else if (ttl < 0) { - addReplyError(c,"Invalid TTL value, must be >= 0"); - return; - } - - rioInitWithBuffer(&payload,c->argv[3]->ptr); - if (((type = rdbLoadObjectType(&payload)) == -1) || - ((obj = rdbLoadObject(type,&payload)) == NULL)) - { - addReplyError(c,"Bad data format"); - return; - } - - /* Create the key and set the TTL if any */ - dbAdd(c->db,c->argv[1],obj); - if (ttl) setExpire(c->db,c->argv[1],time(NULL)+ttl); - signalModifiedKey(c->db,c->argv[1]); - addReply(c,shared.ok); - server.dirty++; -} - -/* MIGRATE host port key dbid timeout */ -void migrateCommand(redisClient *c) { - int fd; - long timeout; - long dbid; - time_t ttl; - robj *o; - rio cmd, payload; - - /* Sanity check */ - if (getLongFromObjectOrReply(c,c->argv[5],&timeout,NULL) != REDIS_OK) - return; - if (getLongFromObjectOrReply(c,c->argv[4],&dbid,NULL) != REDIS_OK) - return; - if (timeout <= 0) timeout = 1; - - /* Check if the key is here. If not we reply with success as there is - * nothing to migrate (for instance the key expired in the meantime), but - * we include such information in the reply string. */ - if ((o = lookupKeyRead(c->db,c->argv[3])) == NULL) { - addReplySds(c,sdsnew("+NOKEY\r\n")); - return; - } - - /* Connect */ - fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, - atoi(c->argv[2]->ptr)); - if (fd == -1) { - addReplyErrorFormat(c,"Can't connect to target node: %s", - server.neterr); - return; - } - if ((aeWait(fd,AE_WRITABLE,timeout*1000) & AE_WRITABLE) == 0) { - addReplyError(c,"Timeout connecting to the client"); - return; - } - - rioInitWithBuffer(&cmd,sdsempty()); - redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); - redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); - - ttl = getExpire(c->db,c->argv[3]); - redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',4)); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); - redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr))); - redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl)); - - /* Finally the last argument that is the serailized object payload - * in the form: . */ - rioInitWithBuffer(&payload,sdsempty()); - redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); - redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o) != -1); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr))); - sdsfree(payload.io.buffer.ptr); - - /* Tranfer the query to the other node in 64K chunks. */ - { - sds buf = cmd.io.buffer.ptr; - size_t pos = 0, towrite; - int nwritten = 0; - - while ((towrite = sdslen(buf)-pos) > 0) { - towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(fd,buf+nwritten,towrite,timeout); - if (nwritten != (signed)towrite) goto socket_wr_err; - pos += nwritten; - } - } - - /* Read back the reply. */ - { - char buf1[1024]; - char buf2[1024]; - - /* Read the two replies */ - if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0) - goto socket_rd_err; - if (syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0) - goto socket_rd_err; - if (buf1[0] == '-' || buf2[0] == '-') { - addReplyErrorFormat(c,"Target instance replied with error: %s", - (buf1[0] == '-') ? buf1+1 : buf2+1); - } else { - robj *aux; - - dbDelete(c->db,c->argv[3]); - signalModifiedKey(c->db,c->argv[3]); - addReply(c,shared.ok); - server.dirty++; - - /* Translate MIGRATE as DEL for replication/AOF. */ - aux = createStringObject("DEL",3); - rewriteClientCommandVector(c,2,aux,c->argv[3]); - decrRefCount(aux); - } - } - - sdsfree(cmd.io.buffer.ptr); - close(fd); - return; - -socket_wr_err: - redisLog(REDIS_NOTICE,"Can't write to target node for MIGRATE: %s", - strerror(errno)); - addReplyErrorFormat(c,"MIGRATE failed, writing to target node: %s.", - strerror(errno)); - sdsfree(cmd.io.buffer.ptr); - close(fd); - return; - -socket_rd_err: - redisLog(REDIS_NOTICE,"Can't read from target node for MIGRATE: %s", - strerror(errno)); - addReplyErrorFormat(c,"MIGRATE failed, reading from target node: %s.", - strerror(errno)); - sdsfree(cmd.io.buffer.ptr); - close(fd); - return; -} - -/* DUMP keyname - * DUMP is actually not used by Redis Cluster but it is the obvious - * complement of RESTORE and can be useful for different applications. */ -void dumpCommand(redisClient *c) { - robj *o, *dumpobj; - rio payload; - - /* Check if the key is here. */ - if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { - addReply(c,shared.nullbulk); - return; - } - - /* Serialize the object in a RDB-like format. It consist of an object type - * byte followed by the serialized object. This is understood by RESTORE. */ - rioInitWithBuffer(&payload,sdsempty()); - redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); - redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o)); - - /* Transfer to the client */ - dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); - return; -} - -/* The ASKING command is required after a -ASK redirection. - * The client should issue ASKING before to actualy send the command to - * the target instance. See the Redis Cluster specification for more - * information. */ -void askingCommand(redisClient *c) { - if (server.cluster_enabled == 0) { - addReplyError(c,"This instance has cluster support disabled"); - return; - } - c->flags |= REDIS_ASKING; - addReply(c,shared.ok); -} - -/* ----------------------------------------------------------------------------- - * Cluster functions related to serving / redirecting clients - * -------------------------------------------------------------------------- */ - -/* Return the pointer to the cluster node that is able to serve the query - * as all the keys belong to hash slots for which the node is in charge. - * - * If the returned node should be used only for this request, the *ask - * integer is set to '1', otherwise to '0'. This is used in order to - * let the caller know if we should reply with -MOVED or with -ASK. - * - * If the request contains more than a single key NULL is returned, - * however a request with more then a key argument where the key is always - * the same is valid, like in: RPOPLPUSH mylist mylist.*/ -clusterNode *getNodeByQuery(redisClient *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask) { - clusterNode *n = NULL; - robj *firstkey = NULL; - multiState *ms, _ms; - multiCmd mc; - int i, slot = 0; - - /* We handle all the cases as if they were EXEC commands, so we have - * a common code path for everything */ - if (cmd->proc == execCommand) { - /* If REDIS_MULTI flag is not set EXEC is just going to return an - * error. */ - if (!(c->flags & REDIS_MULTI)) return server.cluster.myself; - ms = &c->mstate; - } else { - /* In order to have a single codepath create a fake Multi State - * structure if the client is not in MULTI/EXEC state, this way - * we have a single codepath below. */ - ms = &_ms; - _ms.commands = &mc; - _ms.count = 1; - mc.argv = argv; - mc.argc = argc; - mc.cmd = cmd; - } - - /* Check that all the keys are the same key, and get the slot and - * node for this key. */ - for (i = 0; i < ms->count; i++) { - struct redisCommand *mcmd; - robj **margv; - int margc, *keyindex, numkeys, j; - - mcmd = ms->commands[i].cmd; - margc = ms->commands[i].argc; - margv = ms->commands[i].argv; - - keyindex = getKeysFromCommand(mcmd,margv,margc,&numkeys, - REDIS_GETKEYS_ALL); - for (j = 0; j < numkeys; j++) { - if (firstkey == NULL) { - /* This is the first key we see. Check what is the slot - * and node. */ - firstkey = margv[keyindex[j]]; - - slot = keyHashSlot((char*)firstkey->ptr, sdslen(firstkey->ptr)); - n = server.cluster.slots[slot]; - redisAssertWithInfo(c,firstkey,n != NULL); - } else { - /* If it is not the first key, make sure it is exactly - * the same key as the first we saw. */ - if (!equalStringObjects(firstkey,margv[keyindex[j]])) { - decrRefCount(firstkey); - getKeysFreeResult(keyindex); - return NULL; - } - } - } - getKeysFreeResult(keyindex); - } - if (ask) *ask = 0; /* This is the default. Set to 1 if needed later. */ - /* No key at all in command? then we can serve the request - * without redirections. */ - if (n == NULL) return server.cluster.myself; - if (hashslot) *hashslot = slot; - /* This request is about a slot we are migrating into another instance? - * Then we need to check if we have the key. If we have it we can reply. - * If instead is a new key, we pass the request to the node that is - * receiving the slot. */ - if (n == server.cluster.myself && - server.cluster.migrating_slots_to[slot] != NULL) - { - if (lookupKeyRead(&server.db[0],firstkey) == NULL) { - if (ask) *ask = 1; - return server.cluster.migrating_slots_to[slot]; - } - } - /* Handle the case in which we are receiving this hash slot from - * another instance, so we'll accept the query even if in the table - * it is assigned to a different node, but only if the client - * issued an ASKING command before. */ - if (server.cluster.importing_slots_from[slot] != NULL && - c->flags & REDIS_ASKING) { - return server.cluster.myself; - } - /* It's not a -ASK case. Base case: just return the right node. */ - return n; -} diff --git a/src/config.c b/src/config.c index 533a2a572a1..71a800a1230 100644 --- a/src/config.c +++ b/src/config.c @@ -298,13 +298,6 @@ void loadServerConfigFromString(char *config) { err = "Target command name already exists"; goto loaderr; } } - } else if (!strcasecmp(argv[0],"cluster-enabled") && argc == 2) { - if ((server.cluster_enabled = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } - } else if (!strcasecmp(argv[0],"cluster-config-file") && argc == 2) { - zfree(server.cluster.configfile); - server.cluster.configfile = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && diff --git a/src/db.c b/src/db.c index a0775af9650..7ca2a5f66f7 100644 --- a/src/db.c +++ b/src/db.c @@ -86,7 +86,6 @@ void dbAdd(redisDb *db, robj *key, robj *val) { int retval = dictAdd(db->dict, copy, val); redisAssertWithInfo(NULL,key,retval == REDIS_OK); - if (server.cluster_enabled) SlotToKeyAdd(key); } /* Overwrite an existing key with a new value. Incrementing the reference @@ -154,7 +153,6 @@ int dbDelete(redisDb *db, robj *key) { * the key, because it is shared with the main dictionary. */ if (dictSize(db->expires) > 0) dictDelete(db->expires,key->ptr); if (dictDelete(db->dict,key->ptr) == DICT_OK) { - if (server.cluster_enabled) SlotToKeyDel(key); return 1; } else { return 0; @@ -254,10 +252,6 @@ void existsCommand(redisClient *c) { void selectCommand(redisClient *c) { int id = atoi(c->argv[1]->ptr); - if (server.cluster_enabled && id != 0) { - addReplyError(c,"SELECT is not allowed in cluster mode"); - return; - } if (selectDb(c,id) == REDIS_ERR) { addReplyError(c,"invalid DB index"); } else { @@ -398,11 +392,6 @@ void moveCommand(redisClient *c) { redisDb *src, *dst; int srcid; - if (server.cluster_enabled) { - addReplyError(c,"MOVE is not allowed in cluster mode"); - return; - } - /* Obtain source and target DB pointers */ src = c->db; srcid = c->db->id; @@ -714,35 +703,3 @@ int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *num *numkeys = num; return keys; } - -/* Slot to Key API. This is used by Redis Cluster in order to obtain in - * a fast way a key that belongs to a specified hash slot. This is useful - * while rehashing the cluster. */ -void SlotToKeyAdd(robj *key) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); - - zslInsert(server.cluster.slots_to_keys,hashslot,key); - incrRefCount(key); -} - -void SlotToKeyDel(robj *key) { - unsigned int hashslot = keyHashSlot(key->ptr,sdslen(key->ptr)); - - zslDelete(server.cluster.slots_to_keys,hashslot,key); -} - -unsigned int GetKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count) { - zskiplistNode *n; - zrangespec range; - int j = 0; - - range.min = range.max = hashslot; - range.minex = range.maxex = 0; - - n = zslFirstInRange(server.cluster.slots_to_keys, range); - while(n && n->score == hashslot && count--) { - keys[j++] = n->obj; - n = n->level[0].forward; - } - return j; -} diff --git a/src/migrate.c b/src/migrate.c new file mode 100644 index 00000000000..f7a5b730998 --- /dev/null +++ b/src/migrate.c @@ -0,0 +1,190 @@ +#include "redis.h" + +/* ----------------------------------------------------------------------------- + * RESTORE and MIGRATE commands + * -------------------------------------------------------------------------- */ + +/* RESTORE key ttl serialized-value */ +void restoreCommand(redisClient *c) { + long ttl; + rio payload; + int type; + robj *obj; + + /* Make sure this key does not already exist here... */ + if (lookupKeyWrite(c->db,c->argv[1]) != NULL) { + addReplyError(c,"Target key name is busy."); + return; + } + + /* Check if the TTL value makes sense */ + if (getLongFromObjectOrReply(c,c->argv[2],&ttl,NULL) != REDIS_OK) { + return; + } else if (ttl < 0) { + addReplyError(c,"Invalid TTL value, must be >= 0"); + return; + } + + rioInitWithBuffer(&payload,c->argv[3]->ptr); + if (((type = rdbLoadObjectType(&payload)) == -1) || + ((obj = rdbLoadObject(type,&payload)) == NULL)) + { + addReplyError(c,"Bad data format"); + return; + } + + /* Create the key and set the TTL if any */ + dbAdd(c->db,c->argv[1],obj); + if (ttl) setExpire(c->db,c->argv[1],time(NULL)+ttl); + signalModifiedKey(c->db,c->argv[1]); + addReply(c,shared.ok); + server.dirty++; +} + +/* MIGRATE host port key dbid timeout */ +void migrateCommand(redisClient *c) { + int fd; + long timeout; + long dbid; + time_t ttl; + robj *o; + rio cmd, payload; + + /* Sanity check */ + if (getLongFromObjectOrReply(c,c->argv[5],&timeout,NULL) != REDIS_OK) + return; + if (getLongFromObjectOrReply(c,c->argv[4],&dbid,NULL) != REDIS_OK) + return; + if (timeout <= 0) timeout = 1; + + /* Check if the key is here. If not we reply with success as there is + * nothing to migrate (for instance the key expired in the meantime), but + * we include such information in the reply string. */ + if ((o = lookupKeyRead(c->db,c->argv[3])) == NULL) { + addReplySds(c,sdsnew("+NOKEY\r\n")); + return; + } + + /* Connect */ + fd = anetTcpNonBlockConnect(server.neterr,c->argv[1]->ptr, + atoi(c->argv[2]->ptr)); + if (fd == -1) { + addReplyErrorFormat(c,"Can't connect to target node: %s", + server.neterr); + return; + } + if ((aeWait(fd,AE_WRITABLE,timeout*1000) & AE_WRITABLE) == 0) { + addReplyError(c,"Timeout connecting to the client"); + return; + } + + rioInitWithBuffer(&cmd,sdsempty()); + redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); + redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); + + ttl = getExpire(c->db,c->argv[3]); + redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',4)); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); + redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr))); + redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl)); + + /* Finally the last argument that is the serailized object payload + * in the form: . */ + rioInitWithBuffer(&payload,sdsempty()); + redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); + redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o) != -1); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr))); + sdsfree(payload.io.buffer.ptr); + + /* Tranfer the query to the other node in 64K chunks. */ + { + sds buf = cmd.io.buffer.ptr; + size_t pos = 0, towrite; + int nwritten = 0; + + while ((towrite = sdslen(buf)-pos) > 0) { + towrite = (towrite > (64*1024) ? (64*1024) : towrite); + nwritten = syncWrite(fd,buf+nwritten,towrite,timeout); + if (nwritten != (signed)towrite) goto socket_wr_err; + pos += nwritten; + } + } + + /* Read back the reply. */ + { + char buf1[1024]; + char buf2[1024]; + + /* Read the two replies */ + if (syncReadLine(fd, buf1, sizeof(buf1), timeout) <= 0) + goto socket_rd_err; + if (syncReadLine(fd, buf2, sizeof(buf2), timeout) <= 0) + goto socket_rd_err; + if (buf1[0] == '-' || buf2[0] == '-') { + addReplyErrorFormat(c,"Target instance replied with error: %s", + (buf1[0] == '-') ? buf1+1 : buf2+1); + } else { + robj *aux; + + dbDelete(c->db,c->argv[3]); + signalModifiedKey(c->db,c->argv[3]); + addReply(c,shared.ok); + server.dirty++; + + /* Translate MIGRATE as DEL for replication/AOF. */ + aux = createStringObject("DEL",3); + rewriteClientCommandVector(c,2,aux,c->argv[3]); + decrRefCount(aux); + } + } + + sdsfree(cmd.io.buffer.ptr); + close(fd); + return; + +socket_wr_err: + redisLog(REDIS_NOTICE,"Can't write to target node for MIGRATE: %s", + strerror(errno)); + addReplyErrorFormat(c,"MIGRATE failed, writing to target node: %s.", + strerror(errno)); + sdsfree(cmd.io.buffer.ptr); + close(fd); + return; + +socket_rd_err: + redisLog(REDIS_NOTICE,"Can't read from target node for MIGRATE: %s", + strerror(errno)); + addReplyErrorFormat(c,"MIGRATE failed, reading from target node: %s.", + strerror(errno)); + sdsfree(cmd.io.buffer.ptr); + close(fd); + return; +} + +/* DUMP keyname + * DUMP is actually not used by Redis Cluster but it is the obvious + * complement of RESTORE and can be useful for different applications. */ +void dumpCommand(redisClient *c) { + robj *o, *dumpobj; + rio payload; + + /* Check if the key is here. */ + if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { + addReply(c,shared.nullbulk); + return; + } + + /* Serialize the object in a RDB-like format. It consist of an object type + * byte followed by the serialized object. This is understood by RESTORE. */ + rioInitWithBuffer(&payload,sdsempty()); + redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); + redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o)); + + /* Transfer to the client */ + dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr); + addReplyBulk(c,dumpobj); + decrRefCount(dumpobj); + return; +} diff --git a/src/pubsub.c b/src/pubsub.c index 984013bbc46..4e07dbbab1b 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -263,6 +263,5 @@ void punsubscribeCommand(redisClient *c) { void publishCommand(redisClient *c) { int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]); - if (server.cluster_enabled) clusterPropagatePublish(c->argv[1],c->argv[2]); addReplyLongLong(c,receivers); } diff --git a/src/redis.c b/src/redis.c index 3294eea4383..70c7cf0ce6a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -232,10 +232,8 @@ struct redisCommand redisCommandTable[] = { {"publish",publishCommand,3,"rpf",0,NULL,0,0,0,0,0}, {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0}, {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0}, - {"cluster",clusterCommand,-2,"ar",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,4,"awm",0,NULL,1,1,1,0,0}, {"migrate",migrateCommand,6,"aw",0,NULL,0,0,0,0,0}, - {"asking",askingCommand,1,"r",0,NULL,0,0,0,0,0}, {"dump",dumpCommand,2,"ar",0,NULL,1,1,1,0,0}, {"object",objectCommand,-2,"r",0,NULL,2,2,2,0,0}, {"client",clientCommand,-2,"ar",0,NULL,0,0,0,0,0}, @@ -507,17 +505,6 @@ dictType keylistDictType = { dictListDestructor /* val destructor */ }; -/* Cluster nodes hash table, mapping nodes addresses 1.2.3.4:6379 to - * clusterNode structures. */ -dictType clusterNodesDictType = { - dictSdsHash, /* hash function */ - NULL, /* key dup */ - NULL, /* val dup */ - dictSdsKeyCompare, /* key compare */ - dictSdsDestructor, /* key destructor */ - NULL /* val destructor */ -}; - int htNeedsResize(dict *dict) { long long size, used; @@ -792,9 +779,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * to detect transfer failures. */ if (!(loops % 10)) replicationCron(); - /* Run other sub-systems specific cron jobs */ - if (server.cluster_enabled && !(loops % 10)) clusterCron(); - server.cronloops++; return 100; } @@ -949,8 +933,6 @@ void initServerConfig() { server.shutdown_asap = 0; server.repl_ping_slave_period = REDIS_REPL_PING_SLAVE_PERIOD; server.repl_timeout = REDIS_REPL_TIMEOUT; - server.cluster_enabled = 0; - server.cluster.configfile = zstrdup("nodes.conf"); server.lua_caller = NULL; server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; @@ -1151,7 +1133,6 @@ void initServer() { server.maxmemory_policy = REDIS_MAXMEMORY_NO_EVICTION; } - if (server.cluster_enabled) clusterInit(); scriptingInit(); slowlogInit(); bioInit(); @@ -1374,29 +1355,6 @@ int processCommand(redisClient *c) { return REDIS_OK; } - /* If cluster is enabled, redirect here */ - if (server.cluster_enabled && - !(c->cmd->getkeys_proc == NULL && c->cmd->firstkey == 0)) { - int hashslot; - - if (server.cluster.state != REDIS_CLUSTER_OK) { - addReplyError(c,"The cluster is down. Check with CLUSTER INFO for more information"); - return REDIS_OK; - } else { - int ask; - clusterNode *n = getNodeByQuery(c,c->cmd,c->argv,c->argc,&hashslot,&ask); - if (n == NULL) { - addReplyError(c,"Multi keys request invalid in cluster"); - return REDIS_OK; - } else if (n != server.cluster.myself) { - addReplySds(c,sdscatprintf(sdsempty(), - "-%s %d %s:%d\r\n", ask ? "ASK" : "MOVED", - hashslot,n->ip,n->port)); - return REDIS_OK; - } - } - } - /* Handle the maxmemory directive. * * First we try to free some memory if possible (if there are volatile @@ -1884,15 +1842,6 @@ sds genRedisInfoString(char *section) { } } - /* Clusetr */ - if (allsections || defsections || !strcasecmp(section,"cluster")) { - if (sections++) info = sdscat(info,"\r\n"); - info = sdscatprintf(info, - "# Cluster\r\n" - "cluster_enabled:%d\r\n", - server.cluster_enabled); - } - /* Key space */ if (allsections || defsections || !strcasecmp(section,"keyspace")) { if (sections++) info = sdscat(info,"\r\n"); @@ -2172,7 +2121,7 @@ void redisAsciiArt(void) { redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, (sizeof(long) == 8) ? "64" : "32", - server.cluster_enabled ? "cluster" : "stand alone", + "stand alone", server.port, (long) getpid() ); diff --git a/src/redis.h b/src/redis.h index 6ead029d8dc..e47a41a4c7c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -425,134 +425,6 @@ typedef struct redisOpArray { int numops; } redisOpArray; -/*----------------------------------------------------------------------------- - * Redis cluster data structures - *----------------------------------------------------------------------------*/ - -#define REDIS_CLUSTER_SLOTS 4096 -#define REDIS_CLUSTER_OK 0 /* Everything looks ok */ -#define REDIS_CLUSTER_FAIL 1 /* The cluster can't work */ -#define REDIS_CLUSTER_NEEDHELP 2 /* The cluster works, but needs some help */ -#define REDIS_CLUSTER_NAMELEN 40 /* sha1 hex length */ -#define REDIS_CLUSTER_PORT_INCR 10000 /* Cluster port = baseport + PORT_INCR */ - -struct clusterNode; - -/* clusterLink encapsulates everything needed to talk with a remote node. */ -typedef struct clusterLink { - int fd; /* TCP socket file descriptor */ - sds sndbuf; /* Packet send buffer */ - sds rcvbuf; /* Packet reception buffer */ - struct clusterNode *node; /* Node related to this link if any, or NULL */ -} clusterLink; - -/* Node flags */ -#define REDIS_NODE_MASTER 1 /* The node is a master */ -#define REDIS_NODE_SLAVE 2 /* The node is a slave */ -#define REDIS_NODE_PFAIL 4 /* Failure? Need acknowledge */ -#define REDIS_NODE_FAIL 8 /* The node is believed to be malfunctioning */ -#define REDIS_NODE_MYSELF 16 /* This node is myself */ -#define REDIS_NODE_HANDSHAKE 32 /* We have still to exchange the first ping */ -#define REDIS_NODE_NOADDR 64 /* We don't know the address of this node */ -#define REDIS_NODE_MEET 128 /* Send a MEET message to this node */ -#define REDIS_NODE_NULL_NAME "\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" - -struct clusterNode { - char name[REDIS_CLUSTER_NAMELEN]; /* Node name, hex string, sha1-size */ - int flags; /* REDIS_NODE_... */ - unsigned char slots[REDIS_CLUSTER_SLOTS/8]; /* slots handled by this node */ - int numslaves; /* Number of slave nodes, if this is a master */ - struct clusterNode **slaves; /* pointers to slave nodes */ - struct clusterNode *slaveof; /* pointer to the master node */ - time_t ping_sent; /* Unix time we sent latest ping */ - time_t pong_received; /* Unix time we received the pong */ - char *configdigest; /* Configuration digest of this node */ - time_t configdigest_ts; /* Configuration digest timestamp */ - char ip[16]; /* Latest known IP address of this node */ - int port; /* Latest known port of this node */ - clusterLink *link; /* TCP/IP link with this node */ -}; -typedef struct clusterNode clusterNode; - -typedef struct { - char *configfile; - clusterNode *myself; /* This node */ - int state; /* REDIS_CLUSTER_OK, REDIS_CLUSTER_FAIL, ... */ - int node_timeout; - dict *nodes; /* Hash table of name -> clusterNode structures */ - clusterNode *migrating_slots_to[REDIS_CLUSTER_SLOTS]; - clusterNode *importing_slots_from[REDIS_CLUSTER_SLOTS]; - clusterNode *slots[REDIS_CLUSTER_SLOTS]; - zskiplist *slots_to_keys; -} clusterState; - -/* Redis cluster messages header */ - -/* Note that the PING, PONG and MEET messages are actually the same exact - * kind of packet. PONG is the reply to ping, in the extact format as a PING, - * while MEET is a special PING that forces the receiver to add the sender - * as a node (if it is not already in the list). */ -#define CLUSTERMSG_TYPE_PING 0 /* Ping */ -#define CLUSTERMSG_TYPE_PONG 1 /* Pong (reply to Ping) */ -#define CLUSTERMSG_TYPE_MEET 2 /* Meet "let's join" message */ -#define CLUSTERMSG_TYPE_FAIL 3 /* Mark node xxx as failing */ -#define CLUSTERMSG_TYPE_PUBLISH 4 /* Pub/Sub Publish propatagion */ - -/* Initially we don't know our "name", but we'll find it once we connect - * to the first node, using the getsockname() function. Then we'll use this - * address for all the next messages. */ -typedef struct { - char nodename[REDIS_CLUSTER_NAMELEN]; - uint32_t ping_sent; - uint32_t pong_received; - char ip[16]; /* IP address last time it was seen */ - uint16_t port; /* port last time it was seen */ - uint16_t flags; - uint32_t notused; /* for 64 bit alignment */ -} clusterMsgDataGossip; - -typedef struct { - char nodename[REDIS_CLUSTER_NAMELEN]; -} clusterMsgDataFail; - -typedef struct { - uint32_t channel_len; - uint32_t message_len; - unsigned char bulk_data[8]; /* defined as 8 just for alignment concerns. */ -} clusterMsgDataPublish; - -union clusterMsgData { - /* PING, MEET and PONG */ - struct { - /* Array of N clusterMsgDataGossip structures */ - clusterMsgDataGossip gossip[1]; - } ping; - - /* FAIL */ - struct { - clusterMsgDataFail about; - } fail; - - /* PUBLISH */ - struct { - clusterMsgDataPublish msg; - } publish; -}; - -typedef struct { - uint32_t totlen; /* Total length of this message */ - uint16_t type; /* Message type */ - uint16_t count; /* Only used for some kind of messages. */ - char sender[REDIS_CLUSTER_NAMELEN]; /* Name of the sender node */ - unsigned char myslots[REDIS_CLUSTER_SLOTS/8]; - char slaveof[REDIS_CLUSTER_NAMELEN]; - char configdigest[32]; - uint16_t port; /* Sender TCP base port */ - unsigned char state; /* Cluster state from the POV of the sender */ - unsigned char notused[5]; /* Reserved for future use. For alignment. */ - union clusterMsgData data; -} clusterMsg; - /*----------------------------------------------------------------------------- * Global server state *----------------------------------------------------------------------------*/ @@ -578,7 +450,6 @@ struct redisServer { mode_t unixsocketperm; /* UNIX socket permission */ int ipfd; /* TCP socket file descriptor */ int sofd; /* Unix socket file descriptor */ - int cfd; /* Cluster bus lisetning socket */ list *clients; /* List of active clients */ list *clients_to_close; /* Clients to close asynchronously */ list *slaves, *monitors; /* List of slaves and MONITORs */ @@ -696,9 +567,6 @@ struct redisServer { /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ - /* Cluster */ - int cluster_enabled; /* Is cluster enabled? */ - clusterState cluster; /* State of the cluster */ /* Scripting */ lua_State *lua; /* The Lua interpreter. We use just one for all clients */ redisClient *lua_client; /* The "fake client" to query Redis from Lua */ @@ -733,8 +601,7 @@ struct redisCommand { int arity; char *sflags; /* Flags as string represenation, one char per flag. */ int flags; /* The actual flags, obtained from the 'sflags' field. */ - /* Use a function to determine keys arguments in a command line. - * Used for Redis Cluster redirect. */ + /* Use a function to determine keys arguments in a command line. */ redisGetKeysProc *getkeys_proc; /* What keys should be loaded in background when calling this command? */ int firstkey; /* The first argument that's a key (0 = no keys) */ @@ -810,7 +677,6 @@ extern struct redisServer server; extern struct sharedObjectsStruct shared; extern dictType setDictType; extern dictType zsetDictType; -extern dictType clusterNodesDictType; extern dictType dbDictType; extern double R_Zero, R_PosInf, R_NegInf, R_Nan; dictType hashDictType; @@ -1082,16 +948,6 @@ int *noPreloadGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numke int *renameGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numkeys, int flags); int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *numkeys, int flags); -/* Cluster */ -void clusterInit(void); -unsigned short crc16(const char *buf, int len); -unsigned int keyHashSlot(char *key, int keylen); -clusterNode *createClusterNode(char *nodename, int flags); -int clusterAddNode(clusterNode *node); -void clusterCron(void); -clusterNode *getNodeByQuery(redisClient *c, struct redisCommand *cmd, robj **argv, int argc, int *hashslot, int *ask); -void clusterPropagatePublish(robj *channel, robj *message); - /* Scripting */ void scriptingInit(void); @@ -1223,10 +1079,8 @@ void punsubscribeCommand(redisClient *c); void publishCommand(redisClient *c); void watchCommand(redisClient *c); void unwatchCommand(redisClient *c); -void clusterCommand(redisClient *c); void restoreCommand(redisClient *c); void migrateCommand(redisClient *c); -void askingCommand(redisClient *c); void dumpCommand(redisClient *c); void objectCommand(redisClient *c); void clientCommand(redisClient *c); From 00b33363343b2dc7e8ee107090c05ba56c635538 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 10 Mar 2012 12:28:14 +0100 Subject: [PATCH 0002/1752] Build dependencies updated. --- src/Makefile | 61 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 26 deletions(-) diff --git a/src/Makefile b/src/Makefile index 8677b60e7e2..18df4a20509 100644 --- a/src/Makefile +++ b/src/Makefile @@ -98,33 +98,39 @@ ae_kqueue.o: ae_kqueue.c ae_select.o: ae_select.c anet.o: anet.c fmacros.h anet.h aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h bio.o: bio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h bio.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h sha1.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h dict.o: dict.c fmacros.h dict.h zmalloc.h endianconv.o: endianconv.c intset.o: intset.c intset.h zmalloc.h endianconv.h lzf_c.o: lzf_c.c lzfP.h lzf_d.o: lzf_d.c lzfP.h +migrate.o: migrate.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h pqsort.o: pqsort.c pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +rand.o: rand.c rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h lzf.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h lzf.h \ + zipmap.h redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h redis-check-aof.o: redis-check-aof.c fmacros.h config.h @@ -132,34 +138,37 @@ redis-check-dump.o: redis-check-dump.c lzf.h redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h slowlog.h \ - bio.h asciilogo.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ + slowlog.h bio.h asciilogo.h release.o: release.c release.h replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h +rio.o: rio.c fmacros.h rio.h sds.h util.h scripting.o: scripting.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ - sha1.h rand.h -rio.o: rio.c sds.h + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h sha1.h rand.h sds.o: sds.c sds.h zmalloc.h sha1.o: sha1.c sha1.h config.h slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h \ - slowlog.h + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h slowlog.h sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h pqsort.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ + pqsort.h syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h zipmap.h ziplist.h intset.h version.h util.h + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h util.o: util.c fmacros.h util.h ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h zipmap.o: zipmap.c zmalloc.h endianconv.h From 7551f2a0b19e5cf444bf6fed3b8ed5b2c936a228 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 10 Mar 2012 12:29:47 +0100 Subject: [PATCH 0003/1752] Version is now 2.5.1, first unstable release of Redis 2.6 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 9ef74b08025..fd60d69bff9 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.9.5" +#define REDIS_VERSION "2.5.1" From 37180ed9cd6a761aedbd64a2e73e3ece8c98d937 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 10 Mar 2012 12:35:15 +0100 Subject: [PATCH 0004/1752] RDB version is no 4, because small hashes are now encoded as ziplists, so older versions of Redis will not understand this format. --- src/rdb.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 113856d4320..518fef02c10 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -616,7 +616,7 @@ int rdbSave(char *filename) { } rioInitWithFile(&rdb,fp); - if (rdbWriteRaw(&rdb,"REDIS0003",9) == -1) goto werr; + if (rdbWriteRaw(&rdb,"REDIS0004",9) == -1) goto werr; for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; @@ -1023,7 +1023,7 @@ int rdbLoad(char *filename) { return REDIS_ERR; } rdbver = atoi(buf+5); - if (rdbver < 1 || rdbver > 3) { + if (rdbver < 1 || rdbver > 4) { fclose(fp); redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); errno = EINVAL; From b014c1f21119cdedd9a0599d0e82e00035add64b Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 10 Mar 2012 12:38:42 +0100 Subject: [PATCH 0005/1752] RDB4 support in redis-check-dump. --- src/redis-check-dump.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis-check-dump.c b/src/redis-check-dump.c index 77be686c245..42fe5181087 100644 --- a/src/redis-check-dump.c +++ b/src/redis-check-dump.c @@ -20,6 +20,7 @@ #define REDIS_LIST_ZIPLIST 10 #define REDIS_SET_INTSET 11 #define REDIS_ZSET_ZIPLIST 12 +#define REDIS_HASH_ZIPLIST 13 /* Objects encoding. Some kind of objects like Strings and Hashes can be * internally represented in multiple ways. The 'encoding' field of the object @@ -136,7 +137,7 @@ int processHeader() { } dump_version = (int)strtol(buf + 5, NULL, 10); - if (dump_version < 1 || dump_version > 2) { + if (dump_version < 1 || dump_version > 4) { ERROR("Unknown RDB format version: %d\n", dump_version); } return 1; @@ -384,6 +385,7 @@ int loadPair(entry *e) { case REDIS_LIST_ZIPLIST: case REDIS_SET_INTSET: case REDIS_ZSET_ZIPLIST: + case REDIS_HASH_ZIPLIST: if (!processStringObject(NULL)) { SHIFT_ERROR(offset, "Error reading entry value"); return 0; From dfc25454701638d39bc75fbc3f9c55619b503bd0 Mon Sep 17 00:00:00 2001 From: quiver Date: Sat, 10 Mar 2012 21:06:08 +0900 Subject: [PATCH 0006/1752] fix typo of redis.conf --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 80e14ad953d..e03359963ba 100644 --- a/redis.conf +++ b/redis.conf @@ -246,7 +246,7 @@ slave-serve-stale-data yes # volatile-lru -> remove the key with an expire set using an LRU algorithm # allkeys-lru -> remove any key accordingly to the LRU algorithm # volatile-random -> remove a random key with an expire set -# allkeys->random -> remove a random key, any key +# allkeys-random -> remove a random key, any key # volatile-ttl -> remove the key with the nearest expire time (minor TTL) # noeviction -> don't expire at all, just return an error on write operations # From ee61a4b99ee56203edb746c666fa89f88c2d0afa Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 09:49:11 +0100 Subject: [PATCH 0007/1752] RDB hashes loading fixed removing the assertion that failed every time an HT-encoded hash was loaded. --- src/rdb.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 518fef02c10..4a56659de07 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -844,9 +844,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { hashTypeConvert(o, REDIS_ENCODING_HT); /* Load every field and value into the ziplist */ - while (o->encoding == REDIS_ENCODING_ZIPLIST && len-- > 0) { + while (o->encoding == REDIS_ENCODING_ZIPLIST && len > 0) { robj *field, *value; + len--; /* Load raw strings */ field = rdbLoadStringObject(rdb); if (field == NULL) return NULL; @@ -869,9 +870,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { } /* Load remaining fields and values into the hash table */ - while (o->encoding == REDIS_ENCODING_HT && len-- > 0) { + while (o->encoding == REDIS_ENCODING_HT && len > 0) { robj *field, *value; + len--; /* Load encoded strings */ field = rdbLoadEncodedStringObject(rdb); if (field == NULL) return NULL; From a74ab6478ca0d59b9c5f0b92a0ac68a69e56c80a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 10:59:29 +0100 Subject: [PATCH 0008/1752] RDB hashes loading, fixed another bug in the loading of HT-encoded hashes: when the hash entry is too big for ziplist, add the field, then convert. The code used to break before the new entry was inserted, resulting into missing fields in the loaded Hash object. --- src/rdb.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 4a56659de07..519b645d157 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -856,6 +856,10 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { if (value == NULL) return NULL; redisAssert(field->encoding == REDIS_ENCODING_RAW); + /* Add pair to ziplist */ + o->ptr = ziplistPush(o->ptr, field->ptr, sdslen(field->ptr), ZIPLIST_TAIL); + o->ptr = ziplistPush(o->ptr, value->ptr, sdslen(value->ptr), ZIPLIST_TAIL); + /* Convert to hash table if size threshold is exceeded */ if (sdslen(field->ptr) > server.hash_max_ziplist_value || sdslen(value->ptr) > server.hash_max_ziplist_value) @@ -863,10 +867,6 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { hashTypeConvert(o, REDIS_ENCODING_HT); break; } - - /* Add pair to ziplist */ - o->ptr = ziplistPush(o->ptr, field->ptr, sdslen(field->ptr), ZIPLIST_TAIL); - o->ptr = ziplistPush(o->ptr, value->ptr, sdslen(value->ptr), ZIPLIST_TAIL); } /* Load remaining fields and values into the hash table */ From 56de4964ee716729d0b2a03ceef606235e46bb88 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 12:59:27 +0100 Subject: [PATCH 0009/1752] c->bufpos initialization moved for aesthetics. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 40aad83606a..14c5dcce624 100644 --- a/src/networking.c +++ b/src/networking.c @@ -22,7 +22,6 @@ int listMatchObjects(void *a, void *b) { redisClient *createClient(int fd) { redisClient *c = zmalloc(sizeof(redisClient)); - c->bufpos = 0; /* passing -1 as fd it is possible to create a non connected client. * This is useful since all the Redis commands needs to be executed @@ -42,6 +41,7 @@ redisClient *createClient(int fd) { selectDb(c,0); c->fd = fd; + c->bufpos = 0; c->querybuf = sdsempty(); c->reqtype = 0; c->argc = 0; From 41e8e5cb8ec9c4e6ca79750338ad98c4e234a697 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 13:05:08 +0100 Subject: [PATCH 0010/1752] Client creation time in redisClient structure. New age field in CLIENT LIST output. --- src/networking.c | 5 +++-- src/redis.h | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 14c5dcce624..b0f4505446f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -51,7 +51,7 @@ redisClient *createClient(int fd) { c->bulklen = -1; c->sentlen = 0; c->flags = 0; - c->lastinteraction = time(NULL); + c->ctime = c->lastinteraction = time(NULL); c->authenticated = 0; c->replstate = REDIS_REPL_NONE; c->reply = listCreate(); @@ -1111,8 +1111,9 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatprintf(sdsempty(), - "addr=%s:%d fd=%d idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", + "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", ip,port,client->fd, + (long)(now - client->ctime), (long)(now - client->lastinteraction), flags, client->db->id, diff --git a/src/redis.h b/src/redis.h index e47a41a4c7c..28649f6d638 100644 --- a/src/redis.h +++ b/src/redis.h @@ -332,6 +332,7 @@ typedef struct redisClient { list *reply; unsigned long reply_bytes; /* Tot bytes of objects in reply list */ int sentlen; + time_t ctime; /* Client creation time */ time_t lastinteraction; /* time of the last interaction, used for timeout */ time_t obuf_soft_limit_reached_time; int flags; /* REDIS_SLAVE | REDIS_MONITOR | REDIS_MULTI ... */ From 57a5e54ddcbaf0df15771faad3344fa6990e2227 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 13:26:33 +0100 Subject: [PATCH 0011/1752] Added a qbuf-free field to CLIENT LIST output. --- src/networking.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index b0f4505446f..06097b58686 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1111,7 +1111,7 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatprintf(sdsempty(), - "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", + "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", ip,port,client->fd, (long)(now - client->ctime), (long)(now - client->lastinteraction), @@ -1120,6 +1120,7 @@ sds getClientInfoString(redisClient *client) { (int) dictSize(client->pubsub_channels), (int) listLength(client->pubsub_patterns), (unsigned long) sdslen(client->querybuf), + (unsigned long) sdsavail(client->querybuf), (unsigned long) client->bufpos, (unsigned long) listLength(client->reply), getClientOutputBufferMemoryUsage(client), From cfa4b57cb0bc52e5c5207594a9c1e436b67fb9b4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 18:05:11 +0100 Subject: [PATCH 0012/1752] Process async client checks like client timeouts and BLPOP timeouts incrementally using a circular list. --- src/adlist.c | 16 ++++++++++++++++ src/adlist.h | 1 + src/networking.c | 28 --------------------------- src/redis.c | 49 +++++++++++++++++++++++++++++++++++++++++++++--- 4 files changed, 63 insertions(+), 31 deletions(-) diff --git a/src/adlist.c b/src/adlist.c index 51ba03bd5e0..e48957e3af9 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -323,3 +323,19 @@ listNode *listIndex(list *list, long index) { } return n; } + +/* Rotate the list removing the tail node and inserting it to the head. */ +void listRotate(list *list) { + listNode *tail = list->tail; + + if (listLength(list) <= 1) return; + + /* Detatch current tail */ + list->tail = tail->prev; + list->tail->next = NULL; + /* Move it as head */ + list->head->prev = tail; + tail->prev = NULL; + tail->next = list->head; + list->head = tail; +} diff --git a/src/adlist.h b/src/adlist.h index 36dba1ff324..259bd0f8380 100644 --- a/src/adlist.h +++ b/src/adlist.h @@ -84,6 +84,7 @@ listNode *listSearchKey(list *list, void *key); listNode *listIndex(list *list, long index); void listRewind(list *list, listIter *li); void listRewindTail(list *list, listIter *li); +void listRotate(list *list); /* Directions for iterators */ #define AL_START_HEAD 0 diff --git a/src/networking.c b/src/networking.c index 06097b58686..ae77d11b231 100644 --- a/src/networking.c +++ b/src/networking.c @@ -751,34 +751,6 @@ void resetClient(redisClient *c) { if (!(c->flags & REDIS_MULTI)) c->flags &= (~REDIS_ASKING); } -void closeTimedoutClients(void) { - redisClient *c; - listNode *ln; - time_t now = time(NULL); - listIter li; - - listRewind(server.clients,&li); - while ((ln = listNext(&li)) != NULL) { - c = listNodeValue(ln); - if (server.maxidletime && - !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ - !(c->flags & REDIS_MASTER) && /* no timeout for masters */ - !(c->flags & REDIS_BLOCKED) && /* no timeout for BLPOP */ - dictSize(c->pubsub_channels) == 0 && /* no timeout for pubsub */ - listLength(c->pubsub_patterns) == 0 && - (now - c->lastinteraction > server.maxidletime)) - { - redisLog(REDIS_VERBOSE,"Closing idle client"); - freeClient(c); - } else if (c->flags & REDIS_BLOCKED) { - if (c->bpop.timeout != 0 && c->bpop.timeout < now) { - addReply(c,shared.nullmultibulk); - unblockClientWaitingData(c); - } - } - } -} - int processInlineBuffer(redisClient *c) { char *newline = strstr(c->querybuf,"\r\n"); int argc, j; diff --git a/src/redis.c b/src/redis.c index 70c7cf0ce6a..1b576e047cb 100644 --- a/src/redis.c +++ b/src/redis.c @@ -628,6 +628,50 @@ long long getOperationsPerSecond(void) { return sum / REDIS_OPS_SEC_SAMPLES; } +void closeTimedoutClient(redisClient *c) { + time_t now = time(NULL); + + if (server.maxidletime && + !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ + !(c->flags & REDIS_MASTER) && /* no timeout for masters */ + !(c->flags & REDIS_BLOCKED) && /* no timeout for BLPOP */ + dictSize(c->pubsub_channels) == 0 && /* no timeout for pubsub */ + listLength(c->pubsub_patterns) == 0 && + (now - c->lastinteraction > server.maxidletime)) + { + redisLog(REDIS_VERBOSE,"Closing idle client"); + freeClient(c); + } else if (c->flags & REDIS_BLOCKED) { + if (c->bpop.timeout != 0 && c->bpop.timeout < now) { + addReply(c,shared.nullmultibulk); + unblockClientWaitingData(c); + } + } +} + +void clientsCron(void) { + /* Make sure to process at least 1/100 of clients per call. + * Since this function is called 10 times per second we are sure that + * in the worst case we process all the clients in 10 seconds. + * In normal conditions (a reasonable number of clients) we process + * all the clients in a shorter time. */ + int iterations = listLength(server.clients)/100; + if (iterations < 50) iterations = 50; + + while(listLength(server.clients) && iterations--) { + redisClient *c; + listNode *head; + + /* Rotate the list, take the current head, process. + * This way if the client must be removed from the list it's the + * first element and we don't incur into O(N) computation. */ + listRotate(server.clients); + head = listFirst(server.clients); + c = listNodeValue(head); + closeTimedoutClient(c); + } +} + int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { int j, loops = server.cronloops; REDIS_NOTUSED(eventLoop); @@ -699,9 +743,8 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { zmalloc_used_memory()); } - /* Close connections of timedout clients */ - if ((server.maxidletime && !(loops % 100)) || server.bpop_blocked_clients) - closeTimedoutClients(); + /* We need to do a few operations on clients asynchronously. */ + clientsCron(); /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ From 3e8fcb6d031b092677191ffdd4808c4eb4975270 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 13 Mar 2012 18:06:29 +0100 Subject: [PATCH 0013/1752] CLIENT LIST test modified to reflect the new output. --- tests/unit/introspection.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 3daa65e0755..a768e2ab98b 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1,5 +1,5 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list - } {*addr=*:* fd=* idle=* flags=N db=9 sub=0 psub=0 qbuf=0 obl=0 oll=0 omem=0 events=r cmd=client*} + } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} } From 5e473cd8fd461b64dc4909ec5b611ac2f1acbf2a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 14 Mar 2012 09:56:22 +0100 Subject: [PATCH 0014/1752] Call all the helper functions needed by clientsCron() as clientsCronSomething() for clarity. --- src/redis.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 1b576e047cb..58b5f1c4bf2 100644 --- a/src/redis.c +++ b/src/redis.c @@ -628,7 +628,7 @@ long long getOperationsPerSecond(void) { return sum / REDIS_OPS_SEC_SAMPLES; } -void closeTimedoutClient(redisClient *c) { +void clientsCronHandleTimeout(redisClient *c) { time_t now = time(NULL); if (server.maxidletime && @@ -668,7 +668,7 @@ void clientsCron(void) { listRotate(server.clients); head = listFirst(server.clients); c = listNodeValue(head); - closeTimedoutClient(c); + clientsCronHandleTimeout(c); } } From f9322fb8ed9d47a79a64e7b72485e08f256f2936 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 14 Mar 2012 10:13:23 +0100 Subject: [PATCH 0015/1752] sds.c new function sdsRemoveFreeSpace(). The new function is used in order to resize the string allocation so that only the minimal allocation possible is used, removing all the free space at the end of the string normally used to improve efficiency of concatenation operations. --- src/sds.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/sds.c b/src/sds.c index 092a431e5b0..82d12e232a4 100644 --- a/src/sds.c +++ b/src/sds.c @@ -111,6 +111,18 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { return newsh->buf; } +/* Reallocate the sds string so that it has no free space at the end. The + * contained string remains not altered, but next concatenation operations + * will require a reallocation. */ +sds sdsRemoveFreeSpace(sds s) { + struct sdshdr *sh; + + sh = (void*) (s-(sizeof(struct sdshdr))); + sh = zrealloc(sh, sizeof(struct sdshdr)+sh->len+1); + sh->free = 0; + return sh->buf; +} + /* Increment the sds length and decrements the left free space at the * end of the string accordingly to 'incr'. Also set the null term * in the new end of the string. From 6934832e53000943e4ee0bc2a37a0d34cac17af6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 14 Mar 2012 14:58:26 +0100 Subject: [PATCH 0016/1752] sds.c: sdsAllocSize() function added. --- src/sds.c | 6 ++++++ src/sds.h | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/sds.c b/src/sds.c index 82d12e232a4..bc6aa6b2f25 100644 --- a/src/sds.c +++ b/src/sds.c @@ -123,6 +123,12 @@ sds sdsRemoveFreeSpace(sds s) { return sh->buf; } +size_t sdsAllocSize(sds s) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + + return sizeof(*sh)+sh->len+sh->free+1; +} + /* Increment the sds length and decrements the left free space at the * end of the string accordingly to 'incr'. Also set the null term * in the new end of the string. diff --git a/src/sds.h b/src/sds.h index b00551b417d..0648381bb9a 100644 --- a/src/sds.h +++ b/src/sds.h @@ -94,5 +94,7 @@ sds sdsmapchars(sds s, char *from, char *to, size_t setlen); /* Low level functions exposed to the user API */ sds sdsMakeRoomFor(sds s, size_t addlen); void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); #endif From 9fa9ccb04e43950327959b356fff910036c21536 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 14 Mar 2012 15:32:30 +0100 Subject: [PATCH 0017/1752] Reclaim space from the client querybuf if needed. --- src/aof.c | 1 + src/networking.c | 2 ++ src/redis.c | 32 +++++++++++++++++++++++++++++--- src/redis.h | 1 + 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/src/aof.c b/src/aof.c index 64cd76d3d76..83633217f79 100644 --- a/src/aof.c +++ b/src/aof.c @@ -287,6 +287,7 @@ struct redisClient *createFakeClient(void) { selectDb(c,0); c->fd = -1; c->querybuf = sdsempty(); + c->querybuf_peak = 0; c->argc = 0; c->argv = NULL; c->bufpos = 0; diff --git a/src/networking.c b/src/networking.c index ae77d11b231..375186d1e01 100644 --- a/src/networking.c +++ b/src/networking.c @@ -43,6 +43,7 @@ redisClient *createClient(int fd) { c->fd = fd; c->bufpos = 0; c->querybuf = sdsempty(); + c->querybuf_peak = 0; c->reqtype = 0; c->argc = 0; c->argv = NULL; @@ -998,6 +999,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { } qblen = sdslen(c->querybuf); + if (c->querybuf_peak < qblen) c->querybuf_peak = qblen; c->querybuf = sdsMakeRoomFor(c->querybuf, readlen); nread = read(fd, c->querybuf+qblen, readlen); if (nread == -1) { diff --git a/src/redis.c b/src/redis.c index 58b5f1c4bf2..dacf47125b4 100644 --- a/src/redis.c +++ b/src/redis.c @@ -629,7 +629,7 @@ long long getOperationsPerSecond(void) { } void clientsCronHandleTimeout(redisClient *c) { - time_t now = time(NULL); + time_t now = server.unixtime; if (server.maxidletime && !(c->flags & REDIS_SLAVE) && /* no timeout for slaves */ @@ -649,15 +649,40 @@ void clientsCronHandleTimeout(redisClient *c) { } } +/* The client query buffer is an sds.c string that can end with a lot of + * free space not used, this function reclaims space if needed. */ +void clientsCronResizeQueryBuffer(redisClient *c) { + size_t querybuf_size = sdsAllocSize(c->querybuf); + time_t idletime = server.unixtime - c->lastinteraction; + + /* There are two conditions to resize the query buffer: + * 1) Query buffer is > BIG_ARG and too big for latest peak. + * 2) Client is inactive and the buffer is bigger than 1k. */ + if (((querybuf_size > REDIS_MBULK_BIG_ARG) && + (querybuf_size/(c->querybuf_peak+1)) > 2) || + (querybuf_size > 1024 && idletime > 2)) + { + /* Only resize the query buffer if it is actually wasting space. */ + if (sdsavail(c->querybuf) > 1024) { + c->querybuf = sdsRemoveFreeSpace(c->querybuf); + } + } + /* Reset the peak again to capture the peak memory usage in the next + * cycle. */ + c->querybuf_peak = 0; +} + void clientsCron(void) { /* Make sure to process at least 1/100 of clients per call. * Since this function is called 10 times per second we are sure that * in the worst case we process all the clients in 10 seconds. * In normal conditions (a reasonable number of clients) we process * all the clients in a shorter time. */ - int iterations = listLength(server.clients)/100; - if (iterations < 50) iterations = 50; + int numclients = listLength(server.clients); + int iterations = numclients/100; + if (iterations < 50) + iterations = (numclients < 50) ? numclients : 50; while(listLength(server.clients) && iterations--) { redisClient *c; listNode *head; @@ -669,6 +694,7 @@ void clientsCron(void) { head = listFirst(server.clients); c = listNodeValue(head); clientsCronHandleTimeout(c); + clientsCronResizeQueryBuffer(c); } } diff --git a/src/redis.h b/src/redis.h index 28649f6d638..cbb2df791ad 100644 --- a/src/redis.h +++ b/src/redis.h @@ -323,6 +323,7 @@ typedef struct redisClient { redisDb *db; int dictid; sds querybuf; + size_t querybuf_peak; /* Recent (100ms or more) peak of querybuf size */ int argc; robj **argv; struct redisCommand *cmd, *lastcmd; From 749817b7c3873f66c42e75a2bdc9b6a5aad5d67f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 14 Mar 2012 15:37:47 +0100 Subject: [PATCH 0018/1752] Version bumped to 2.5.2 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index fd60d69bff9..a79cbe97edd 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.1" +#define REDIS_VERSION "2.5.2" From f1eaf572005ea3a0c8687ec96b3d13ae77be3908 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 15 Mar 2012 20:51:35 +0100 Subject: [PATCH 0019/1752] Fix for issue #391. Use a simple protocol between clientsCron() and helper functions to understand if the client is still valind and clientsCron() should continue processing or if the client was freed and we should continue with the next one. --- src/redis.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/redis.c b/src/redis.c index dacf47125b4..cee7b32b8eb 100644 --- a/src/redis.c +++ b/src/redis.c @@ -628,7 +628,8 @@ long long getOperationsPerSecond(void) { return sum / REDIS_OPS_SEC_SAMPLES; } -void clientsCronHandleTimeout(redisClient *c) { +/* Check for timeouts. Returns non-zero if the client was terminated */ +int clientsCronHandleTimeout(redisClient *c) { time_t now = server.unixtime; if (server.maxidletime && @@ -641,17 +642,21 @@ void clientsCronHandleTimeout(redisClient *c) { { redisLog(REDIS_VERBOSE,"Closing idle client"); freeClient(c); + return 1; } else if (c->flags & REDIS_BLOCKED) { if (c->bpop.timeout != 0 && c->bpop.timeout < now) { addReply(c,shared.nullmultibulk); unblockClientWaitingData(c); } } + return 0; } /* The client query buffer is an sds.c string that can end with a lot of - * free space not used, this function reclaims space if needed. */ -void clientsCronResizeQueryBuffer(redisClient *c) { + * free space not used, this function reclaims space if needed. + * + * The funciton always returns 0 as it never terminates the client. */ +int clientsCronResizeQueryBuffer(redisClient *c) { size_t querybuf_size = sdsAllocSize(c->querybuf); time_t idletime = server.unixtime - c->lastinteraction; @@ -670,6 +675,7 @@ void clientsCronResizeQueryBuffer(redisClient *c) { /* Reset the peak again to capture the peak memory usage in the next * cycle. */ c->querybuf_peak = 0; + return 0; } void clientsCron(void) { @@ -693,8 +699,11 @@ void clientsCron(void) { listRotate(server.clients); head = listFirst(server.clients); c = listNodeValue(head); - clientsCronHandleTimeout(c); - clientsCronResizeQueryBuffer(c); + /* The following functions do different service checks on the client. + * The protocol is that they return non-zero if the client was + * terminated. */ + if (clientsCronHandleTimeout(c)) continue; + if (clientsCronResizeQueryBuffer(c)) continue; } } From 78d6a02b0c0d20a643b82ebd744c85b8d62fd3f8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 16 Mar 2012 17:17:39 +0100 Subject: [PATCH 0020/1752] First implementation of --test-memory. Still a work in progress. --- src/Makefile | 2 +- src/redis.c | 16 +++++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 18df4a20509..a09e1b42bdd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -73,7 +73,7 @@ QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); endif -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o +OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o diff --git a/src/redis.c b/src/redis.c index cee7b32b8eb..9c400ecdcc7 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2180,7 +2180,8 @@ void usage() { fprintf(stderr,"Usage: ./redis-server [/path/to/redis.conf] [options]\n"); fprintf(stderr," ./redis-server - (read config from stdin)\n"); fprintf(stderr," ./redis-server -v or --version\n"); - fprintf(stderr," ./redis-server -h or --help\n\n"); + fprintf(stderr," ./redis-server -h or --help\n"); + fprintf(stderr," ./redis-server --test-memory \n\n"); fprintf(stderr,"Examples:\n"); fprintf(stderr," ./redis-server (run the server with default conf)\n"); fprintf(stderr," ./redis-server /etc/redis/6379.conf\n"); @@ -2236,6 +2237,8 @@ void setupSignalHandlers(void) { return; } +void memtest(size_t megabytes, int passes); + int main(int argc, char **argv) { long long start; struct timeval tv; @@ -2257,6 +2260,17 @@ int main(int argc, char **argv) { strcmp(argv[1], "--version") == 0) version(); if (strcmp(argv[1], "--help") == 0 || strcmp(argv[1], "-h") == 0) usage(); + if (strcmp(argv[1], "--test-memory") == 0) { + if (argc == 3) { + memtest(atoi(argv[2]),10000); + exit(0); + } else { + fprintf(stderr,"Please specify the amount of memory to test in megabytes.\n"); + fprintf(stderr,"Example: ./redis-server --test-memory 4096\n\n"); + exit(1); + } + } + /* First argument is the config file name? */ if (argv[j][0] != '-' || argv[j][1] != '-') configfile = argv[j++]; From f4df22d1c5a487a2f00fe62fcec46366fbd7b77b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 16 Mar 2012 17:21:49 +0100 Subject: [PATCH 0021/1752] Hem... actual memtest.c file added. --- src/memtest.c | 91 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 src/memtest.c diff --git a/src/memtest.c b/src/memtest.c new file mode 100644 index 00000000000..a0e76f192cf --- /dev/null +++ b/src/memtest.c @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include +#include + +#if (ULONG_MAX == 4294967295UL) +#define MEMTEST_32BIT +#elif (ULONG_MAX == 18446744073709551615ULL) +#define MEMTEST_64BIT +#else +#error "ULONG_MAX value not supported." +#endif + +/* Fill words stepping a single page at every write, so we continue to + * touch all the pages in the smallest amount of time reducing the + * effectiveness of caches, and making it hard for the OS to transfer + * pages on the swap. */ +void memtest_fill(unsigned long *l, size_t bytes) { + unsigned long step = 4096/sizeof(unsigned long); + unsigned long words = bytes/sizeof(unsigned long)/2; + unsigned long iwords = words/step; /* words per iteration */ + unsigned long off, w, *l1, *l2; + + assert((bytes & 4095) == 0); + for (off = 0; off < step; off++) { + l1 = l+off; + l2 = l1+words; + for (w = 0; w < iwords; w++) { +#ifdef MEMTEST_32BIT + *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | + (((unsigned long) (rand()&0xffff)) << 16); +#else + *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | + (((unsigned long) (rand()&0xffff)) << 16) | + (((unsigned long) (rand()&0xffff)) << 32) | + (((unsigned long) (rand()&0xffff)) << 48); +#endif + l1 += step; + l2 += step; + } + } +} + +void memtest_compare(unsigned long *l, size_t bytes) { + unsigned long words = bytes/sizeof(unsigned long)/2; + unsigned long w, *l1, *l2; + + assert((bytes & 4095) == 0); + l1 = l; + l2 = l1+words; + for (w = 0; w < words; w++) { + if (*l1 != *l2) { + printf("\n*** MEMORY ERROR DETECTED: %p != %p (%lu vs %lu)\n", + (void*)l1, (void*)l2, *l1, *l2); + exit(1); + } + l1 ++; + l2 ++; + } +} + +void memtest_test(size_t megabytes, int passes) { + size_t bytes = megabytes*1024*1024; + unsigned long *m = malloc(bytes); + int pass = 0; + + if (m == NULL) { + fprintf(stderr,"Unable to allocate %zu megabytes: %s", + megabytes, strerror(errno)); + exit(1); + } + while (pass != passes) { + pass++; + printf("PASS %d... ", pass); + fflush(stdout); + memtest_fill(m,bytes); + memtest_compare(m,bytes); + printf("ok\n"); + } +} + +void memtest(size_t megabytes, int passes) { + memtest_test(megabytes,passes); + printf("\nYour memory passed this test.\n"); + printf("Please if you are stil in doubt use the following two tools:\n"); + printf("1) memtest86: http://www.memtest86.com/\n"); + printf("2) memtester: http://pyropus.ca/software/memtester/\n"); + exit(0); +} From d605fdabfae3722ce41efa5ec42524effa0445e9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 16 Mar 2012 21:19:53 +0100 Subject: [PATCH 0022/1752] Memory test function now less boring thanks to screen-wide progress bar. --- src/memtest.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/src/memtest.c b/src/memtest.c index a0e76f192cf..56d566abd18 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -4,6 +4,8 @@ #include #include #include +#include +#include #if (ULONG_MAX == 4294967295UL) #define MEMTEST_32BIT @@ -13,6 +15,37 @@ #error "ULONG_MAX value not supported." #endif +static struct winsize ws; +size_t progress_printed; /* Printed chars in screen-wide progress bar. */ +size_t progress_full; /* How many chars to write to fill the progress bar. */ + +void memtest_progress_start(char *title, int pass) { + int j; + + printf("\x1b[H\x1b[2J"); /* Cursor home, clear screen. */ + /* Fill with dots. */ + for (j = 0; j < ws.ws_col*ws.ws_row; j++) printf("."); + printf("\x1b[H\x1b[2K"); /* Cursor home, clear current line. */ + printf("%s [%d]\n", title, pass); /* Print title. */ + progress_printed = 0; + progress_full = ws.ws_col*(ws.ws_row-1); + fflush(stdout); +} + +void memtest_progress_end(void) { + printf("\x1b[H\x1b[2J"); /* Cursor home, clear screen. */ +} + +void memtest_progress_step(size_t curr, size_t size, char c) { + size_t chars = (curr*progress_full)/size, j; + + for (j = 0; j < chars-progress_printed; j++) { + printf("%c",c); + progress_printed++; + } + fflush(stdout); +} + /* Fill words stepping a single page at every write, so we continue to * touch all the pages in the smallest amount of time reducing the * effectiveness of caches, and making it hard for the OS to transfer @@ -39,6 +72,8 @@ void memtest_fill(unsigned long *l, size_t bytes) { #endif l1 += step; l2 += step; + if ((w & 0xffff) == 0) + memtest_progress_step(w+iwords*off,words,'+'); } } } @@ -58,13 +93,14 @@ void memtest_compare(unsigned long *l, size_t bytes) { } l1 ++; l2 ++; + if ((w & 0xffff) == 0) memtest_progress_step(w,words,'='); } } void memtest_test(size_t megabytes, int passes) { size_t bytes = megabytes*1024*1024; unsigned long *m = malloc(bytes); - int pass = 0; + int pass = 0, j; if (m == NULL) { fprintf(stderr,"Unable to allocate %zu megabytes: %s", @@ -73,15 +109,22 @@ void memtest_test(size_t megabytes, int passes) { } while (pass != passes) { pass++; - printf("PASS %d... ", pass); - fflush(stdout); + memtest_progress_start("Random fill",pass); memtest_fill(m,bytes); - memtest_compare(m,bytes); - printf("ok\n"); + memtest_progress_end(); + for (j = 0; j < 4; j++) { + memtest_progress_start("Compare",pass); + memtest_compare(m,bytes); + memtest_progress_end(); + } } } void memtest(size_t megabytes, int passes) { + if (ioctl(1, TIOCGWINSZ, &ws) == -1) { + ws.ws_col = 80; + ws.ws_row = 20; + } memtest_test(megabytes,passes); printf("\nYour memory passed this test.\n"); printf("Please if you are stil in doubt use the following two tools:\n"); From d4e6ce3e97a4bb2ea9c25af2ad48e5ed9d0c5024 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 18 Mar 2012 11:35:35 +0100 Subject: [PATCH 0023/1752] On crash suggest to give --test-memory a try. --- src/debug.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index a355df05f6d..8eea3bf8d04 100644 --- a/src/debug.c +++ b/src/debug.c @@ -636,8 +636,9 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { redisLog(REDIS_WARNING, "\n=== REDIS BUG REPORT END. Make sure to include from START to END. ===\n\n" -" Please report the crash opening an issue on github:\n\n" -" http://github.com/antirez/redis/issues\n\n" +" Please report the crash opening an issue on github:\n\n" +" http://github.com/antirez/redis/issues\n\n" +" Suspect RAM error? Use redis-server --test-memory to veryfy it.\n\n" ); /* free(messages); Don't call free() with possibly corrupted memory. */ if (server.daemonize) unlink(server.pidfile); From 32f62ed6d0f1cad1cc22a55d934ff4487ebd7f82 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 18 Mar 2012 17:24:48 +0100 Subject: [PATCH 0024/1752] Number of iteration of --test-memory is now 300 (several minutes per gigabyte). Memtest86 and Memtester links are also displayed while running the test. --- src/memtest.c | 6 ++++-- src/redis.c | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/memtest.c b/src/memtest.c index 56d566abd18..18e2e6e8278 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -24,11 +24,13 @@ void memtest_progress_start(char *title, int pass) { printf("\x1b[H\x1b[2J"); /* Cursor home, clear screen. */ /* Fill with dots. */ - for (j = 0; j < ws.ws_col*ws.ws_row; j++) printf("."); + for (j = 0; j < ws.ws_col*(ws.ws_row-2); j++) printf("."); + printf("Please keep the test running several minutes per GB of memory.\n"); + printf("Also check http://www.memtest86.com/ and http://pyropus.ca/software/memtester/"); printf("\x1b[H\x1b[2K"); /* Cursor home, clear current line. */ printf("%s [%d]\n", title, pass); /* Print title. */ progress_printed = 0; - progress_full = ws.ws_col*(ws.ws_row-1); + progress_full = ws.ws_col*(ws.ws_row-3); fflush(stdout); } diff --git a/src/redis.c b/src/redis.c index 9c400ecdcc7..4e0042960c0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2262,7 +2262,7 @@ int main(int argc, char **argv) { strcmp(argv[1], "-h") == 0) usage(); if (strcmp(argv[1], "--test-memory") == 0) { if (argc == 3) { - memtest(atoi(argv[2]),10000); + memtest(atoi(argv[2]),300); exit(0); } else { fprintf(stderr,"Please specify the amount of memory to test in megabytes.\n"); From 74760d3ccd0c4bc21b8743d5044253465ebc301e Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 18 Mar 2012 17:27:56 +0100 Subject: [PATCH 0025/1752] Fixed typo. --- src/memtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memtest.c b/src/memtest.c index 18e2e6e8278..645d1da1ef4 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -129,7 +129,7 @@ void memtest(size_t megabytes, int passes) { } memtest_test(megabytes,passes); printf("\nYour memory passed this test.\n"); - printf("Please if you are stil in doubt use the following two tools:\n"); + printf("Please if you are still in doubt use the following two tools:\n"); printf("1) memtest86: http://www.memtest86.com/\n"); printf("2) memtester: http://pyropus.ca/software/memtester/\n"); exit(0); From ea693f028246300e9ca7ee5efc9112436f53145c Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 18 Mar 2012 18:03:27 +0100 Subject: [PATCH 0026/1752] More memory tests implemented. Default number of iterations lowered to a more acceptable value of 50. --- src/memtest.c | 75 ++++++++++++++++++++++++++++++++++++++++++++------- src/redis.c | 2 +- 2 files changed, 67 insertions(+), 10 deletions(-) diff --git a/src/memtest.c b/src/memtest.c index 645d1da1ef4..01569d059a5 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -15,6 +15,14 @@ #error "ULONG_MAX value not supported." #endif +#ifdef MEMTEST_32BIT +#define ULONG_ONEZERO 0xaaaaaaaaaaaaaaaaUL +#define ULONG_ZEROONE 0x5555555555555555UL +#else +#define ULONG_ONEZERO 0xaaaaaaaaUL +#define ULONG_ZEROONE 0x55555555UL +#endif + static struct winsize ws; size_t progress_printed; /* Printed chars in screen-wide progress bar. */ size_t progress_full; /* How many chars to write to fill the progress bar. */ @@ -52,7 +60,7 @@ void memtest_progress_step(size_t curr, size_t size, char c) { * touch all the pages in the smallest amount of time reducing the * effectiveness of caches, and making it hard for the OS to transfer * pages on the swap. */ -void memtest_fill(unsigned long *l, size_t bytes) { +void memtest_fill_random(unsigned long *l, size_t bytes) { unsigned long step = 4096/sizeof(unsigned long); unsigned long words = bytes/sizeof(unsigned long)/2; unsigned long iwords = words/step; /* words per iteration */ @@ -75,7 +83,40 @@ void memtest_fill(unsigned long *l, size_t bytes) { l1 += step; l2 += step; if ((w & 0xffff) == 0) - memtest_progress_step(w+iwords*off,words,'+'); + memtest_progress_step(w+iwords*off,words,'R'); + } + } +} + +/* Like memtest_fill_random() but uses the two specified values to fill + * memory, in an alternated way (v1|v2|v1|v2|...) */ +void memtest_fill_value(unsigned long *l, size_t bytes, unsigned long v1, + unsigned long v2, char sym) +{ + unsigned long step = 4096/sizeof(unsigned long); + unsigned long words = bytes/sizeof(unsigned long)/2; + unsigned long iwords = words/step; /* words per iteration */ + unsigned long off, w, *l1, *l2, v; + + assert((bytes & 4095) == 0); + for (off = 0; off < step; off++) { + l1 = l+off; + l2 = l1+words; + v = (off & 1) ? v2 : v1; + for (w = 0; w < iwords; w++) { +#ifdef MEMTEST_32BIT + *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | + (((unsigned long) (rand()&0xffff)) << 16); +#else + *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | + (((unsigned long) (rand()&0xffff)) << 16) | + (((unsigned long) (rand()&0xffff)) << 32) | + (((unsigned long) (rand()&0xffff)) << 48); +#endif + l1 += step; + l2 += step; + if ((w & 0xffff) == 0) + memtest_progress_step(w+iwords*off,words,sym); } } } @@ -99,10 +140,20 @@ void memtest_compare(unsigned long *l, size_t bytes) { } } +void memtest_compare_times(unsigned long *m, size_t bytes, int pass, int times) { + int j; + + for (j = 0; j < times; j++) { + memtest_progress_start("Compare",pass); + memtest_compare(m,bytes); + memtest_progress_end(); + } +} + void memtest_test(size_t megabytes, int passes) { size_t bytes = megabytes*1024*1024; unsigned long *m = malloc(bytes); - int pass = 0, j; + int pass = 0; if (m == NULL) { fprintf(stderr,"Unable to allocate %zu megabytes: %s", @@ -112,13 +163,19 @@ void memtest_test(size_t megabytes, int passes) { while (pass != passes) { pass++; memtest_progress_start("Random fill",pass); - memtest_fill(m,bytes); + memtest_fill_random(m,bytes); memtest_progress_end(); - for (j = 0; j < 4; j++) { - memtest_progress_start("Compare",pass); - memtest_compare(m,bytes); - memtest_progress_end(); - } + memtest_compare_times(m,bytes,pass,4); + + memtest_progress_start("Solid fill",pass); + memtest_fill_value(m,bytes,0,(unsigned long)-1,'S'); + memtest_progress_end(); + memtest_compare_times(m,bytes,pass,4); + + memtest_progress_start("Checkerboard fill",pass); + memtest_fill_value(m,bytes,ULONG_ONEZERO,ULONG_ZEROONE,'C'); + memtest_progress_end(); + memtest_compare_times(m,bytes,pass,4); } } diff --git a/src/redis.c b/src/redis.c index 4e0042960c0..eb19fbbf301 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2262,7 +2262,7 @@ int main(int argc, char **argv) { strcmp(argv[1], "-h") == 0) usage(); if (strcmp(argv[1], "--test-memory") == 0) { if (argc == 3) { - memtest(atoi(argv[2]),300); + memtest(atoi(argv[2]),50); exit(0); } else { fprintf(stderr,"Please specify the amount of memory to test in megabytes.\n"); From a7ef5ce1b0f66e4eac2b0eca61b9dd8e9ddef649 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Mar 2012 14:02:34 +0100 Subject: [PATCH 0027/1752] Memory addressing test implemented. --- src/memtest.c | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/src/memtest.c b/src/memtest.c index 01569d059a5..272ec502255 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -56,6 +56,33 @@ void memtest_progress_step(size_t curr, size_t size, char c) { fflush(stdout); } +/* Test that addressing is fine. Every location is populated with its own + * address, and finally verified. This test is very fast but may detect + * ASAP big issues with the memory subsystem. */ +void memtest_addressing(unsigned long *l, size_t bytes) { + unsigned long words = bytes/sizeof(unsigned long); + unsigned long j, *p; + + /* Fill */ + p = l; + for (j = 0; j < words; j++) { + *p = (unsigned long)p; + p++; + if ((j & 0xffff) == 0) memtest_progress_step(j,words*2,'A'); + } + /* Test */ + p = l; + for (j = 0; j < words; j++) { + if (*p != (unsigned long)p) { + printf("\n*** MEMORY ADDRESSING ERROR: %p contains %lu\n", + (void*) p, *p); + exit(1); + } + p++; + if ((j & 0xffff) == 0) memtest_progress_step(j+words,words*2,'A'); + } +} + /* Fill words stepping a single page at every write, so we continue to * touch all the pages in the smallest amount of time reducing the * effectiveness of caches, and making it hard for the OS to transfer @@ -162,6 +189,11 @@ void memtest_test(size_t megabytes, int passes) { } while (pass != passes) { pass++; + + memtest_progress_start("Addressing test",pass); + memtest_addressing(m,bytes); + memtest_progress_end(); + memtest_progress_start("Random fill",pass); memtest_fill_random(m,bytes); memtest_progress_end(); From 6e6bbac7a5ebc24ff7bf5913b4b8b49433b8e303 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Mar 2012 19:16:41 +0100 Subject: [PATCH 0028/1752] Read-only flag removed from PUBLISH command. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index eb19fbbf301..0b5d672bfd2 100644 --- a/src/redis.c +++ b/src/redis.c @@ -229,7 +229,7 @@ struct redisCommand redisCommandTable[] = { {"unsubscribe",unsubscribeCommand,-1,"rps",0,NULL,0,0,0,0,0}, {"psubscribe",psubscribeCommand,-2,"rps",0,NULL,0,0,0,0,0}, {"punsubscribe",punsubscribeCommand,-1,"rps",0,NULL,0,0,0,0,0}, - {"publish",publishCommand,3,"rpf",0,NULL,0,0,0,0,0}, + {"publish",publishCommand,3,"pf",0,NULL,0,0,0,0,0}, {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0}, {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,4,"awm",0,NULL,1,1,1,0,0}, From 24b0942275360c12a87fcccd38b17f32678aa6e5 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Mar 2012 19:29:06 +0100 Subject: [PATCH 0029/1752] Suppress warnings compiling redis-cli with certain gcc versions. --- src/redis-cli.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index c631aa79143..bdaa3964217 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -478,7 +478,7 @@ static sds cliFormatReplyCSV(redisReply *r) { static int cliReadReply(int output_raw_strings) { void *_reply; redisReply *reply; - sds out; + sds out = NULL; int output = 1; if (redisGetReply(context,&_reply) != REDIS_OK) { @@ -498,7 +498,8 @@ static int cliReadReply(int output_raw_strings) { reply = (redisReply*)_reply; - /* Check if we need to connect to a different node and reissue the request. */ + /* Check if we need to connect to a different node and reissue the + * request. */ if (config.cluster_mode && reply->type == REDIS_REPLY_ERROR && (!strncmp(reply->str,"MOVED",5) || !strcmp(reply->str,"ASK"))) { @@ -926,7 +927,10 @@ static void slaveMode(void) { unsigned long long payload; /* Send the SYNC command. */ - write(fd,"SYNC\r\n",6); + if (write(fd,"SYNC\r\n",6) != 6) { + fprintf(stderr,"Error writing to master\n"); + exit(1); + } /* Read $\r\n, making sure to read just up to "\n" */ p = buf; From 518e7202b29a287f49671fd32f706054570b3f4e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Mar 2012 22:10:18 +0100 Subject: [PATCH 0030/1752] Fixed typo in 2.6 release notes. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 8a225b7179c..64acd87d1df 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -46,7 +46,7 @@ UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to in slaves. * Milliseconds resolution expires, also added new commands with milliseconds precision (PEXPIRE, PTTL, ...). -* Clinets max output buffer soft and hard limits. You can specifiy different +* Clients max output buffer soft and hard limits. You can specifiy different limits for different classes of clients (normal,pubsub,slave). * AOF is now able to rewrite aggregate data types using variadic commands, often producing an AOF that is faster to save, load, and is smaller in size. From 0380c13bbe140b627f1e1c713f1d9ca032c44c0c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 20 Mar 2012 13:06:50 +0100 Subject: [PATCH 0031/1752] redis_init_script template updated. --- utils/redis.conf.tpl | 194 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 153 insertions(+), 41 deletions(-) diff --git a/utils/redis.conf.tpl b/utils/redis.conf.tpl index 9e2e1355712..e7febedaba8 100644 --- a/utils/redis.conf.tpl +++ b/utils/redis.conf.tpl @@ -1,6 +1,6 @@ # Redis configuration file example -# Note on units: when memory size is needed, it is possible to specifiy +# Note on units: when memory size is needed, it is possible to specify # it in the usual form of 1k 5GB 4M and so forth: # # 1k => 1000 bytes @@ -34,9 +34,10 @@ port $REDIS_PORT # on a unix socket when not specified. # # unixsocket /tmp/redis.sock +# unixsocketperm 755 # Close the connection after a client is idle for N seconds (0 to disable) -timeout 300 +timeout 0 # Set server verbosity to 'debug' # it can be one of: @@ -44,7 +45,7 @@ timeout 300 # verbose (many rarely useful info, but not a mess like the debug level) # notice (moderately verbose, what you want in production probably) # warning (only very important / critical messages are logged) -loglevel verbose +loglevel notice # Specify the log file name. Also 'stdout' can be used to force # Redis to log on the standard output. Note that if you use standard @@ -81,11 +82,32 @@ databases 16 # after 60 sec if at least 10000 keys changed # # Note: you can disable saving at all commenting all the "save" lines. +# +# It is also possible to remove all the previously configured save +# points by adding a save directive with a single empty string argument +# like in the following example: +# +# save "" save 900 1 save 300 10 save 60 10000 +# By default Redis will stop accepting writes if RDB snapshots are enabled +# (at least one save point) and the latest background save failed. +# This will make the user aware (in an hard way) that data is not persisting +# on disk properly, otherwise chances are that no one will notice and some +# distater will happen. +# +# If the background saving process will start working again Redis will +# automatically allow writes again. +# +# However if you have setup your proper monitoring of the Redis server +# and persistence, you may want to disable this feature so that Redis will +# continue to work as usually even if there are problems with disk, +# permissions, and so forth. +stop-writes-on-bgsave-error yes + # Compress string objects using LZF when dump .rdb databases? # For default that's set to 'yes' as it's almost always a win. # If you want to save some CPU in the saving child set it to 'no' but @@ -125,7 +147,7 @@ dir $REDIS_DATA_DIR # is still in progress, the slave can act in two different ways: # # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will -# still reply to client requests, possibly with out of data data, or the +# still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # # 2) if slave-serve-stale data is set to 'no' the slave will reply with @@ -134,6 +156,21 @@ dir $REDIS_DATA_DIR # slave-serve-stale-data yes +# Slaves send PINGs to server in a predefined interval. It's possible to change +# this interval with the repl_ping_slave_period option. The default value is 10 +# seconds. +# +# repl-ping-slave-period 10 + +# The following option sets a timeout for both Bulk transfer I/O timeout and +# master data or ping response timeout. The default value is 60 seconds. +# +# It is important to make sure that this value is greater than the value +# specified for repl-ping-slave-period otherwise a timeout will be detected +# every time there is low traffic between the master and the slave. +# +# repl-timeout 60 + ################################## SECURITY ################################### # Require clients to issue AUTH before processing any other @@ -151,7 +188,7 @@ slave-serve-stale-data yes # Command renaming. # -# It is possilbe to change the name of dangerous commands in a shared +# It is possible to change the name of dangerous commands in a shared # environment. For instance the CONFIG command may be renamed into something # of hard to guess so that it will be still available for internal-use # tools but not available for general clients. @@ -160,37 +197,46 @@ slave-serve-stale-data yes # # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 # -# It is also possilbe to completely kill a command renaming it into +# It is also possible to completely kill a command renaming it into # an empty string: # # rename-command CONFIG "" ################################### LIMITS #################################### -# Set the max number of connected clients at the same time. By default there -# is no limit, and it's up to the number of file descriptors the Redis process -# is able to open. The special value '0' means no limits. +# Set the max number of connected clients at the same time. By default +# this limit is set to 10000 clients, however if the Redis server is not +# able ot configure the process file limit to allow for the specified limit +# the max number of allowed clients is set to the current file limit +# minus 32 (as Redis reserves a few file descriptors for internal uses). +# # Once the limit is reached Redis will close all the new connections sending # an error 'max number of clients reached'. # -# maxclients 128 +# maxclients 10000 # Don't use more memory than the specified amount of bytes. -# When the memory limit is reached Redis will try to remove keys with an -# EXPIRE set. It will try to start freeing keys that are going to expire -# in little time and preserve keys with a longer time to live. -# Redis will also try to remove objects from free lists if possible. -# -# If all this fails, Redis will start to reply with errors to commands -# that will use more memory, like SET, LPUSH, and so on, and will continue -# to reply to most read-only commands like GET. -# -# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a -# 'state' server or cache, not as a real DB. When Redis is used as a real -# database the memory usage will grow over the weeks, it will be obvious if -# it is going to use too much memory in the long run, and you'll have the time -# to upgrade. With maxmemory after the limit is reached you'll start to get -# errors for write operations, and this may even lead to DB inconsistency. +# When the memory limit is reached Redis will try to remove keys +# accordingly to the eviction policy selected (see maxmemmory-policy). +# +# If Redis can't remove keys according to the policy, or if the policy is +# set to 'noeviction', Redis will start to reply with errors to commands +# that would use more memory, like SET, LPUSH, and so on, and will continue +# to reply to read-only commands like GET. +# +# This option is usually useful when using Redis as an LRU cache, or to set +# an hard memory limit for an instance (using the 'noeviction' policy). +# +# WARNING: If you have slaves attached to an instance with maxmemory on, +# the size of the output buffers needed to feed the slaves are subtracted +# from the used memory count, so that network problems / resyncs will +# not trigger a loop where keys are evicted, and in turn the output +# buffer of slaves is full with DELs of keys evicted triggering the deletion +# of more keys, and so forth until the database is completely emptied. +# +# In short... if you have slaves attached it is suggested that you set a lower +# limit for maxmemory so that there is some free RAM on the system for slave +# output buffers (but this is not needed if the policy is 'noeviction'). # # maxmemory @@ -200,7 +246,7 @@ slave-serve-stale-data yes # volatile-lru -> remove the key with an expire set using an LRU algorithm # allkeys-lru -> remove any key accordingly to the LRU algorithm # volatile-random -> remove a random key with an expire set -# allkeys->random -> remove a random key, any key +# allkeys-random -> remove a random key, any key # volatile-ttl -> remove the key with the nearest expire time (minor TTL) # noeviction -> don't expire at all, just return an error on write operations # @@ -260,7 +306,7 @@ appendonly no # # The default is "everysec" that's usually the right compromise between # speed and data safety. It's up to you to understand if you can relax this to -# "no" that will will let the operating system flush the output buffer when +# "no" that will let the operating system flush the output buffer when # it wants, for better performances (but if you can live with the idea of # some data loss consider the default persistence mode that's snapshotting), # or on the contrary, use "always" that's very slow but a bit safer than @@ -284,7 +330,7 @@ appendfsync everysec # BGSAVE or BGREWRITEAOF is in progress. # # This means that while another child is saving the durability of Redis is -# the same as "appendfsync none", that in pratical terms means that it is +# the same as "appendfsync none", that in practical terms means that it is # possible to lost up to 30 seconds of log in the worst scenario (with the # default Linux settings). # @@ -306,7 +352,7 @@ no-appendfsync-on-rewrite no # is useful to avoid rewriting the AOF file even if the percentage increase # is reached but it is still pretty small. # -# Specify a precentage of zero in order to disable the automatic AOF +# Specify a percentage of zero in order to disable the automatic AOF # rewrite feature. auto-aof-rewrite-percentage 100 @@ -315,9 +361,39 @@ auto-aof-rewrite-min-size 64mb ################################ LUA SCRIPTING ############################### # Max execution time of a Lua script in milliseconds. -# This prevents that a programming error generating an infinite loop will block -# your server forever. Set it to 0 or a negative value for unlimited execution. -#lua-time-limit 60000 +# +# If the maximum execution time is reached Redis will log that a script is +# still in execution after the maximum allowed time and will start to +# reply to queries with an error. +# +# When a long running script exceed the maximum execution time only the +# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be +# used to stop a script that did not yet called write commands. The second +# is the only way to shut down the server in the case a write commands was +# already issue by the script but the user don't want to wait for the natural +# termination of the script. +# +# Set it to 0 or a negative value for unlimited execution without warnings. +lua-time-limit 5000 + +################################ REDIS CLUSTER ############################### +# +# Normal Redis instances can't be part of a Redis Cluster, only nodes that are +# started as cluster nodes can. In order to start a Redis instance as a +# cluster node enable the cluster support uncommenting the following: +# +# cluster-enabled yes + +# Every cluster node has a cluster configuration file. This file is not +# intended to be edited by hand. It is created and updated by Redis nodes. +# Every Redis Cluster node requires a different cluster configuration file. +# Make sure that instances running in the same system does not have +# overlapping cluster configuration file names. +# +# cluster-config-file nodes-6379.conf + +# In order to setup your cluster make sure to read the documentation +# available at http://redis.io web site. ################################## SLOW LOG ################################### @@ -345,12 +421,11 @@ slowlog-max-len 1024 ############################### ADVANCED CONFIG ############################### -# Hashes are encoded in a special way (much more memory efficient) when they -# have at max a given numer of elements, and the biggest element does not -# exceed a given threshold. You can configure this limits with the following -# configuration directives. -hash-max-zipmap-entries 512 -hash-max-zipmap-value 64 +# Hashes are encoded using a memory efficient data structure when they have a +# small number of entries, and the biggest entry does not exceed a given +# threshold. These thresholds can be configured using the following directives. +hash-max-ziplist-entries 512 +hash-max-ziplist-value 64 # Similarly to hashes, small lists are also encoded in a special way in order # to save a lot of space. The special representation is only used when @@ -373,9 +448,9 @@ zset-max-ziplist-value 64 # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level -# keys to values). The hash table implementation redis uses (see dict.c) +# keys to values). The hash table implementation Redis uses (see dict.c) # performs a lazy rehashing: the more operation you run into an hash table -# that is rhashing, the more rehashing "steps" are performed, so if the +# that is rehashing, the more rehashing "steps" are performed, so if the # server is idle the rehashing is never complete and some more memory is used # by the hash table. # @@ -391,10 +466,47 @@ zset-max-ziplist-value 64 # want to free memory asap when possible. activerehashing yes +# The client output buffer limits can be used to force disconnection of clients +# that are not reading data from the server fast enough for some reason (a +# common reason is that a Pub/Sub client can't consume messages as fast as the +# publisher can produce them). +# +# The limit can be set differently for the three different classes of clients: +# +# normal -> normal clients +# slave -> slave clients and MONITOR clients +# pubsub -> clients subcribed to at least one pubsub channel or pattern +# +# The syntax of every client-output-buffer-limit directive is the following: +# +# client-output-buffer-limit +# +# A client is immediately disconnected once the hard limit is reached, or if +# the soft limit is reached and remains reached for the specified number of +# seconds (continuously). +# So for instance if the hard limit is 32 megabytes and the soft limit is +# 16 megabytes / 10 seconds, the client will get disconnected immediately +# if the size of the output buffers reach 32 megabytes, but will also get +# disconnected if the client reaches 16 megabytes and continuously overcomes +# the limit for 10 seconds. +# +# By default normal clients are not limited because they don't receive data +# without asking (in a push way), but just after a request, so only +# asynchronous clients may create a scenario where data is requested faster +# than it can read. +# +# Instead there is a default limit for pubsub and slave clients, since +# subscribers and slaves receive data in a push fashion. +# +# Both the hard or the soft limit can be disabled just setting it to zero. +client-output-buffer-limit normal 0 0 0 +client-output-buffer-limit slave 256mb 64mb 60 +client-output-buffer-limit pubsub 32mb 8mb 60 + ################################## INCLUDES ################################### # Include one or more other config files here. This is useful if you -# have a standard template that goes to all redis server but also need +# have a standard template that goes to all Redis server but also need # to customize a few per-server settings. Include files can include # other files, so use this wisely. # From 7c1cec25251eabb2e3e1259e2111fddb46ceb675 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 20 Mar 2012 13:07:26 +0100 Subject: [PATCH 0032/1752] Redis cluster stuff removed from 2.6 redis.conf file. --- redis.conf | 19 ------------------- utils/redis.conf.tpl | 19 ------------------- 2 files changed, 38 deletions(-) diff --git a/redis.conf b/redis.conf index e03359963ba..1c375305dfe 100644 --- a/redis.conf +++ b/redis.conf @@ -376,25 +376,6 @@ auto-aof-rewrite-min-size 64mb # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 -################################ REDIS CLUSTER ############################### -# -# Normal Redis instances can't be part of a Redis Cluster, only nodes that are -# started as cluster nodes can. In order to start a Redis instance as a -# cluster node enable the cluster support uncommenting the following: -# -# cluster-enabled yes - -# Every cluster node has a cluster configuration file. This file is not -# intended to be edited by hand. It is created and updated by Redis nodes. -# Every Redis Cluster node requires a different cluster configuration file. -# Make sure that instances running in the same system does not have -# overlapping cluster configuration file names. -# -# cluster-config-file nodes-6379.conf - -# In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. - ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified diff --git a/utils/redis.conf.tpl b/utils/redis.conf.tpl index e7febedaba8..33d99a50935 100644 --- a/utils/redis.conf.tpl +++ b/utils/redis.conf.tpl @@ -376,25 +376,6 @@ auto-aof-rewrite-min-size 64mb # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 -################################ REDIS CLUSTER ############################### -# -# Normal Redis instances can't be part of a Redis Cluster, only nodes that are -# started as cluster nodes can. In order to start a Redis instance as a -# cluster node enable the cluster support uncommenting the following: -# -# cluster-enabled yes - -# Every cluster node has a cluster configuration file. This file is not -# intended to be edited by hand. It is created and updated by Redis nodes. -# Every Redis Cluster node requires a different cluster configuration file. -# Make sure that instances running in the same system does not have -# overlapping cluster configuration file names. -# -# cluster-config-file nodes-6379.conf - -# In order to setup your cluster make sure to read the documentation -# available at http://redis.io web site. - ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified From 054061685add05f30722971537e084f0589b601d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 20 Mar 2012 17:32:48 +0100 Subject: [PATCH 0033/1752] Support for read-only slaves. Semantical fixes. This commit introduces support for read only slaves via redis.conf and CONFIG GET/SET commands. Also various semantical fixes are implemented here: 1) MULTI/EXEC with only read commands now work where the server is into a state where writes (or commands increasing memory usage) are not allowed. Before this patch everything inside a transaction would fail in this conditions. 2) Scripts just calling read-only commands will work against read only slaves, when the server is out of memory, or when persistence is into an error condition. Before the patch EVAL always failed in this condition. --- redis.conf | 8 ++++++++ src/config.c | 11 +++++++++++ src/multi.c | 13 ++++++++----- src/redis.c | 26 ++++++++++++++++++++------ src/redis.h | 6 ++++-- src/scripting.c | 38 ++++++++++++++++++++++++++++++++++---- 6 files changed, 85 insertions(+), 17 deletions(-) diff --git a/redis.conf b/redis.conf index 1c375305dfe..52520fe2c41 100644 --- a/redis.conf +++ b/redis.conf @@ -156,6 +156,14 @@ dir ./ # slave-serve-stale-data yes +# You can configure a slave instance to accept writes or not. Writing against +# a slave instance may be useful to store some ephemeral data (because data +# written on a slave will be easily deleted after resync with the master) but +# may also cause problems if clients are writing to it for an error. +# +# Since Redis 2.6 by default slaves are read-only. +slave-read-only yes + # Slaves send PINGs to server in a predefined interval. It's possible to change # this interval with the repl_ping_slave_period option. The default value is 10 # seconds. diff --git a/src/config.c b/src/config.c index 71a800a1230..238c9462f8a 100644 --- a/src/config.c +++ b/src/config.c @@ -202,6 +202,10 @@ void loadServerConfigFromString(char *config) { if ((server.repl_serve_stale_data = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"slave-read-only") && argc == 2) { + if ((server.repl_slave_ro = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"rdbcompression") && argc == 2) { if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -507,6 +511,11 @@ void configSetCommand(redisClient *c) { if (yn == -1) goto badfmt; server.repl_serve_stale_data = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"slave-read-only")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.repl_slave_ro = yn; } else if (!strcasecmp(c->argv[2]->ptr,"dir")) { if (chdir((char*)o->ptr) == -1) { addReplyErrorFormat(c,"Changing directory: %s", strerror(errno)); @@ -705,6 +714,8 @@ void configGetCommand(redisClient *c) { server.aof_no_fsync_on_rewrite); config_get_bool_field("slave-serve-stale-data", server.repl_serve_stale_data); + config_get_bool_field("slave-read-only", + server.repl_slave_ro); config_get_bool_field("stop-writes-on-bgsave-error", server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); diff --git a/src/multi.c b/src/multi.c index 65ec38a8d0e..eee9748c5fa 100644 --- a/src/multi.c +++ b/src/multi.c @@ -40,6 +40,13 @@ void queueMultiCommand(redisClient *c) { c->mstate.count++; } +void discardTransaction(redisClient *c) { + freeClientMultiState(c); + initClientMultiState(c); + c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS);; + unwatchAllKeys(c); +} + void multiCommand(redisClient *c) { if (c->flags & REDIS_MULTI) { addReplyError(c,"MULTI calls can not be nested"); @@ -54,11 +61,7 @@ void discardCommand(redisClient *c) { addReplyError(c,"DISCARD without MULTI"); return; } - - freeClientMultiState(c); - initClientMultiState(c); - c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS);; - unwatchAllKeys(c); + discardTransaction(c); addReply(c,shared.ok); } diff --git a/src/redis.c b/src/redis.c index 0b5d672bfd2..b4de5a4317f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -211,7 +211,7 @@ struct redisCommand redisCommandTable[] = { {"lastsave",lastsaveCommand,1,"r",0,NULL,0,0,0,0,0}, {"type",typeCommand,2,"r",0,NULL,1,1,1,0,0}, {"multi",multiCommand,1,"rs",0,NULL,0,0,0,0,0}, - {"exec",execCommand,1,"wms",0,NULL,0,0,0,0,0}, + {"exec",execCommand,1,"s",0,NULL,0,0,0,0,0}, {"discard",discardCommand,1,"rs",0,NULL,0,0,0,0,0}, {"sync",syncCommand,1,"ars",0,NULL,0,0,0,0,0}, {"flushdb",flushdbCommand,1,"w",0,NULL,0,0,0,0,0}, @@ -237,8 +237,8 @@ struct redisCommand redisCommandTable[] = { {"dump",dumpCommand,2,"ar",0,NULL,1,1,1,0,0}, {"object",objectCommand,-2,"r",0,NULL,2,2,2,0,0}, {"client",clientCommand,-2,"ar",0,NULL,0,0,0,0,0}, - {"eval",evalCommand,-3,"wms",0,zunionInterGetKeys,0,0,0,0,0}, - {"evalsha",evalShaCommand,-3,"wms",0,zunionInterGetKeys,0,0,0,0,0}, + {"eval",evalCommand,-3,"s",0,zunionInterGetKeys,0,0,0,0,0}, + {"evalsha",evalShaCommand,-3,"s",0,zunionInterGetKeys,0,0,0,0,0}, {"slowlog",slowlogCommand,-2,"r",0,NULL,0,0,0,0,0}, {"script",scriptCommand,-2,"ras",0,NULL,0,0,0,0,0}, {"time",timeCommand,1,"rR",0,NULL,0,0,0,0,0} @@ -923,7 +923,11 @@ void createSharedObjects(void) { shared.slowscripterr = createObject(REDIS_STRING,sdsnew( "-BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.\r\n")); shared.bgsaveerr = createObject(REDIS_STRING,sdsnew( - "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Write commands are disabled. Please check Redis logs for details about the error.\r\n")); + "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Commands that may modify the data set are disabled. Please check Redis logs for details about the error.\r\n")); + shared.roslaveerr = createObject(REDIS_STRING,sdsnew( + "-READONLY You can't write against a read only slave.\r\n")); + shared.oomerr = createObject(REDIS_STRING, + "-OOM command not allowed when used memory > 'maxmemory'.\r\n"); shared.space = createObject(REDIS_STRING,sdsnew(" ")); shared.colon = createObject(REDIS_STRING,sdsnew(":")); shared.plus = createObject(REDIS_STRING,sdsnew("+")); @@ -1030,6 +1034,7 @@ void initServerConfig() { server.repl_state = REDIS_REPL_NONE; server.repl_syncio_timeout = REDIS_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = 1; + server.repl_slave_ro = 1; server.repl_down_since = -1; /* Client output buffer limits */ @@ -1441,8 +1446,7 @@ int processCommand(redisClient *c) { if (server.maxmemory) { int retval = freeMemoryIfNeeded(); if ((c->cmd->flags & REDIS_CMD_DENYOOM) && retval == REDIS_ERR) { - addReplyError(c, - "command not allowed when used memory > 'maxmemory'"); + addReply(c, shared.oomerr); return REDIS_OK; } } @@ -1457,6 +1461,16 @@ int processCommand(redisClient *c) { return REDIS_OK; } + /* Don't accept wirte commands if this is a read only slave. But + * accept write commands if this is our master. */ + if (server.masterhost && server.repl_slave_ro && + !(c->flags & REDIS_MASTER) && + c->cmd->flags & REDIS_CMD_WRITE) + { + addReply(c, shared.roslaveerr); + return REDIS_OK; + } + /* Only allow SUBSCRIBE and UNSUBSCRIBE in the context of Pub/Sub */ if ((dictSize(c->pubsub_channels) > 0 || listLength(c->pubsub_patterns) > 0) && diff --git a/src/redis.h b/src/redis.h index cbb2df791ad..c6498c459f7 100644 --- a/src/redis.h +++ b/src/redis.h @@ -366,8 +366,8 @@ struct sharedObjectsStruct { *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, - *plus, *select0, *select1, *select2, *select3, *select4, - *select5, *select6, *select7, *select8, *select9, + *roslaveerr, *oomerr, *plus, *select0, *select1, *select2, *select3, + *select4, *select5, *select6, *select7, *select8, *select9, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, *integers[REDIS_SHARED_INTEGERS], @@ -542,6 +542,7 @@ struct redisServer { char *repl_transfer_tmpfile; /* Slave-> master SYNC temp file name */ time_t repl_transfer_lastio; /* Unix time of the latest read, for timeout */ int repl_serve_stale_data; /* Serve stale data when link is down? */ + int repl_slave_ro; /* Slave is read only? */ time_t repl_down_since; /* Unix time at which link with master went down */ /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ @@ -767,6 +768,7 @@ void freeClientMultiState(redisClient *c); void queueMultiCommand(redisClient *c); void touchWatchedKey(redisDb *db, robj *key); void touchWatchedKeysOnFlush(int dbid); +void discardTransaction(redisClient *c); /* Redis object implementation */ void decrRefCount(void *o); diff --git a/src/scripting.c b/src/scripting.c index ce1f0877bf4..e38d08077d0 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -206,15 +206,45 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { goto cleanup; } + /* There are commands that are not allowed inside scripts. */ if (cmd->flags & REDIS_CMD_NOSCRIPT) { luaPushError(lua, "This Redis command is not allowed from scripts"); goto cleanup; } - if (cmd->flags & REDIS_CMD_WRITE && server.lua_random_dirty) { - luaPushError(lua, - "Write commands not allowed after non deterministic commands"); - goto cleanup; + /* Write commands are forbidden against read-only slaves, or if a + * command marked as non-deterministic was already called in the context + * of this script. */ + if (cmd->flags & REDIS_CMD_WRITE) { + if (server.lua_random_dirty) { + luaPushError(lua, + "Write commands not allowed after non deterministic commands"); + goto cleanup; + } else if (server.masterhost && server.repl_slave_ro && + !(server.lua_caller->flags & REDIS_MASTER)) + { + luaPushError(lua, shared.roslaveerr->ptr); + goto cleanup; + } else if (server.stop_writes_on_bgsave_err && + server.saveparamslen > 0 && + server.lastbgsave_status == REDIS_ERR) + { + luaPushError(lua, shared.bgsaveerr->ptr); + goto cleanup; + } + } + + /* If we reached the memory limit configured via maxmemory, commands that + * could enlarge the memory usage are not allowed, but only if this is the + * first write in the context of this script, otherwise we can't stop + * in the middle. */ + if (server.maxmemory && server.lua_write_dirty == 0 && + (cmd->flags & REDIS_CMD_DENYOOM)) + { + if (freeMemoryIfNeeded() == REDIS_ERR) { + luaPushError(lua, shared.oomerr->ptr); + goto cleanup; + } } if (cmd->flags & REDIS_CMD_RANDOM) server.lua_random_dirty = 1; From 38bb45223a0d4759ff453e84171974575a6c04e6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 20 Mar 2012 17:53:47 +0100 Subject: [PATCH 0034/1752] DEBUG should not be flagged as w otherwise we can not call DEBUG DIGEST and other commands against read only slaves. --- src/redis.c | 2 +- tests/support/redis.tcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index b4de5a4317f..9cd54c75cb6 100644 --- a/src/redis.c +++ b/src/redis.c @@ -223,7 +223,7 @@ struct redisCommand redisCommandTable[] = { {"pttl",pttlCommand,2,"r",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"w",0,NULL,1,1,1,0,0}, {"slaveof",slaveofCommand,3,"aws",0,NULL,0,0,0,0,0}, - {"debug",debugCommand,-2,"aws",0,NULL,0,0,0,0,0}, + {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, {"config",configCommand,-2,"ar",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"rps",0,NULL,0,0,0,0,0}, {"unsubscribe",unsubscribeCommand,-1,"rps",0,NULL,0,0,0,0,0}, diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index 4f8ac485dc6..ca6cf34b601 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -160,7 +160,7 @@ proc ::redis::redis_read_reply fd { - {return -code error [redis_read_line $fd]} $ {redis_bulk_read $fd} * {redis_multi_bulk_read $fd} - default {return -code error "Bad protocol, $type as reply type byte"} + default {return -code error "Bad protocol, '$type' as reply type byte"} } } From 3f7ad83398f66dfd7a4beb1b5bf610b499f6b942 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 21 Mar 2012 12:11:07 +0100 Subject: [PATCH 0035/1752] Correctly create shared.oomerr as an sds string. --- src/redis.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 9cd54c75cb6..8cfabba7faa 100644 --- a/src/redis.c +++ b/src/redis.c @@ -926,8 +926,8 @@ void createSharedObjects(void) { "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Commands that may modify the data set are disabled. Please check Redis logs for details about the error.\r\n")); shared.roslaveerr = createObject(REDIS_STRING,sdsnew( "-READONLY You can't write against a read only slave.\r\n")); - shared.oomerr = createObject(REDIS_STRING, - "-OOM command not allowed when used memory > 'maxmemory'.\r\n"); + shared.oomerr = createObject(REDIS_STRING,sdsnew( + "-OOM command not allowed when used memory > 'maxmemory'.\r\n")); shared.space = createObject(REDIS_STRING,sdsnew(" ")); shared.colon = createObject(REDIS_STRING,sdsnew(":")); shared.plus = createObject(REDIS_STRING,sdsnew("+")); From 9aba884b348432a3ef802723a5f6692b353bbaa8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 21 Mar 2012 12:26:05 +0100 Subject: [PATCH 0036/1752] Comments about security of slave-read-only in redis.coinf. --- redis.conf | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 52520fe2c41..5c16143c7c8 100644 --- a/redis.conf +++ b/redis.conf @@ -159,9 +159,17 @@ slave-serve-stale-data yes # You can configure a slave instance to accept writes or not. Writing against # a slave instance may be useful to store some ephemeral data (because data # written on a slave will be easily deleted after resync with the master) but -# may also cause problems if clients are writing to it for an error. +# may also cause problems if clients are writing to it because of a +# misconfiguration. # # Since Redis 2.6 by default slaves are read-only. +# +# Note: read only slaves are not designed to be exposed to untrusted clients +# on the internet. It's just a protection layer against misuse of the instance. +# Still a read only slave exports by default all the administrative commands +# such as CONFIG, DEBUG, and so forth. To a limited extend you can improve +# security of read only slaves using 'rename-command' to shadow all the +# administrative / dangerous commands. slave-read-only yes # Slaves send PINGs to server in a predefined interval. It's possible to change From 7b22c44ccd24e7b0b3033803e9117a7da27f2fd3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 22 Mar 2012 18:14:32 +0100 Subject: [PATCH 0037/1752] Result of INCRBYFLOAT and HINCRBYFLOAT should never be in exponential form, and also should never contain trailing zeroes. This is not possible with vanilla printf() format specifiers, so we alter the output. --- src/object.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index ccb07208511..9699ea53e31 100644 --- a/src/object.c +++ b/src/object.c @@ -48,7 +48,7 @@ robj *createStringObjectFromLongLong(long long value) { /* Note: this function is defined into object.c since here it is where it * belongs but it is actually designed to be used just for INCRBYFLOAT */ robj *createStringObjectFromLongDouble(long double value) { - char buf[256]; + char buf[256], *p; int len; /* We use 17 digits precision since with 128 bit floats that precision @@ -56,7 +56,16 @@ robj *createStringObjectFromLongDouble(long double value) { * that is "non surprising" for the user (that is, most small decimal * numbers will be represented in a way that when converted back into * a string are exactly the same as what the user typed.) */ - len = snprintf(buf,sizeof(buf),"%.17Lg", value); + len = snprintf(buf,sizeof(buf),"%.17Lf", value); + /* Now remove trailing zeroes after the '.' */ + if ((p = strchr(buf,'.')) != NULL) { + p = buf+len-1; + while(*p == '0') { + p--; + len--; + } + if (*p == '.') len--; + } return createStringObject(buf,len); } From b54cdfb2b705d5effef1ebb7637ef77f0fe4b351 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 22 Mar 2012 18:16:41 +0100 Subject: [PATCH 0038/1752] Code style hack. --- src/object.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/object.c b/src/object.c index 9699ea53e31..91e1933af00 100644 --- a/src/object.c +++ b/src/object.c @@ -48,7 +48,7 @@ robj *createStringObjectFromLongLong(long long value) { /* Note: this function is defined into object.c since here it is where it * belongs but it is actually designed to be used just for INCRBYFLOAT */ robj *createStringObjectFromLongDouble(long double value) { - char buf[256], *p; + char buf[256]; int len; /* We use 17 digits precision since with 128 bit floats that precision @@ -58,8 +58,8 @@ robj *createStringObjectFromLongDouble(long double value) { * a string are exactly the same as what the user typed.) */ len = snprintf(buf,sizeof(buf),"%.17Lf", value); /* Now remove trailing zeroes after the '.' */ - if ((p = strchr(buf,'.')) != NULL) { - p = buf+len-1; + if (strchr(buf,'.') != NULL) { + char *p = buf+len-1; while(*p == '0') { p--; len--; From 52192552bde7b0486620df56e5c80c88366ce39c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 23 Mar 2012 10:22:58 +0100 Subject: [PATCH 0039/1752] Replicate HINCRBYFLOAT as HSET. --- src/t_hash.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index b3928450533..5b7a347abbc 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -135,7 +135,9 @@ int hashTypeExists(robj *o, robj *field) { } /* Add an element, discard the old if the key already exists. - * Return 0 on insert and 1 on update. */ + * Return 0 on insert and 1 on update. + * This function will take care of incrementing the reference count of the + * retained fields and value objects. */ int hashTypeSet(robj *o, robj *field, robj *value) { int update = 0; @@ -168,30 +170,23 @@ int hashTypeSet(robj *o, robj *field, robj *value) { zl = ziplistPush(zl, field->ptr, sdslen(field->ptr), ZIPLIST_TAIL); zl = ziplistPush(zl, value->ptr, sdslen(value->ptr), ZIPLIST_TAIL); } - o->ptr = zl; - decrRefCount(field); decrRefCount(value); /* Check if the ziplist needs to be converted to a hash table */ - if (hashTypeLength(o) > server.hash_max_ziplist_entries) { + if (hashTypeLength(o) > server.hash_max_ziplist_entries) hashTypeConvert(o, REDIS_ENCODING_HT); - } - } else if (o->encoding == REDIS_ENCODING_HT) { if (dictReplace(o->ptr, field, value)) { /* Insert */ incrRefCount(field); } else { /* Update */ update = 1; } - incrRefCount(value); - } else { redisPanic("Unknown hash encoding"); } - return update; } @@ -520,7 +515,7 @@ void hincrbyCommand(redisClient *c) { void hincrbyfloatCommand(redisClient *c) { double long value, incr; - robj *o, *current, *new; + robj *o, *current, *new, *aux; if (getLongDoubleFromObjectOrReply(c,c->argv[3],&incr,NULL) != REDIS_OK) return; if ((o = hashTypeLookupWriteOrCreate(c,c->argv[1])) == NULL) return; @@ -540,9 +535,17 @@ void hincrbyfloatCommand(redisClient *c) { hashTypeTryObjectEncoding(o,&c->argv[2],NULL); hashTypeSet(o,c->argv[2],new); addReplyBulk(c,new); - decrRefCount(new); signalModifiedKey(c->db,c->argv[1]); server.dirty++; + + /* Always replicate HINCRBYFLOAT as an HSET command with the final value + * in order to make sure that differences in float pricision or formatting + * will not create differences in replicas or after an AOF restart. */ + aux = createStringObject("HSET",4); + rewriteClientCommandArgument(c,0,aux); + decrRefCount(aux); + rewriteClientCommandArgument(c,3,new); + decrRefCount(new); } static void addHashFieldToReply(redisClient *c, robj *o, robj *field) { From b64281cc0ed8543864030f4de577228b92b5c21b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 23 Mar 2012 12:42:20 +0100 Subject: [PATCH 0040/1752] Big endian fix. The bug was introduced because of a typo. --- src/ziplist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ziplist.c b/src/ziplist.c index 5962510d51c..b214b2da6d9 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -240,7 +240,7 @@ static void zipPrevEncodeLengthForceLarge(unsigned char *p, unsigned int len) { } else if ((prevlensize) == 5) { \ assert(sizeof((prevlensize)) == 4); \ memcpy(&(prevlen), ((char*)(ptr)) + 1, 4); \ - memrev32ifbe(&len); \ + memrev32ifbe(&prevlen); \ } \ } while(0); From ab0603812d6e21a3330f58bb79c5e6a9248f9847 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 23 Mar 2012 15:22:25 +0100 Subject: [PATCH 0041/1752] RDB load of different encodings test added. --- tests/assets/encodings.rdb | Bin 0 -> 667 bytes tests/integration/rdb.tcl | 25 +++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 3 files changed, 26 insertions(+) create mode 100644 tests/assets/encodings.rdb create mode 100644 tests/integration/rdb.tcl diff --git a/tests/assets/encodings.rdb b/tests/assets/encodings.rdb new file mode 100644 index 0000000000000000000000000000000000000000..9fd9b705d16220065ee117a1c1c094f40fb122f2 GIT binary patch literal 667 zcmbVKu}UOC5UuKJ7oAzb-~v%NHW5S&dS+bpFfmX#Qw_|N?w&>$M|aur5EcU?;j)7> zGV&XY4SF>ZBR^rkz`zf1tzKQwOdP0r-Cfn)uU@~+^|g&HrPRU;knEK1xGIbhsS?(T zOp!5$Ql%uLiRxVU_K~%gGG1r2V@aAV)EAeQf1$=iXe|~B7q#x40{=g8Nhbr35aH0(af04| z31?FkfXdOIL*v>$`m`ZiW?H~$fQQSK0B})18Q{+2^#ErNo(A|lG8gy_c?x0;Mx(`{ zoa#1QiP|F?FVK2I8DyCB=mk$S8nlC&4|~3w8;|#Ox&QtGwHmXU=HND1)gUq}8+2xM zS?Yc@4yO2OwUpuPICQ~2@KI=m_~m`huJS+FRQ_l1RQDc;oztC1%JaPY56L Date: Fri, 23 Mar 2012 20:21:19 +0100 Subject: [PATCH 0042/1752] Fixed memory leak in hash loading. --- src/rdb.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 519b645d157..1e23fa70cf3 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -859,14 +859,17 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { /* Add pair to ziplist */ o->ptr = ziplistPush(o->ptr, field->ptr, sdslen(field->ptr), ZIPLIST_TAIL); o->ptr = ziplistPush(o->ptr, value->ptr, sdslen(value->ptr), ZIPLIST_TAIL); - /* Convert to hash table if size threshold is exceeded */ if (sdslen(field->ptr) > server.hash_max_ziplist_value || sdslen(value->ptr) > server.hash_max_ziplist_value) { + decrRefCount(field); + decrRefCount(value); hashTypeConvert(o, REDIS_ENCODING_HT); break; } + decrRefCount(field); + decrRefCount(value); } /* Load remaining fields and values into the hash table */ From 27688de10e4047c5a747330b441d79a4c43f0c98 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 23 Mar 2012 15:22:25 +0100 Subject: [PATCH 0043/1752] RDB load of different encodings test added. --- tests/assets/encodings.rdb | Bin 0 -> 667 bytes tests/integration/rdb.tcl | 25 +++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 3 files changed, 26 insertions(+) create mode 100644 tests/assets/encodings.rdb create mode 100644 tests/integration/rdb.tcl diff --git a/tests/assets/encodings.rdb b/tests/assets/encodings.rdb new file mode 100644 index 0000000000000000000000000000000000000000..9fd9b705d16220065ee117a1c1c094f40fb122f2 GIT binary patch literal 667 zcmbVKu}UOC5UuKJ7oAzb-~v%NHW5S&dS+bpFfmX#Qw_|N?w&>$M|aur5EcU?;j)7> zGV&XY4SF>ZBR^rkz`zf1tzKQwOdP0r-Cfn)uU@~+^|g&HrPRU;knEK1xGIbhsS?(T zOp!5$Ql%uLiRxVU_K~%gGG1r2V@aAV)EAeQf1$=iXe|~B7q#x40{=g8Nhbr35aH0(af04| z31?FkfXdOIL*v>$`m`ZiW?H~$fQQSK0B})18Q{+2^#ErNo(A|lG8gy_c?x0;Mx(`{ zoa#1QiP|F?FVK2I8DyCB=mk$S8nlC&4|~3w8;|#Ox&QtGwHmXU=HND1)gUq}8+2xM zS?Yc@4yO2OwUpuPICQ~2@KI=m_~m`huJS+FRQ_l1RQDc;oztC1%JaPY56L Date: Fri, 23 Mar 2012 20:20:43 +0100 Subject: [PATCH 0044/1752] Contextualize comment. --- tests/integration/rdb.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index fdc5495f753..85c5db9f5a3 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -1,6 +1,6 @@ set server_path [tmpdir "server.rdb-encoding-test"] -# Copy RDB with zipmap encoded hash to server path +# Copy RDB with different encodings in server path exec cp tests/assets/encodings.rdb $server_path start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rdb"]] { From 9542d9d8d7e5035e5f884eec3e4f922d51145bd5 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 24 Mar 2012 11:42:20 +0100 Subject: [PATCH 0045/1752] convert-zipmap-hash-on-load test enabled --- tests/test_helper.tcl | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 4954b79e5b5..bef8231a71a 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -31,6 +31,7 @@ set ::all_tests { integration/replication-3 integration/aof integration/rdb + integration/convert-zipmap-hash-on-load unit/pubsub unit/slowlog unit/scripting From 188a17ed563ef59bffa8b47b365079f264859b90 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 24 Mar 2012 11:52:56 +0100 Subject: [PATCH 0046/1752] Add used allocator in redis-server -v output. --- src/redis.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 8cfabba7faa..44c75353ca5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2185,8 +2185,8 @@ void daemonize(void) { } void version() { - printf("Redis server version %s (%s:%d)\n", REDIS_VERSION, - redisGitSHA1(), atoi(redisGitDirty()) > 0); + printf("Redis server v=%s sha=%s:%d malloc=%s\n", REDIS_VERSION, + redisGitSHA1(), atoi(redisGitDirty()) > 0, ZMALLOC_LIB); exit(0); } From 29e1976855f9aabb77407c018db5649c15427e22 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 24 Mar 2012 12:06:56 +0100 Subject: [PATCH 0047/1752] When running the test in valgrind mode, pass the right flags to show memory leaks stack traces but only including the "definitely lost" items. --- tests/support/server.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 948b5356ed1..928e5c40abe 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -176,7 +176,7 @@ proc start_server {options {code undefined}} { set stderr [format "%s/%s" [dict get $config "dir"] "stderr"] if {$::valgrind} { - exec valgrind --suppressions=src/valgrind.sup src/redis-server $config_file > $stdout 2> $stderr & + exec valgrind --suppressions=src/valgrind.sup --show-reachable=no --show-possibly-lost=no --leak-check=full src/redis-server $config_file > $stdout 2> $stderr & } else { exec src/redis-server $config_file > $stdout 2> $stderr & } From 0387e9c1994e1c8699ee1252a2dd2ab6757e91da Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 25 Mar 2012 10:57:34 +0200 Subject: [PATCH 0048/1752] convert-zipmap-hash-on-load false positive fixed. Apparently because the sample RDB file was not copied before every test Redis had a chance to replace it with a newly written one, so that the next test could fail. --- tests/integration/convert-zipmap-hash-on-load.tcl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/convert-zipmap-hash-on-load.tcl b/tests/integration/convert-zipmap-hash-on-load.tcl index 75a65d3e8f7..cf3577f2846 100644 --- a/tests/integration/convert-zipmap-hash-on-load.tcl +++ b/tests/integration/convert-zipmap-hash-on-load.tcl @@ -1,8 +1,7 @@ -set server_path [tmpdir "server.convert-zipmap-hash-on-load"] - # Copy RDB with zipmap encoded hash to server path -exec cp tests/assets/hash-zipmap.rdb $server_path +set server_path [tmpdir "server.convert-zipmap-hash-on-load"] +exec cp -f tests/assets/hash-zipmap.rdb $server_path start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb"]] { test "RDB load zipmap hash: converts to ziplist" { r select 0 @@ -13,6 +12,7 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap. } } +exec cp -f tests/assets/hash-zipmap.rdb $server_path start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-entries" 1]] { test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-entries is exceeded" { r select 0 @@ -23,6 +23,7 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap. } } +exec cp -f tests/assets/hash-zipmap.rdb $server_path start_server [list overrides [list "dir" $server_path "dbfilename" "hash-zipmap.rdb" "hash-max-ziplist-value" 1]] { test "RDB load zipmap hash: converts to hash table when hash-max-ziplist-value is exceeded" { r select 0 From 754643c1c703c773e03efec23a2b55ac5ba324cb Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 25 Mar 2012 11:18:24 +0200 Subject: [PATCH 0049/1752] Version is now 2.5.3. --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index a79cbe97edd..63238004f3e 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.2" +#define REDIS_VERSION "2.5.3" From 81f32c7b65c72a45fe1637ab0a661b144c54eb7c Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 25 Mar 2012 11:27:35 +0200 Subject: [PATCH 0050/1752] New INFO field aof_delayed_fsync introduced. This new field counts all the times Redis is configured with AOF enabled and fsync policy 'everysec', but the previous fsync performed by the background thread was not able to complete within two seconds, forcing Redis to perform a write against the AOF file while the fsync is still in progress (likely a blocking operation). --- src/aof.c | 1 + src/config.c | 1 + src/redis.c | 7 +++++-- src/redis.h | 1 + 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/aof.c b/src/aof.c index 83633217f79..4d3ce096ace 100644 --- a/src/aof.c +++ b/src/aof.c @@ -108,6 +108,7 @@ void flushAppendOnlyFile(int force) { } /* Otherwise fall trough, and go write since we can't wait * over two seconds. */ + server.aof_delayed_fsync++; redisLog(REDIS_NOTICE,"Asynchronous AOF fsync is taking too long (disk is busy?). Writing the AOF buffer without waiting for fsync to complete, this may slow down Redis."); } } diff --git a/src/config.c b/src/config.c index 238c9462f8a..d38ba72b13f 100644 --- a/src/config.c +++ b/src/config.c @@ -852,6 +852,7 @@ void configCommand(redisClient *c) { server.stat_numcommands = 0; server.stat_numconnections = 0; server.stat_expiredkeys = 0; + server.aof_delayed_fsync = 0; resetCommandTableStats(); addReply(c,shared.ok); } else { diff --git a/src/redis.c b/src/redis.c index 44c75353ca5..39de86af641 100644 --- a/src/redis.c +++ b/src/redis.c @@ -991,6 +991,7 @@ void initServerConfig() { server.aof_rewrite_base_size = 0; server.aof_rewrite_scheduled = 0; server.aof_last_fsync = time(NULL); + server.aof_delayed_fsync = 0; server.aof_fd = -1; server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; @@ -1760,12 +1761,14 @@ sds genRedisInfoString(char *section) { "aof_base_size:%lld\r\n" "aof_pending_rewrite:%d\r\n" "aof_buffer_length:%zu\r\n" - "aof_pending_bio_fsync:%llu\r\n", + "aof_pending_bio_fsync:%llu\r\n" + "aof_delayed_fsync:%lu\r\n", (long long) server.aof_current_size, (long long) server.aof_rewrite_base_size, server.aof_rewrite_scheduled, sdslen(server.aof_buf), - bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC)); + bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC), + server.aof_delayed_fsync); } if (server.loading) { diff --git a/src/redis.h b/src/redis.h index c6498c459f7..4b5c94dc3ea 100644 --- a/src/redis.h +++ b/src/redis.h @@ -509,6 +509,7 @@ struct redisServer { int aof_selected_db; /* Currently selected DB in AOF */ time_t aof_flush_postponed_start; /* UNIX time of postponed AOF flush */ time_t aof_last_fsync; /* UNIX time of last fsync() */ + unsigned long aof_delayed_fsync; /* delayed AOF fsync() counter */ /* RDB persistence */ long long dirty; /* Changes to DB from the last save */ long long dirty_before_bgsave; /* Used to restore dirty on failed BGSAVE */ From d0407c2d85f9c2ce2971a3cb96b958ae7e03b40f Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 25 Mar 2012 11:43:19 +0200 Subject: [PATCH 0051/1752] CONFIG RESETSTAT resets two more fields. --- src/config.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config.c b/src/config.c index d38ba72b13f..a882db35704 100644 --- a/src/config.c +++ b/src/config.c @@ -852,6 +852,8 @@ void configCommand(redisClient *c) { server.stat_numcommands = 0; server.stat_numconnections = 0; server.stat_expiredkeys = 0; + server.stat_rejected_conn = 0; + server.stat_fork_time = 0; server.aof_delayed_fsync = 0; resetCommandTableStats(); addReply(c,shared.ok); From 9ea95e6c64ca00b85f7da967f7a9c1a48d87635d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Mar 2012 10:33:45 +0200 Subject: [PATCH 0052/1752] SIGSEGV handler refactored so that we can reuse stack trace and current client logging functionalities in other contexts. --- src/debug.c | 109 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 47 deletions(-) diff --git a/src/debug.c b/src/debug.c index 8eea3bf8d04..9c8b6ef97d1 100644 --- a/src/debug.c +++ b/src/debug.c @@ -558,21 +558,11 @@ void logRegisters(ucontext_t *uc) { #endif } -void sigsegvHandler(int sig, siginfo_t *info, void *secret) { +/* Logs the stack trace using the backtrace() call. */ +void logStackTrace(ucontext_t *uc) { void *trace[100]; - char **messages = NULL; int i, trace_size = 0; - ucontext_t *uc = (ucontext_t*) secret; - sds infostring, clients; - struct sigaction act; - REDIS_NOTUSED(info); - - bugReportStart(); - redisLog(REDIS_WARNING, - " Redis %s crashed by signal: %d", REDIS_VERSION, sig); - redisLog(REDIS_WARNING, - " Failed assertion: %s (%s:%d)", server.assert_failed, - server.assert_file, server.assert_line); + char **messages = NULL; /* Generate the stack trace */ trace_size = backtrace(trace, 100); @@ -585,6 +575,61 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { redisLog(REDIS_WARNING, "--- STACK TRACE"); for (i=1; iargc; j++) { + robj *decoded; + + decoded = getDecodedObject(cc->argv[j]); + redisLog(REDIS_WARNING,"argv[%d]: '%s'", j, (char*)decoded->ptr); + decrRefCount(decoded); + } + /* Check if the first argument, usually a key, is found inside the + * selected DB, and if so print info about the associated object. */ + if (cc->argc >= 1) { + robj *val, *key; + dictEntry *de; + + key = getDecodedObject(cc->argv[1]); + de = dictFind(cc->db->dict, key->ptr); + if (de) { + val = dictGetVal(de); + redisLog(REDIS_WARNING,"key '%s' found in DB containing the following object:", key->ptr); + redisLogObjectDebugInfo(val); + } + decrRefCount(key); + } +} + +void sigsegvHandler(int sig, siginfo_t *info, void *secret) { + ucontext_t *uc = (ucontext_t*) secret; + sds infostring, clients; + struct sigaction act; + REDIS_NOTUSED(info); + + bugReportStart(); + redisLog(REDIS_WARNING, + " Redis %s crashed by signal: %d", REDIS_VERSION, sig); + redisLog(REDIS_WARNING, + " Failed assertion: %s (%s:%d)", server.assert_failed, + server.assert_file, server.assert_line); + + /* Log the stack trace */ + logStackTrace(uc); /* Log INFO and CLIENT LIST */ redisLog(REDIS_WARNING, "--- INFO OUTPUT"); @@ -595,41 +640,11 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { redisLog(REDIS_WARNING, "--- CLIENT LIST OUTPUT"); clients = getAllClientsInfoString(); redisLogRaw(REDIS_WARNING, clients); - /* Don't sdsfree() strings to avoid a crash. Memory may be corrupted. */ - - /* Log CURRENT CLIENT info */ - if (server.current_client) { - redisClient *cc = server.current_client; - sds client; - int j; + sdsfree(infostring); + sdsfree(clients); - redisLog(REDIS_WARNING, "--- CURRENT CLIENT INFO"); - client = getClientInfoString(cc); - redisLog(REDIS_WARNING,"client: %s", client); - /* Missing sdsfree(client) to avoid crash if memory is corrupted. */ - for (j = 0; j < cc->argc; j++) { - robj *decoded; - - decoded = getDecodedObject(cc->argv[j]); - redisLog(REDIS_WARNING,"argv[%d]: '%s'", j, (char*)decoded->ptr); - decrRefCount(decoded); - } - /* Check if the first argument, usually a key, is found inside the - * selected DB, and if so print info about the associated object. */ - if (cc->argc >= 1) { - robj *val, *key; - dictEntry *de; - - key = getDecodedObject(cc->argv[1]); - de = dictFind(cc->db->dict, key->ptr); - if (de) { - val = dictGetVal(de); - redisLog(REDIS_WARNING,"key '%s' found in DB containing the following object:", key->ptr); - redisLogObjectDebugInfo(val); - } - decrRefCount(key); - } - } + /* Log the current client */ + logCurrentClient(); /* Log dump of processor registers */ logRegisters(uc); From efd412f908a03696aa0a44dbf661be6bd12f51ff Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 27 Mar 2012 18:46:51 +0200 Subject: [PATCH 0053/1752] remove disk-store related comments --- src/db.c | 24 ------------------------ src/object.c | 4 +--- 2 files changed, 1 insertion(+), 27 deletions(-) diff --git a/src/db.c b/src/db.c index 7ca2a5f66f7..160ffb79090 100644 --- a/src/db.c +++ b/src/db.c @@ -10,28 +10,6 @@ void SlotToKeyDel(robj *key); * C-level DB API *----------------------------------------------------------------------------*/ -/* Important notes on lookup and disk store. - * - * When disk store is enabled on lookup we can have different cases. - * - * a) The key is in memory: - * - If the key is not in IO_SAVEINPROG state we can access it. - * As if it's just IO_SAVE this means we have the key in the IO queue - * but can't be accessed by the IO thread (it requires to be - * translated into an IO Job by the cache cron function.) - * - If the key is in IO_SAVEINPROG we can't touch the key and have - * to blocking wait completion of operations. - * b) The key is not in memory: - * - If it's marked as non existing on disk as well (negative cache) - * we don't need to perform the disk access. - * - if the key MAY EXIST, but is not in memory, and it is marked as IO_SAVE - * then the key can only be a deleted one. As IO_SAVE keys are never - * evicted (dirty state), so the only possibility is that key was deleted. - * - if the key MAY EXIST we need to blocking load it. - * We check that the key is not in IO_SAVEINPROG state before accessing - * the disk object. If it is in this state, we wait. - */ - robj *lookupKey(redisDb *db, robj *key) { dictEntry *de = dictFind(db->dict,key->ptr); if (de) { @@ -159,8 +137,6 @@ int dbDelete(redisDb *db, robj *key) { } } -/* Empty the whole database. - * If diskstore is enabled this function will just flush the in-memory cache. */ long long emptyDb() { int j; long long removed = 0; diff --git a/src/object.c b/src/object.c index 91e1933af00..a15ebcfa837 100644 --- a/src/object.c +++ b/src/object.c @@ -270,9 +270,7 @@ robj *tryObjectEncoding(robj *o) { /* Ok, this object can be encoded... * - * Can I use a shared object? Only if the object is inside a given - * range and if the back end in use is in-memory. For disk store every - * object in memory used as value should be independent. + * Can I use a shared object? Only if the object is inside a given range * * Note that we also avoid using shared integers when maxmemory is used * because every object needs to have a private LRU field for the LRU From 38d6976c0340863daae3e67d549413ee0694c7aa Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 27 Mar 2012 18:18:57 +0200 Subject: [PATCH 0054/1752] declare hashDictType as external too --- src/redis.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.h b/src/redis.h index 4b5c94dc3ea..de18fef44bf 100644 --- a/src/redis.h +++ b/src/redis.h @@ -683,7 +683,7 @@ extern dictType setDictType; extern dictType zsetDictType; extern dictType dbDictType; extern double R_Zero, R_PosInf, R_NegInf, R_Nan; -dictType hashDictType; +extern dictType hashDictType; /*----------------------------------------------------------------------------- * Functions prototypes From 56ff70f8e0eec1f883166c50c877a1e78a7d3123 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 27 Mar 2012 17:39:58 +0200 Subject: [PATCH 0055/1752] use server.unixtime instead of time(NULL) where possible (cluster.c not checked though) --- src/aof.c | 2 +- src/networking.c | 13 ++++++------- src/redis.c | 16 +++++++--------- src/replication.c | 10 +++++----- src/t_list.c | 2 +- 5 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/aof.c b/src/aof.c index 4d3ce096ace..3f7cd10ffb8 100644 --- a/src/aof.c +++ b/src/aof.c @@ -46,7 +46,7 @@ void stopAppendOnly(void) { /* Called when the user switches from "appendonly no" to "appendonly yes" * at runtime using the CONFIG command. */ int startAppendOnly(void) { - server.aof_last_fsync = time(NULL); + server.aof_last_fsync = server.unixtime; server.aof_fd = open(server.aof_filename,O_WRONLY|O_APPEND|O_CREAT,0644); redisAssert(server.aof_state == REDIS_AOF_OFF); if (server.aof_fd == -1) { diff --git a/src/networking.c b/src/networking.c index 375186d1e01..937f8402a56 100644 --- a/src/networking.c +++ b/src/networking.c @@ -52,7 +52,7 @@ redisClient *createClient(int fd) { c->bulklen = -1; c->sentlen = 0; c->flags = 0; - c->ctime = c->lastinteraction = time(NULL); + c->ctime = c->lastinteraction = server.unixtime; c->authenticated = 0; c->replstate = REDIS_REPL_NONE; c->reply = listCreate(); @@ -604,7 +604,7 @@ void freeClient(redisClient *c) { if (c->flags & REDIS_MASTER) { server.master = NULL; server.repl_state = REDIS_REPL_CONNECT; - server.repl_down_since = time(NULL); + server.repl_down_since = server.unixtime; /* Since we lost the connection with the master, we should also * close the connection with all our slaves if we have any, so * when we'll resync with the master the other slaves will sync again @@ -732,7 +732,7 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { return; } } - if (totwritten > 0) c->lastinteraction = time(NULL); + if (totwritten > 0) c->lastinteraction = server.unixtime; if (c->bufpos == 0 && listLength(c->reply) == 0) { c->sentlen = 0; aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); @@ -1017,7 +1017,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { } if (nread) { sdsIncrLen(c->querybuf,nread); - c->lastinteraction = time(NULL); + c->lastinteraction = server.unixtime; } else { server.current_client = NULL; return; @@ -1058,7 +1058,6 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, sds getClientInfoString(redisClient *client) { char ip[32], flags[16], events[3], *p; int port; - time_t now = time(NULL); int emask; anetPeerToString(client->fd,ip,&port); @@ -1087,8 +1086,8 @@ sds getClientInfoString(redisClient *client) { return sdscatprintf(sdsempty(), "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", ip,port,client->fd, - (long)(now - client->ctime), - (long)(now - client->lastinteraction), + (long)(server.unixtime - client->ctime), + (long)(server.unixtime - client->lastinteraction), flags, client->db->id, (int) dictSize(client->pubsub_channels), diff --git a/src/redis.c b/src/redis.c index 39de86af641..a91acf0e732 100644 --- a/src/redis.c +++ b/src/redis.c @@ -599,7 +599,7 @@ void activeExpireCycle(void) { } void updateLRUClock(void) { - server.lruclock = (time(NULL)/REDIS_LRU_CLOCK_RESOLUTION) & + server.lruclock = (server.unixtime/REDIS_LRU_CLOCK_RESOLUTION) & REDIS_LRU_CLOCK_MAX; } @@ -808,15 +808,13 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { updateDictResizePolicy(); } } else { - time_t now = time(NULL); - /* If there is not a background saving/rewrite in progress check if * we have to save/rewrite now */ for (j = 0; j < server.saveparamslen; j++) { struct saveparam *sp = server.saveparams+j; if (server.dirty >= sp->changes && - now-server.lastsave > sp->seconds) { + server.unixtime-server.lastsave > sp->seconds) { redisLog(REDIS_NOTICE,"%d changes in %d seconds. Saving...", sp->changes, sp->seconds); rdbSaveBackground(server.rdb_filename); @@ -1641,7 +1639,7 @@ void bytesToHuman(char *s, unsigned long long n) { * on memory corruption problems. */ sds genRedisInfoString(char *section) { sds info = sdsempty(); - time_t uptime = time(NULL)-server.stat_starttime; + time_t uptime = server.unixtime-server.stat_starttime; int j, numcommands; struct rusage self_ru, c_ru; unsigned long lol, bib; @@ -1780,7 +1778,7 @@ sds genRedisInfoString(char *section) { perc = ((double)server.loading_loaded_bytes / server.loading_total_bytes) * 100; - elapsed = time(NULL)-server.loading_start_time; + elapsed = server.unixtime-server.loading_start_time; if (elapsed == 0) { eta = 1; /* A fake 1 second figure if we don't have enough info */ @@ -1851,7 +1849,7 @@ sds genRedisInfoString(char *section) { (server.repl_state == REDIS_REPL_CONNECTED) ? "up" : "down", server.master ? - ((int)(time(NULL)-server.master->lastinteraction)) : -1, + ((int)(server.unixtime-server.master->lastinteraction)) : -1, server.repl_state == REDIS_REPL_TRANSFER ); @@ -1860,14 +1858,14 @@ sds genRedisInfoString(char *section) { "master_sync_left_bytes:%ld\r\n" "master_sync_last_io_seconds_ago:%d\r\n" ,(long)server.repl_transfer_left, - (int)(time(NULL)-server.repl_transfer_lastio) + (int)(server.unixtime-server.repl_transfer_lastio) ); } if (server.repl_state != REDIS_REPL_CONNECTED) { info = sdscatprintf(info, "master_link_down_since_seconds:%ld\r\n", - (long)time(NULL)-server.repl_down_since); + (long)server.unixtime-server.repl_down_since); } } info = sdscatprintf(info, diff --git a/src/replication.c b/src/replication.c index 6c0091e8c62..170bee73e1e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -307,7 +307,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { /* At this stage just a newline works as a PING in order to take * the connection live. So we refresh our last interaction * timestamp. */ - server.repl_transfer_lastio = time(NULL); + server.repl_transfer_lastio = server.unixtime; return; } else if (buf[0] != '$') { redisLog(REDIS_WARNING,"Bad protocol from MASTER, the first byte is not '$', are you sure the host and port are right?"); @@ -330,7 +330,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { replicationAbortSyncTransfer(); return; } - server.repl_transfer_lastio = time(NULL); + server.repl_transfer_lastio = server.unixtime; if (write(server.repl_transfer_fd,buf,nread) != nread) { redisLog(REDIS_WARNING,"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchrnonization: %s", strerror(errno)); goto error; @@ -441,7 +441,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* Prepare a suitable temp file for bulk transfer */ while(maxtries--) { snprintf(tmpfile,256, - "temp-%d.%ld.rdb",(int)time(NULL),(long int)getpid()); + "temp-%d.%ld.rdb",(int)server.unixtime,(long int)getpid()); dfd = open(tmpfile,O_CREAT|O_WRONLY|O_EXCL,0644); if (dfd != -1) break; sleep(1); @@ -462,7 +462,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { server.repl_state = REDIS_REPL_TRANSFER; server.repl_transfer_left = -1; server.repl_transfer_fd = dfd; - server.repl_transfer_lastio = time(NULL); + server.repl_transfer_lastio = server.unixtime; server.repl_transfer_tmpfile = zstrdup(tmpfile); return; @@ -490,7 +490,7 @@ int connectWithMaster(void) { return REDIS_ERR; } - server.repl_transfer_lastio = time(NULL); + server.repl_transfer_lastio = server.unixtime; server.repl_transfer_s = fd; server.repl_state = REDIS_REPL_CONNECTING; return REDIS_OK; diff --git a/src/t_list.c b/src/t_list.c index 2be8074a7cc..6a16a632019 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -910,7 +910,7 @@ int getTimeoutFromObjectOrReply(redisClient *c, robj *object, time_t *timeout) { return REDIS_ERR; } - if (tval > 0) tval += time(NULL); + if (tval > 0) tval += server.unixtime; *timeout = tval; return REDIS_OK; From 92dd4e4301e8b2ff4324e968ced6b9eb8f8a2328 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 27 Mar 2012 17:19:54 +0200 Subject: [PATCH 0056/1752] fix time() instead of mstime() in expireIfNeeded --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 160ffb79090..73ee9166a00 100644 --- a/src/db.c +++ b/src/db.c @@ -483,7 +483,7 @@ int expireIfNeeded(redisDb *db, robj *key) { * that is, 0 if we think the key should be still valid, 1 if * we think the key is expired at this time. */ if (server.masterhost != NULL) { - return time(NULL) > when; + return mstime() > when; } /* Return when this key has not expired */ From 085aaef325ff841dbf412861b238b7e093b2867d Mon Sep 17 00:00:00 2001 From: huangz1990 Date: Thu, 15 Mar 2012 14:27:14 +0800 Subject: [PATCH 0057/1752] fix typo --- src/dict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.c b/src/dict.c index 53e16be0ff2..6fd4584eede 100644 --- a/src/dict.c +++ b/src/dict.c @@ -150,7 +150,7 @@ int _dictInit(dict *d, dictType *type, } /* Resize the table to the minimal size that contains all the elements, - * but with the invariant of a USER/BUCKETS ratio near to <= 1 */ + * but with the invariant of a USED/BUCKETS ratio near to <= 1 */ int dictResize(dict *d) { int minimal; From fc030ac7deecab6769177c44fe2859a76cce4286 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Mar 2012 11:47:51 +0200 Subject: [PATCH 0058/1752] Redis software watchdog. --- src/config.c | 7 ++++++ src/debug.c | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++-- src/redis.c | 7 +++++- src/redis.h | 4 ++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index a882db35704..804e251493c 100644 --- a/src/config.c +++ b/src/config.c @@ -620,6 +620,12 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"repl-timeout")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; server.repl_timeout = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"watchdog-period")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + if (ll) + enableWatchdog(ll); + else + disableWatchdog(); } else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -708,6 +714,7 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); config_get_numerical_field("repl-timeout",server.repl_timeout); config_get_numerical_field("maxclients",server.maxclients); + config_get_numerical_field("watchdog-period",server.watchdog_period); /* Bool (yes/no) values */ config_get_bool_field("no-appendfsync-on-rewrite", diff --git a/src/debug.c b/src/debug.c index 9c8b6ef97d1..2df913b071d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -661,11 +661,72 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { /* Make sure we exit with the right signal at the end. So for instance * the core will be dumped if enabled. */ sigemptyset (&act.sa_mask); - /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction - * is used. Otherwise, sa_handler is used */ act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; act.sa_handler = SIG_DFL; sigaction (sig, &act, NULL); kill(getpid(),sig); } #endif /* HAVE_BACKTRACE */ + +/* =========================== Software Watchdog ============================ */ +#include + +void watchdogSignalHandler(int sig, siginfo_t *info, void *secret) { + ucontext_t *uc = (ucontext_t*) secret; + REDIS_NOTUSED(info); + REDIS_NOTUSED(sig); + + /* Log INFO and CLIENT LIST */ + redisLog(REDIS_WARNING, "--- WATCHDOG TIMER EXPIRED ---"); +#ifdef HAVE_BACKTRACE + logStackTrace(uc); + redisLog(REDIS_WARNING, "------"); +#endif +} + +/* Schedule a SIGALRM delivery after the specified period in milliseconds. + * If a timer is already scheduled, this function will re-schedule it to the + * specified time. If period is 0 the current timer is disabled. */ +void watchdogScheduleSignal(int period) { + struct itimerval it; + + /* Will stop the timer if period is 0. */ + it.it_value.tv_sec = period/1000; + it.it_value.tv_usec = period%1000; + /* Don't automatically restart. */ + it.it_interval.tv_sec = 0; + it.it_interval.tv_usec = 0; + setitimer(ITIMER_REAL, &it, NULL); +} + +/* Enable the software watchdong with the specified period in milliseconds. */ +void enableWatchdog(int period) { + if (server.watchdog_period == 0) { + struct sigaction act; + + /* Watchdog was actually disabled, so we have to setup the signal + * handler. */ + sigemptyset(&act.sa_mask); + act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_SIGINFO; + act.sa_sigaction = watchdogSignalHandler; + sigaction(SIGALRM, &act, NULL); + } + if (period < 200) period = 200; /* We don't accept periods < 200 ms. */ + watchdogScheduleSignal(period); /* Adjust the current timer. */ + server.watchdog_period = period; +} + +/* Disable the software watchdog. */ +void disableWatchdog(void) { + struct sigaction act; + if (server.watchdog_period == 0) return; /* Already disabled. */ + watchdogScheduleSignal(0); /* Stop the current timer. */ + + /* Set the signal handler to SIG_IGN, this will also remove pending + * signals from the queue. */ + sigemptyset(&act.sa_mask); + act.sa_flags = 0; + act.sa_handler = SIG_IGN; + sigaction(SIGALRM, &act, NULL); + server.watchdog_period = 0; +} diff --git a/src/redis.c b/src/redis.c index a91acf0e732..958827f720f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -713,6 +713,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { REDIS_NOTUSED(id); REDIS_NOTUSED(clientData); + /* Software watchdog: deliver the SIGALRM that will reach the signal + * handler if we don't return here fast enough. */ + if (server.watchdog_period) watchdogScheduleSignal(server.watchdog_period); + /* We take a cached value of the unix time in the global state because * with virtual memory and aging there is to store the current time * in objects at every object access, and accuracy is not needed. @@ -1066,11 +1070,12 @@ void initServerConfig() { server.slowlog_log_slower_than = REDIS_SLOWLOG_LOG_SLOWER_THAN; server.slowlog_max_len = REDIS_SLOWLOG_MAX_LEN; - /* Assert */ + /* Debugging */ server.assert_failed = ""; server.assert_file = ""; server.assert_line = 0; server.bug_report_start = 0; + server.watchdog_period = 0; } /* This function will try to raise the max number of open files accordingly to diff --git a/src/redis.h b/src/redis.h index de18fef44bf..3392b5dcffa 100644 --- a/src/redis.h +++ b/src/redis.h @@ -590,6 +590,7 @@ struct redisServer { char *assert_file; int assert_line; int bug_report_start; /* True if bug report header was already logged. */ + int watchdog_period; /* Software watchdog period in ms. 0 = off */ }; typedef struct pubsubPattern { @@ -1109,4 +1110,7 @@ void bugReportStart(void); void redisLogObjectDebugInfo(robj *o); void sigsegvHandler(int sig, siginfo_t *info, void *secret); sds genRedisInfoString(char *section); +void enableWatchdog(int period); +void disableWatchdog(void); +void watchdogScheduleSignal(int period); #endif From 1507be5b01df4fcbaf42ca514a79b4b6c7f19d23 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Mar 2012 12:11:37 +0200 Subject: [PATCH 0059/1752] Correctly set the SIGARLM timer for the software watchdog. --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index 2df913b071d..30fb0091598 100644 --- a/src/debug.c +++ b/src/debug.c @@ -692,7 +692,7 @@ void watchdogScheduleSignal(int period) { /* Will stop the timer if period is 0. */ it.it_value.tv_sec = period/1000; - it.it_value.tv_usec = period%1000; + it.it_value.tv_usec = (period%1000)*1000; /* Don't automatically restart. */ it.it_interval.tv_sec = 0; it.it_interval.tv_usec = 0; From e4cd5838fc6a816146fecf94f1c33441ff61ec89 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Mar 2012 13:48:53 +0200 Subject: [PATCH 0060/1752] Mask SIGALRM everything but in the main thread. This is required to ensure that the signal will be delivered to the main thread when the watchdog timer expires. --- src/bio.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/bio.c b/src/bio.c index eaac8e40d55..aa2cdf9fbb5 100644 --- a/src/bio.c +++ b/src/bio.c @@ -108,9 +108,18 @@ void bioCreateBackgroundJob(int type, void *arg1, void *arg2, void *arg3) { void *bioProcessBackgroundJobs(void *arg) { struct bio_job *job; unsigned long type = (unsigned long) arg; + sigset_t sigset; pthread_detach(pthread_self()); pthread_mutex_lock(&bio_mutex[type]); + /* Block SIGALRM so we are sure that only the main thread will + * receive the watchdog signal. */ + sigemptyset(&sigset); + sigaddset(&sigset, SIGALRM); + if (pthread_sigmask(SIG_BLOCK, &sigset, NULL)) + redisLog(REDIS_WARNING, + "Warning: can't mask SIGALRM in bio.c thread: %s", strerror(errno)); + while(1) { listNode *ln; From 59d884af8c43a82a4f3d67c55059a7c453601d05 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Mar 2012 15:24:33 +0200 Subject: [PATCH 0061/1752] Produce the watchlog warning log in a way that is safer from a signal handler. Fix a memory leak in the backtrace generation function. --- src/debug.c | 41 +++++++++++++++++++++++++++++++---------- src/zmalloc.c | 4 ++++ src/zmalloc.h | 1 + 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/src/debug.c b/src/debug.c index 30fb0091598..49c76824d22 100644 --- a/src/debug.c +++ b/src/debug.c @@ -559,10 +559,11 @@ void logRegisters(ucontext_t *uc) { } /* Logs the stack trace using the backtrace() call. */ -void logStackTrace(ucontext_t *uc) { +sds getStackTrace(ucontext_t *uc) { void *trace[100]; int i, trace_size = 0; char **messages = NULL; + sds st = sdsempty(); /* Generate the stack trace */ trace_size = backtrace(trace, 100); @@ -572,9 +573,12 @@ void logStackTrace(ucontext_t *uc) { trace[1] = getMcontextEip(uc); } messages = backtrace_symbols(trace, trace_size); - redisLog(REDIS_WARNING, "--- STACK TRACE"); - for (i=1; i Date: Tue, 27 Mar 2012 16:54:53 +0200 Subject: [PATCH 0062/1752] define zlibc_free() in a way that is not shadowed by jemalloc. --- src/zmalloc.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/zmalloc.c b/src/zmalloc.c index 5e5598cf1ce..79b56158613 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -30,6 +30,15 @@ #include #include + +/* This function provide us access to the original libc free(). This is useful + * for instance to free results obtained by backtrace_symbols(). We need + * to define this function before including zmalloc.h that may shadow the + * free implementation if we use jemalloc or another non standard allocator. */ +void zlibc_free(void *ptr) { + free(ptr); +} + #include #include #include "config.h" @@ -227,10 +236,6 @@ void zmalloc_enable_thread_safeness(void) { zmalloc_thread_safe = 1; } -void zlibc_free(void *ptr) { - free(ptr); -} - /* Get the RSS information in an OS-specific way. * * WARNING: the function zmalloc_get_rss() is not designed to be fast From f3e159bc931e8417df0ec432b8760bacc163530e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 28 Mar 2012 10:55:17 +0200 Subject: [PATCH 0063/1752] Redis test: regexp to check if valgrind reported errors modified. Now we no longer look at the total count because this includes "possibly lost" bytes that are not interesting for Redis (tons of false positives because of how sds.c works). --- tests/support/server.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 928e5c40abe..984270adb1b 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -17,7 +17,7 @@ proc check_valgrind_errors stderr { set buf [read $fd] close $fd - if {![regexp -- {ERROR SUMMARY: 0 errors} $buf] || + if {[regexp -- { at 0x} $buf] || (![regexp -- {definitely lost: 0 bytes} $buf] && ![regexp -- {no leaks are possible} $buf])} { send_data_packet $::test_server_fd err "Valgrind error: $buf\n" From be4f8cccaa49d45dca61b398e5dd6922c5eeada0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 28 Mar 2012 13:45:39 +0200 Subject: [PATCH 0064/1752] Log from signal handlers is now safer. --- src/debug.c | 15 +++------------ src/redis.c | 30 +++++++++++++++++++++++++++++- src/redis.h | 1 + 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/src/debug.c b/src/debug.c index 49c76824d22..e7b3ba407fd 100644 --- a/src/debug.c +++ b/src/debug.c @@ -682,25 +682,16 @@ void watchdogSignalHandler(int sig, siginfo_t *info, void *secret) { REDIS_NOTUSED(info); REDIS_NOTUSED(sig); sds st, log; - time_t now = time(NULL); - char date[128]; - FILE *fp; - fp = (server.logfile == NULL) ? stdout : fopen(server.logfile,"a"); - if (fp == NULL) return; - - strftime(date,sizeof(date),"%d %b %H:%M:%S",localtime(&now)); - log = sdscatprintf(sdsempty(), - "\n--- WATCHDOG TIMER EXPIRED (%s) ---\n",date); + log = sdsnew("\n--- WATCHDOG TIMER EXPIRED ---\n"); #ifdef HAVE_BACKTRACE st = getStackTrace(uc); #else st = sdsnew("Sorry: no support for backtrace().\n"); #endif log = sdscatsds(log,st); - log = sdscat(log,"------\n\n"); - fprintf(fp,"%s",log); - if (server.logfile) fclose(fp); + log = sdscat(log,"------\n"); + redisLogFromHandler(REDIS_WARNING,log); sdsfree(st); sdsfree(log); } diff --git a/src/redis.c b/src/redis.c index 958827f720f..a403bc92d24 100644 --- a/src/redis.c +++ b/src/redis.c @@ -291,6 +291,34 @@ void redisLog(int level, const char *fmt, ...) { redisLogRaw(level,msg); } +/* Log a fixed message without printf-alike capabilities, in a way that is + * safe to call from a signal handler. + * + * We actually use this only for signals that are not fatal from the point + * of view of Redis. Signals that are going to kill the server anyway and + * where we need printf-alike features are served by redisLog(). */ +void redisLogFromHandler(int level, const char *msg) { + int fd; + char buf[64]; + + if ((level&0xff) < server.verbosity || + (server.logfile == NULL && server.daemonize)) return; + fd = server.logfile ? + open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644) : + STDIN_FILENO; + if (fd == -1) return; + ll2string(buf,sizeof(buf),getpid()); + write(fd,"[",1); + write(fd,buf,strlen(buf)); + write(fd," | signal handler] (",20); + ll2string(buf,sizeof(buf),time(NULL)); + write(fd,buf,strlen(buf)); + write(fd,") ",2); + write(fd,msg,strlen(msg)); + write(fd,"\n",1); + close(fd); +} + /* Redis generally does not try to recover from out of memory conditions * when allocating objects or strings, it is not clear if it will be possible * to report this condition to the client since the networking layer itself @@ -2231,7 +2259,7 @@ void redisAsciiArt(void) { static void sigtermHandler(int sig) { REDIS_NOTUSED(sig); - redisLog(REDIS_WARNING,"Received SIGTERM, scheduling shutdown..."); + redisLogFromHandler(REDIS_WARNING,"Received SIGTERM, scheduling shutdown..."); server.shutdown_asap = 1; } diff --git a/src/redis.h b/src/redis.h index 3392b5dcffa..35f36d667e7 100644 --- a/src/redis.h +++ b/src/redis.h @@ -870,6 +870,7 @@ void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, in int prepareForShutdown(); void redisLog(int level, const char *fmt, ...); void redisLogRaw(int level, const char *msg); +void redisLogFromHandler(int level, const char *msg); void usage(); void updateDictResizePolicy(void); int htNeedsResize(dict *dict); From e51f7d2c173bb0a29893e24b56355df806f44bb2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 28 Mar 2012 13:51:23 +0200 Subject: [PATCH 0065/1752] Fixes for redisLogFromHandler(). --- src/redis.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index a403bc92d24..9c49bc228fd 100644 --- a/src/redis.c +++ b/src/redis.c @@ -305,7 +305,7 @@ void redisLogFromHandler(int level, const char *msg) { (server.logfile == NULL && server.daemonize)) return; fd = server.logfile ? open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644) : - STDIN_FILENO; + STDOUT_FILENO; if (fd == -1) return; ll2string(buf,sizeof(buf),getpid()); write(fd,"[",1); @@ -316,7 +316,7 @@ void redisLogFromHandler(int level, const char *msg) { write(fd,") ",2); write(fd,msg,strlen(msg)); write(fd,"\n",1); - close(fd); + if (server.logfile) close(fd); } /* Redis generally does not try to recover from out of memory conditions From 2f2e6ad4879c9efa44ef6d2a726a7acbf288a76e Mon Sep 17 00:00:00 2001 From: Nathan Fritz Date: Wed, 28 Mar 2012 11:10:24 -0700 Subject: [PATCH 0066/1752] added redis.sha1hex(string) as lua scripting function. (The original implementation was modified by @antirez to conform Redis coding standards.) --- src/scripting.c | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index e38d08077d0..0f876961a22 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -15,6 +15,7 @@ char *redisProtocolToLuaType_Error(lua_State *lua, char *reply); char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply); int redis_math_random (lua_State *L); int redis_math_randomseed (lua_State *L); +void sha1hex(char *digest, char *script, size_t len); /* Take a Redis reply in the Redis protocol format and convert it into a * Lua type. Thanks to this function, and the introduction of not connected @@ -306,6 +307,25 @@ int luaRedisPCallCommand(lua_State *lua) { return luaRedisGenericCommand(lua,0); } +/* This adds redis.sha1hex(string) to Lua scripts using the same hashing + * function used for sha1ing lua scripts. */ +int luaRedisSha1hexCommand(lua_State *lua) { + int argc = lua_gettop(lua); + char digest[41]; + size_t len; + char *s; + + if (argc != 1) { + luaPushError(lua, "wrong number of arguments"); + return 1; + } + + s = (char*)lua_tolstring(lua,1,&len); + sha1hex(digest,s,len); + lua_pushstring(lua,digest); + return 1; +} + int luaLogCommand(lua_State *lua) { int j, argc = lua_gettop(lua); int level; @@ -439,6 +459,11 @@ void scriptingInit(void) { lua_pushnumber(lua,REDIS_WARNING); lua_settable(lua,-3); + /* redis.sha1hex */ + lua_pushstring(lua, "sha1hex"); + lua_pushcfunction(lua, luaRedisSha1hexCommand); + lua_settable(lua, -3); + /* Finally set the table as 'redis' global var. */ lua_setglobal(lua,"redis"); @@ -491,10 +516,13 @@ void scriptingReset(void) { scriptingInit(); } -/* Hash the scripit into a SHA1 digest. We use this as Lua function name. - * Digest should point to a 41 bytes buffer: 40 for SHA1 converted into an +/* Perform the SHA1 of the input string. We use this both for hasing script + * bodies in order to obtain the Lua function name, and in the implementation + * of redis.sha1(). + * + * 'digest' should point to a 41 bytes buffer: 40 for SHA1 converted into an * hexadecimal number, plus 1 byte for null term. */ -void hashScript(char *digest, char *script, size_t len) { +void sha1hex(char *digest, char *script, size_t len) { SHA1_CTX ctx; unsigned char hash[20]; char *cset = "0123456789abcdef"; @@ -667,7 +695,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { funcname[1] = '_'; if (!evalsha) { /* Hash the code if this is an EVAL call */ - hashScript(funcname+2,c->argv[1]->ptr,sdslen(c->argv[1]->ptr)); + sha1hex(funcname+2,c->argv[1]->ptr,sdslen(c->argv[1]->ptr)); } else { /* We already have the SHA if it is a EVALSHA */ int j; @@ -841,7 +869,7 @@ void scriptCommand(redisClient *c) { funcname[0] = 'f'; funcname[1] = '_'; - hashScript(funcname+2,c->argv[2]->ptr,sdslen(c->argv[2]->ptr)); + sha1hex(funcname+2,c->argv[2]->ptr,sdslen(c->argv[2]->ptr)); sha = sdsnewlen(funcname+2,40); if (dictFind(server.lua_scripts,sha) == NULL) { if (luaCreateFunction(c,server.lua,funcname,c->argv[2]) From ee704a0ff15f783fd804964d361b160dd09afa80 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 28 Mar 2012 20:47:50 +0200 Subject: [PATCH 0067/1752] Test for redis.sha1hex(). --- tests/unit/scripting.tcl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 86e51c17bf3..b9307cc1930 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -214,6 +214,11 @@ start_server {tags {"scripting"}} { r sadd myset a b c r eval {return redis.call('sort','myset','by','_','get','#','get','_:*')} 0 } {{} {} {} a b c} + + test "redis.sha1hex() implementation" { + list [r eval {return redis.sha1hex('')} 0] \ + [r eval {return redis.sha1hex('Pizza & Mandolino')} 0] + } {da39a3ee5e6b4b0d3255bfef95601890afd80709 74822d82031af7493c20eefa13bd07ec4fada82f} } start_server {tags {"scripting repl"}} { From ed4d4f1145cf448456a5b73cc1a1f9b06318496c Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Mar 2012 09:24:02 +0200 Subject: [PATCH 0068/1752] Fix for slaves chains. Force resync of slaves (simply disconnecting them) when SLAVEOF turns a master into a slave. --- src/networking.c | 29 +++++++++++++++-------------- src/redis.h | 1 + src/replication.c | 1 + 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/networking.c b/src/networking.c index 937f8402a56..406d1cde3bc 100644 --- a/src/networking.c +++ b/src/networking.c @@ -547,6 +547,16 @@ static void freeClientArgv(redisClient *c) { c->cmd = NULL; } +/* Close all the slaves connections. This is useful in chained replication + * when we resync with our own master and want to force all our slaves to + * resync with us as well. */ +void disconnectSlaves(void) { + while (listLength(server.slaves)) { + listNode *ln = listFirst(server.slaves); + freeClient((redisClient*)ln->value); + } +} + void freeClient(redisClient *c) { listNode *ln; @@ -605,21 +615,12 @@ void freeClient(redisClient *c) { server.master = NULL; server.repl_state = REDIS_REPL_CONNECT; server.repl_down_since = server.unixtime; - /* Since we lost the connection with the master, we should also - * close the connection with all our slaves if we have any, so - * when we'll resync with the master the other slaves will sync again - * with us as well. Note that also when the slave is not connected - * to the master it will keep refusing connections by other slaves. + /* We lost connection with our master, force our slaves to resync + * with us as well to load the new data set. * - * We do this only if server.masterhost != NULL. If it is NULL this - * means the user called SLAVEOF NO ONE and we are freeing our - * link with the master, so no need to close link with slaves. */ - if (server.masterhost != NULL) { - while (listLength(server.slaves)) { - ln = listFirst(server.slaves); - freeClient((redisClient*)ln->value); - } - } + * If server.masterhost is NULL te user called SLAVEOF NO ONE so + * slave resync is not needed. */ + if (server.masterhost != NULL) disconnectSlaves(); } /* If this client was scheduled for async freeing we need to remove it diff --git a/src/redis.h b/src/redis.h index 35f36d667e7..e36e13f10b9 100644 --- a/src/redis.h +++ b/src/redis.h @@ -735,6 +735,7 @@ void asyncCloseClientOnOutputBufferLimitReached(redisClient *c); int getClientLimitClassByName(char *name); char *getClientLimitClassName(int class); void flushSlavesOutputBuffers(void); +void disconnectSlaves(void); #ifdef __GNUC__ void addReplyErrorFormat(redisClient *c, const char *fmt, ...) diff --git a/src/replication.c b/src/replication.c index 170bee73e1e..c99be32aff6 100644 --- a/src/replication.c +++ b/src/replication.c @@ -541,6 +541,7 @@ void slaveofCommand(redisClient *c) { server.masterhost = sdsdup(c->argv[1]->ptr); server.masterport = port; if (server.master) freeClient(server.master); + disconnectSlaves(); /* Force our slaves to resync with us as well. */ if (server.repl_state == REDIS_REPL_TRANSFER) replicationAbortSyncTransfer(); server.repl_state = REDIS_REPL_CONNECT; From f1e38b352e6721929b948b6b207488a654690c14 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Mar 2012 09:33:29 +0200 Subject: [PATCH 0069/1752] Fixed typo in comment: "te" -> "the". --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 406d1cde3bc..75f6cf2213c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -618,7 +618,7 @@ void freeClient(redisClient *c) { /* We lost connection with our master, force our slaves to resync * with us as well to load the new data set. * - * If server.masterhost is NULL te user called SLAVEOF NO ONE so + * If server.masterhost is NULL the user called SLAVEOF NO ONE so * slave resync is not needed. */ if (server.masterhost != NULL) disconnectSlaves(); } From 0f51e3c5640c2f5be2766f10660f668bc71c7dd9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Mar 2012 10:26:07 +0200 Subject: [PATCH 0070/1752] Regression test for issue 417 (memory leak when replicating to DB with id >= 10) --- tests/helpers/bg_complex_data.tcl | 10 ++++++ tests/integration/replication-4.tcl | 54 +++++++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 3 files changed, 65 insertions(+) create mode 100644 tests/helpers/bg_complex_data.tcl create mode 100644 tests/integration/replication-4.tcl diff --git a/tests/helpers/bg_complex_data.tcl b/tests/helpers/bg_complex_data.tcl new file mode 100644 index 00000000000..dffd7c66884 --- /dev/null +++ b/tests/helpers/bg_complex_data.tcl @@ -0,0 +1,10 @@ +source tests/support/redis.tcl +source tests/support/util.tcl + +proc bg_complex_data {host port db ops} { + set r [redis $host $port] + $r select $db + createComplexDataset $r $ops +} + +bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3] diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl new file mode 100644 index 00000000000..69fcab37324 --- /dev/null +++ b/tests/integration/replication-4.tcl @@ -0,0 +1,54 @@ +proc start_bg_complex_data {host port db ops} { + exec tclsh8.5 tests/helpers/bg_complex_data.tcl $host $port $db $ops & +} + +proc stop_bg_complex_data {handle} { + catch {exec /bin/kill -9 $handle} +} + +start_server {tags {"repl"}} { + start_server {} { + + set master [srv 0 client] + set master_host [srv 0 host] + set master_port [srv 0 port] + set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] + set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] + set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] + + test {First server should have role slave after SLAVEOF} { + r -1 slaveof [srv 0 host] [srv 0 port] + after 1000 + s -1 role + } {slave} + + test {Test replication with parallel clients writing in differnet DBs} { + lappend slave [srv 0 client] + after 5000 + stop_bg_complex_data $load_handle0 + stop_bg_complex_data $load_handle1 + stop_bg_complex_data $load_handle2 + set retry 10 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} + + if {[r debug digest] ne [r -1 debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + } + } +} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index bef8231a71a..7978d8a4dee 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -29,6 +29,7 @@ set ::all_tests { integration/replication integration/replication-2 integration/replication-3 + integration/replication-4 integration/aof integration/rdb integration/convert-zipmap-hash-on-load From ae15f75089117c3e65a7de85266a97c3011caf78 Mon Sep 17 00:00:00 2001 From: Joseph Jang Date: Fri, 30 Mar 2012 02:06:53 +0900 Subject: [PATCH 0071/1752] Fixed a memory leak with replication occurs when two or more dbs are replicated and at least one of them is >db10 --- src/redis.c | 15 +++++---------- src/redis.h | 8 ++++---- src/replication.c | 19 +++++-------------- 3 files changed, 14 insertions(+), 28 deletions(-) diff --git a/src/redis.c b/src/redis.c index 9c49bc228fd..557b361a5bb 100644 --- a/src/redis.c +++ b/src/redis.c @@ -961,16 +961,11 @@ void createSharedObjects(void) { shared.space = createObject(REDIS_STRING,sdsnew(" ")); shared.colon = createObject(REDIS_STRING,sdsnew(":")); shared.plus = createObject(REDIS_STRING,sdsnew("+")); - shared.select0 = createStringObject("select 0\r\n",10); - shared.select1 = createStringObject("select 1\r\n",10); - shared.select2 = createStringObject("select 2\r\n",10); - shared.select3 = createStringObject("select 3\r\n",10); - shared.select4 = createStringObject("select 4\r\n",10); - shared.select5 = createStringObject("select 5\r\n",10); - shared.select6 = createStringObject("select 6\r\n",10); - shared.select7 = createStringObject("select 7\r\n",10); - shared.select8 = createStringObject("select 8\r\n",10); - shared.select9 = createStringObject("select 9\r\n",10); + + for (j = 0; j < REDIS_SHARED_SELECT_CMDS; j++) { + shared.select[j] = createObject(REDIS_STRING, + sdscatprintf(sdsempty(),"select %d\r\n", j)); + } shared.messagebulk = createStringObject("$7\r\nmessage\r\n",13); shared.pmessagebulk = createStringObject("$8\r\npmessage\r\n",14); shared.subscribebulk = createStringObject("$9\r\nsubscribe\r\n",15); diff --git a/src/redis.h b/src/redis.h index e36e13f10b9..5d16722e37a 100644 --- a/src/redis.h +++ b/src/redis.h @@ -44,6 +44,7 @@ #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_EXPIRELOOKUPS_PER_CRON 10 /* lookup 10 expires per loop */ #define REDIS_MAX_WRITE_PER_EVENT (1024*64) +#define REDIS_SHARED_SELECT_CMDS 10 #define REDIS_SHARED_INTEGERS 10000 #define REDIS_SHARED_BULKHDR_LEN 32 #define REDIS_MAX_LOGMSG_LEN 1024 /* Default maximum length of syslog messages */ @@ -366,10 +367,9 @@ struct sharedObjectsStruct { *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, - *roslaveerr, *oomerr, *plus, *select0, *select1, *select2, *select3, - *select4, *select5, *select6, *select7, *select8, *select9, - *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, - *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, + *roslaveerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, + *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, + *select[REDIS_SHARED_SELECT_CMDS], *integers[REDIS_SHARED_INTEGERS], *mbulkhdr[REDIS_SHARED_BULKHDR_LEN], /* "*\r\n" */ *bulkhdr[REDIS_SHARED_BULKHDR_LEN]; /* "$\r\n" */ diff --git a/src/replication.c b/src/replication.c index c99be32aff6..f474d982537 100644 --- a/src/replication.c +++ b/src/replication.c @@ -25,24 +25,15 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { if (slave->slaveseldb != dictid) { robj *selectcmd; - switch(dictid) { - case 0: selectcmd = shared.select0; break; - case 1: selectcmd = shared.select1; break; - case 2: selectcmd = shared.select2; break; - case 3: selectcmd = shared.select3; break; - case 4: selectcmd = shared.select4; break; - case 5: selectcmd = shared.select5; break; - case 6: selectcmd = shared.select6; break; - case 7: selectcmd = shared.select7; break; - case 8: selectcmd = shared.select8; break; - case 9: selectcmd = shared.select9; break; - default: + if (dictid >= 0 && dictid < REDIS_SHARED_SELECT_CMDS) { + incrRefCount(shared.select[dictid]); + selectcmd = shared.select[dictid]; + } else { selectcmd = createObject(REDIS_STRING, sdscatprintf(sdsempty(),"select %d\r\n",dictid)); - selectcmd->refcount = 0; - break; } addReply(slave,selectcmd); + decrRefCount(selectcmd); slave->slaveseldb = dictid; } addReplyMultiBulkLen(slave,argc); From 88bd32f1b7bba8894b0a86c72d5cbc867f3b8bee Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Mar 2012 10:39:34 +0200 Subject: [PATCH 0072/1752] Purely aesthetic code change. --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index f474d982537..a6356a07f75 100644 --- a/src/replication.c +++ b/src/replication.c @@ -26,8 +26,8 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { robj *selectcmd; if (dictid >= 0 && dictid < REDIS_SHARED_SELECT_CMDS) { - incrRefCount(shared.select[dictid]); selectcmd = shared.select[dictid]; + incrRefCount(selectcmd); } else { selectcmd = createObject(REDIS_STRING, sdscatprintf(sdsempty(),"select %d\r\n",dictid)); From 4ccf671cc6fbce349c7deca69cfa8e566823d4ed Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Mar 2012 11:21:45 +0200 Subject: [PATCH 0073/1752] Better syncio.c with millisecond resolution. --- src/redis.h | 6 +++--- src/syncio.c | 61 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/src/redis.h b/src/redis.h index 5d16722e37a..f3defca47a2 100644 --- a/src/redis.h +++ b/src/redis.h @@ -811,9 +811,9 @@ int equalStringObjects(robj *a, robj *b); unsigned long estimateObjectIdleTime(robj *o); /* Synchronous I/O with timeout */ -int syncWrite(int fd, char *ptr, ssize_t size, int timeout); -int syncRead(int fd, char *ptr, ssize_t size, int timeout); -int syncReadLine(int fd, char *ptr, ssize_t size, int timeout); +ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout); +ssize_t syncRead(int fd, char *ptr, ssize_t size, long long timeout); +ssize_t syncReadLine(int fd, char *ptr, ssize_t size, long long timeout); /* Replication */ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc); diff --git a/src/syncio.c b/src/syncio.c index 9958363be23..b0c2969e2cc 100644 --- a/src/syncio.c +++ b/src/syncio.c @@ -36,50 +36,79 @@ * of the SYNC command where the slave does it in a blocking way, and * the MIGRATE command that must be blocking in order to be atomic from the * point of view of the two instances (one migrating the key and one receiving - * the key). This is why need the following blocking I/O functions. */ + * the key). This is why need the following blocking I/O functions. + * + * All the functions take the timeout in milliseconds. */ + +#define REDIS_SYNCIO_RESOLUTION 10 /* Resolution in milliseconds */ -int syncWrite(int fd, char *ptr, ssize_t size, int timeout) { +/* Write the specified payload to 'fd'. If writing the whole payload will be done + * within 'timeout' milliseconds the operation succeeds and 'size' is returned. + * Otherwise the operation fails, -1 is returned, and an unspecified partial write + * could be performed against the file descriptor. */ +ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout) { ssize_t nwritten, ret = size; - time_t start = time(NULL); + long long start = mstime(); + long long remaining = timeout; - timeout++; - while(size) { - if (aeWait(fd,AE_WRITABLE,1000) & AE_WRITABLE) { + while(1) { + long long wait = (remaining > REDIS_SYNCIO_RESOLUTION) ? + remaining : REDIS_SYNCIO_RESOLUTION; + long long elapsed; + + if (aeWait(fd,AE_WRITABLE,wait) & AE_WRITABLE) { nwritten = write(fd,ptr,size); if (nwritten == -1) return -1; ptr += nwritten; size -= nwritten; + if (size == 0) return ret; } - if ((time(NULL)-start) > timeout) { + elapsed = mstime() - start; + if (elapsed >= timeout) { errno = ETIMEDOUT; return -1; } + remaining = timeout - elapsed; } - return ret; } -int syncRead(int fd, char *ptr, ssize_t size, int timeout) { +/* Read the specified amount of bytes from 'fd'. If all the bytes are read within + * 'timeout' milliseconds the operation succeed and 'size' is returned. + * Otherwise the operation fails, -1 is returned, and an unspecified amount of + * data could be read from the file descriptor. */ +ssize_t syncRead(int fd, char *ptr, ssize_t size, long long timeout) { ssize_t nread, totread = 0; - time_t start = time(NULL); + long long start = mstime(); + long long remaining = timeout; - timeout++; - while(size) { - if (aeWait(fd,AE_READABLE,1000) & AE_READABLE) { + while(1) { + long long wait = (remaining > REDIS_SYNCIO_RESOLUTION) ? + remaining : REDIS_SYNCIO_RESOLUTION; + long long elapsed; + + if (aeWait(fd,AE_READABLE,wait) & AE_READABLE) { nread = read(fd,ptr,size); if (nread <= 0) return -1; ptr += nread; size -= nread; totread += nread; + if (size == 0) return totread; } - if ((time(NULL)-start) > timeout) { + elapsed = mstime() - start; + if (elapsed >= timeout) { errno = ETIMEDOUT; return -1; } + remaining = timeout - elapsed; } - return totread; } -int syncReadLine(int fd, char *ptr, ssize_t size, int timeout) { +/* Read a line making sure that every char will not require more than 'timeout' + * milliseconds to be read. + * + * On success the number of bytes read is returned, otherwise -1. + * On success the string is always correctly terminated with a 0 byte. */ +ssize_t syncReadLine(int fd, char *ptr, ssize_t size, long long timeout) { ssize_t nread = 0; size--; From fa34ba39084255aeb0646a75c114c1f2e626e5c3 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Mar 2012 11:23:30 +0200 Subject: [PATCH 0074/1752] syncio.c calls in replication.c fixed for the new millisecond timeout API. --- src/replication.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index a6356a07f75..ef148f1c28e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -282,7 +282,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { /* If repl_transfer_left == -1 we still have to read the bulk length * from the master reply. */ if (server.repl_transfer_left == -1) { - if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout) == -1) { + if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout*1000) == -1) { redisLog(REDIS_WARNING, "I/O error reading bulk count from MASTER: %s", strerror(errno)); @@ -405,13 +405,13 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { size_t authlen; authlen = snprintf(authcmd,sizeof(authcmd),"AUTH %s\r\n",server.masterauth); - if (syncWrite(fd,authcmd,authlen,server.repl_syncio_timeout) == -1) { + if (syncWrite(fd,authcmd,authlen,server.repl_syncio_timeout*1000) == -1) { redisLog(REDIS_WARNING,"Unable to AUTH to MASTER: %s", strerror(errno)); goto error; } /* Read the AUTH result. */ - if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout) == -1) { + if (syncReadLine(fd,buf,1024,server.repl_syncio_timeout*1000) == -1) { redisLog(REDIS_WARNING,"I/O error reading auth result from MASTER: %s", strerror(errno)); goto error; @@ -423,7 +423,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { } /* Issue the SYNC command */ - if (syncWrite(fd,"SYNC \r\n",7,server.repl_syncio_timeout) == -1) { + if (syncWrite(fd,"SYNC \r\n",7,server.repl_syncio_timeout*1000) == -1) { redisLog(REDIS_WARNING,"I/O error writing to MASTER: %s", strerror(errno)); goto error; From f08aa2bf48883fa3c00266229f17efe4af201078 Mon Sep 17 00:00:00 2001 From: ThePicard Date: Sat, 31 Mar 2012 23:39:58 -0700 Subject: [PATCH 0075/1752] Fixed a typo in install_server.sh --- utils/install_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install_server.sh b/utils/install_server.sh index 06570a87f7c..93b5b411b80 100755 --- a/utils/install_server.sh +++ b/utils/install_server.sh @@ -166,7 +166,7 @@ if [[ ! `which chkconfig` ]] ; then else # we're chkconfig, so lets add to chkconfig and put in runlevel 345 chkconfig --add redis_$REDIS_PORT && echo "Successfully added to chkconfig!" - chkconfig--level 345 redis_$REDIS_PORT on && echo "Successfully added to runlevels 345!" + chkconfig --level 345 redis_$REDIS_PORT on && echo "Successfully added to runlevels 345!" fi /etc/init.d/redis_$REDIS_PORT start || die "Failed starting service..." From 11dae1711fa799e23eacc4f142c09b425c75a8b8 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 31 Mar 2012 17:08:40 +0200 Subject: [PATCH 0076/1752] Write RDB magic using a REDIS_RDB_VERSION define that is defined inside rdb.h --- src/rdb.c | 4 +++- src/rdb.h | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 1e23fa70cf3..481efe9de8b 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -602,6 +602,7 @@ int rdbSave(char *filename) { dictIterator *di = NULL; dictEntry *de; char tmpfile[256]; + char magic[10]; int j; long long now = mstime(); FILE *fp; @@ -616,7 +617,8 @@ int rdbSave(char *filename) { } rioInitWithFile(&rdb,fp); - if (rdbWriteRaw(&rdb,"REDIS0004",9) == -1) goto werr; + snprintf(magic,sizeof(magic),"REDIS%04d",REDIS_RDB_VERSION); + if (rdbWriteRaw(&rdb,magic,9) == -1) goto werr; for (j = 0; j < server.dbnum; j++) { redisDb *db = server.db+j; diff --git a/src/rdb.h b/src/rdb.h index 45beaa93a73..60157ad8740 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -7,6 +7,10 @@ /* TBD: include only necessary headers. */ #include "redis.h" +/* The current RDB version. When the format changes in a way that is no longer + * backward compatible this number gets incremented. */ +#define REDIS_RDB_VERSION 4 + /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of * the first byte to interpreter the length: From 179ee2db78ca77efe755303b2e20c6c4c6d4fedf Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Apr 2012 12:31:44 +0200 Subject: [PATCH 0077/1752] CRC64 implementation added to Redis code base. --- src/Makefile | 2 +- src/crc64.c | 192 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/redis.h | 1 + 3 files changed, 194 insertions(+), 1 deletion(-) create mode 100644 src/crc64.c diff --git a/src/Makefile b/src/Makefile index a09e1b42bdd..34ef7a7cea2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -73,7 +73,7 @@ QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); endif -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o +OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o diff --git a/src/crc64.c b/src/crc64.c new file mode 100644 index 00000000000..f2ea8d4ae4c --- /dev/null +++ b/src/crc64.c @@ -0,0 +1,192 @@ +/* Redis uses the CRC64 variant with "Jones" coefficients and init value of 0. + * + * Specification of this CRC64 variant follows: + * Name: crc-64-jones + * Width: 64 bites + * Poly: 0xad93d23594c935a9 + * Reflected In: True + * Xor_In: 0xffffffffffffffff + * Reflected_Out: True + * Xor_Out: 0x0 + * Check("123456789"): 0xe9c6d914c4b8d9ca + * + * Copyright (c) 2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. */ + +#include + +static const uint64_t crc64_tab[256] = { + UINT64_C(0x0000000000000000), UINT64_C(0x7ad870c830358979), + UINT64_C(0xf5b0e190606b12f2), UINT64_C(0x8f689158505e9b8b), + UINT64_C(0xc038e5739841b68f), UINT64_C(0xbae095bba8743ff6), + UINT64_C(0x358804e3f82aa47d), UINT64_C(0x4f50742bc81f2d04), + UINT64_C(0xab28ecb46814fe75), UINT64_C(0xd1f09c7c5821770c), + UINT64_C(0x5e980d24087fec87), UINT64_C(0x24407dec384a65fe), + UINT64_C(0x6b1009c7f05548fa), UINT64_C(0x11c8790fc060c183), + UINT64_C(0x9ea0e857903e5a08), UINT64_C(0xe478989fa00bd371), + UINT64_C(0x7d08ff3b88be6f81), UINT64_C(0x07d08ff3b88be6f8), + UINT64_C(0x88b81eabe8d57d73), UINT64_C(0xf2606e63d8e0f40a), + UINT64_C(0xbd301a4810ffd90e), UINT64_C(0xc7e86a8020ca5077), + UINT64_C(0x4880fbd87094cbfc), UINT64_C(0x32588b1040a14285), + UINT64_C(0xd620138fe0aa91f4), UINT64_C(0xacf86347d09f188d), + UINT64_C(0x2390f21f80c18306), UINT64_C(0x594882d7b0f40a7f), + UINT64_C(0x1618f6fc78eb277b), UINT64_C(0x6cc0863448deae02), + UINT64_C(0xe3a8176c18803589), UINT64_C(0x997067a428b5bcf0), + UINT64_C(0xfa11fe77117cdf02), UINT64_C(0x80c98ebf2149567b), + UINT64_C(0x0fa11fe77117cdf0), UINT64_C(0x75796f2f41224489), + UINT64_C(0x3a291b04893d698d), UINT64_C(0x40f16bccb908e0f4), + UINT64_C(0xcf99fa94e9567b7f), UINT64_C(0xb5418a5cd963f206), + UINT64_C(0x513912c379682177), UINT64_C(0x2be1620b495da80e), + UINT64_C(0xa489f35319033385), UINT64_C(0xde51839b2936bafc), + UINT64_C(0x9101f7b0e12997f8), UINT64_C(0xebd98778d11c1e81), + UINT64_C(0x64b116208142850a), UINT64_C(0x1e6966e8b1770c73), + UINT64_C(0x8719014c99c2b083), UINT64_C(0xfdc17184a9f739fa), + UINT64_C(0x72a9e0dcf9a9a271), UINT64_C(0x08719014c99c2b08), + UINT64_C(0x4721e43f0183060c), UINT64_C(0x3df994f731b68f75), + UINT64_C(0xb29105af61e814fe), UINT64_C(0xc849756751dd9d87), + UINT64_C(0x2c31edf8f1d64ef6), UINT64_C(0x56e99d30c1e3c78f), + UINT64_C(0xd9810c6891bd5c04), UINT64_C(0xa3597ca0a188d57d), + UINT64_C(0xec09088b6997f879), UINT64_C(0x96d1784359a27100), + UINT64_C(0x19b9e91b09fcea8b), UINT64_C(0x636199d339c963f2), + UINT64_C(0xdf7adabd7a6e2d6f), UINT64_C(0xa5a2aa754a5ba416), + UINT64_C(0x2aca3b2d1a053f9d), UINT64_C(0x50124be52a30b6e4), + UINT64_C(0x1f423fcee22f9be0), UINT64_C(0x659a4f06d21a1299), + UINT64_C(0xeaf2de5e82448912), UINT64_C(0x902aae96b271006b), + UINT64_C(0x74523609127ad31a), UINT64_C(0x0e8a46c1224f5a63), + UINT64_C(0x81e2d7997211c1e8), UINT64_C(0xfb3aa75142244891), + UINT64_C(0xb46ad37a8a3b6595), UINT64_C(0xceb2a3b2ba0eecec), + UINT64_C(0x41da32eaea507767), UINT64_C(0x3b024222da65fe1e), + UINT64_C(0xa2722586f2d042ee), UINT64_C(0xd8aa554ec2e5cb97), + UINT64_C(0x57c2c41692bb501c), UINT64_C(0x2d1ab4dea28ed965), + UINT64_C(0x624ac0f56a91f461), UINT64_C(0x1892b03d5aa47d18), + UINT64_C(0x97fa21650afae693), UINT64_C(0xed2251ad3acf6fea), + UINT64_C(0x095ac9329ac4bc9b), UINT64_C(0x7382b9faaaf135e2), + UINT64_C(0xfcea28a2faafae69), UINT64_C(0x8632586aca9a2710), + UINT64_C(0xc9622c4102850a14), UINT64_C(0xb3ba5c8932b0836d), + UINT64_C(0x3cd2cdd162ee18e6), UINT64_C(0x460abd1952db919f), + UINT64_C(0x256b24ca6b12f26d), UINT64_C(0x5fb354025b277b14), + UINT64_C(0xd0dbc55a0b79e09f), UINT64_C(0xaa03b5923b4c69e6), + UINT64_C(0xe553c1b9f35344e2), UINT64_C(0x9f8bb171c366cd9b), + UINT64_C(0x10e3202993385610), UINT64_C(0x6a3b50e1a30ddf69), + UINT64_C(0x8e43c87e03060c18), UINT64_C(0xf49bb8b633338561), + UINT64_C(0x7bf329ee636d1eea), UINT64_C(0x012b592653589793), + UINT64_C(0x4e7b2d0d9b47ba97), UINT64_C(0x34a35dc5ab7233ee), + UINT64_C(0xbbcbcc9dfb2ca865), UINT64_C(0xc113bc55cb19211c), + UINT64_C(0x5863dbf1e3ac9dec), UINT64_C(0x22bbab39d3991495), + UINT64_C(0xadd33a6183c78f1e), UINT64_C(0xd70b4aa9b3f20667), + UINT64_C(0x985b3e827bed2b63), UINT64_C(0xe2834e4a4bd8a21a), + UINT64_C(0x6debdf121b863991), UINT64_C(0x1733afda2bb3b0e8), + UINT64_C(0xf34b37458bb86399), UINT64_C(0x8993478dbb8deae0), + UINT64_C(0x06fbd6d5ebd3716b), UINT64_C(0x7c23a61ddbe6f812), + UINT64_C(0x3373d23613f9d516), UINT64_C(0x49aba2fe23cc5c6f), + UINT64_C(0xc6c333a67392c7e4), UINT64_C(0xbc1b436e43a74e9d), + UINT64_C(0x95ac9329ac4bc9b5), UINT64_C(0xef74e3e19c7e40cc), + UINT64_C(0x601c72b9cc20db47), UINT64_C(0x1ac40271fc15523e), + UINT64_C(0x5594765a340a7f3a), UINT64_C(0x2f4c0692043ff643), + UINT64_C(0xa02497ca54616dc8), UINT64_C(0xdafce7026454e4b1), + UINT64_C(0x3e847f9dc45f37c0), UINT64_C(0x445c0f55f46abeb9), + UINT64_C(0xcb349e0da4342532), UINT64_C(0xb1eceec59401ac4b), + UINT64_C(0xfebc9aee5c1e814f), UINT64_C(0x8464ea266c2b0836), + UINT64_C(0x0b0c7b7e3c7593bd), UINT64_C(0x71d40bb60c401ac4), + UINT64_C(0xe8a46c1224f5a634), UINT64_C(0x927c1cda14c02f4d), + UINT64_C(0x1d148d82449eb4c6), UINT64_C(0x67ccfd4a74ab3dbf), + UINT64_C(0x289c8961bcb410bb), UINT64_C(0x5244f9a98c8199c2), + UINT64_C(0xdd2c68f1dcdf0249), UINT64_C(0xa7f41839ecea8b30), + UINT64_C(0x438c80a64ce15841), UINT64_C(0x3954f06e7cd4d138), + UINT64_C(0xb63c61362c8a4ab3), UINT64_C(0xcce411fe1cbfc3ca), + UINT64_C(0x83b465d5d4a0eece), UINT64_C(0xf96c151de49567b7), + UINT64_C(0x76048445b4cbfc3c), UINT64_C(0x0cdcf48d84fe7545), + UINT64_C(0x6fbd6d5ebd3716b7), UINT64_C(0x15651d968d029fce), + UINT64_C(0x9a0d8ccedd5c0445), UINT64_C(0xe0d5fc06ed698d3c), + UINT64_C(0xaf85882d2576a038), UINT64_C(0xd55df8e515432941), + UINT64_C(0x5a3569bd451db2ca), UINT64_C(0x20ed197575283bb3), + UINT64_C(0xc49581ead523e8c2), UINT64_C(0xbe4df122e51661bb), + UINT64_C(0x3125607ab548fa30), UINT64_C(0x4bfd10b2857d7349), + UINT64_C(0x04ad64994d625e4d), UINT64_C(0x7e7514517d57d734), + UINT64_C(0xf11d85092d094cbf), UINT64_C(0x8bc5f5c11d3cc5c6), + UINT64_C(0x12b5926535897936), UINT64_C(0x686de2ad05bcf04f), + UINT64_C(0xe70573f555e26bc4), UINT64_C(0x9ddd033d65d7e2bd), + UINT64_C(0xd28d7716adc8cfb9), UINT64_C(0xa85507de9dfd46c0), + UINT64_C(0x273d9686cda3dd4b), UINT64_C(0x5de5e64efd965432), + UINT64_C(0xb99d7ed15d9d8743), UINT64_C(0xc3450e196da80e3a), + UINT64_C(0x4c2d9f413df695b1), UINT64_C(0x36f5ef890dc31cc8), + UINT64_C(0x79a59ba2c5dc31cc), UINT64_C(0x037deb6af5e9b8b5), + UINT64_C(0x8c157a32a5b7233e), UINT64_C(0xf6cd0afa9582aa47), + UINT64_C(0x4ad64994d625e4da), UINT64_C(0x300e395ce6106da3), + UINT64_C(0xbf66a804b64ef628), UINT64_C(0xc5bed8cc867b7f51), + UINT64_C(0x8aeeace74e645255), UINT64_C(0xf036dc2f7e51db2c), + UINT64_C(0x7f5e4d772e0f40a7), UINT64_C(0x05863dbf1e3ac9de), + UINT64_C(0xe1fea520be311aaf), UINT64_C(0x9b26d5e88e0493d6), + UINT64_C(0x144e44b0de5a085d), UINT64_C(0x6e963478ee6f8124), + UINT64_C(0x21c640532670ac20), UINT64_C(0x5b1e309b16452559), + UINT64_C(0xd476a1c3461bbed2), UINT64_C(0xaeaed10b762e37ab), + UINT64_C(0x37deb6af5e9b8b5b), UINT64_C(0x4d06c6676eae0222), + UINT64_C(0xc26e573f3ef099a9), UINT64_C(0xb8b627f70ec510d0), + UINT64_C(0xf7e653dcc6da3dd4), UINT64_C(0x8d3e2314f6efb4ad), + UINT64_C(0x0256b24ca6b12f26), UINT64_C(0x788ec2849684a65f), + UINT64_C(0x9cf65a1b368f752e), UINT64_C(0xe62e2ad306bafc57), + UINT64_C(0x6946bb8b56e467dc), UINT64_C(0x139ecb4366d1eea5), + UINT64_C(0x5ccebf68aecec3a1), UINT64_C(0x2616cfa09efb4ad8), + UINT64_C(0xa97e5ef8cea5d153), UINT64_C(0xd3a62e30fe90582a), + UINT64_C(0xb0c7b7e3c7593bd8), UINT64_C(0xca1fc72bf76cb2a1), + UINT64_C(0x45775673a732292a), UINT64_C(0x3faf26bb9707a053), + UINT64_C(0x70ff52905f188d57), UINT64_C(0x0a2722586f2d042e), + UINT64_C(0x854fb3003f739fa5), UINT64_C(0xff97c3c80f4616dc), + UINT64_C(0x1bef5b57af4dc5ad), UINT64_C(0x61372b9f9f784cd4), + UINT64_C(0xee5fbac7cf26d75f), UINT64_C(0x9487ca0fff135e26), + UINT64_C(0xdbd7be24370c7322), UINT64_C(0xa10fceec0739fa5b), + UINT64_C(0x2e675fb4576761d0), UINT64_C(0x54bf2f7c6752e8a9), + UINT64_C(0xcdcf48d84fe75459), UINT64_C(0xb71738107fd2dd20), + UINT64_C(0x387fa9482f8c46ab), UINT64_C(0x42a7d9801fb9cfd2), + UINT64_C(0x0df7adabd7a6e2d6), UINT64_C(0x772fdd63e7936baf), + UINT64_C(0xf8474c3bb7cdf024), UINT64_C(0x829f3cf387f8795d), + UINT64_C(0x66e7a46c27f3aa2c), UINT64_C(0x1c3fd4a417c62355), + UINT64_C(0x935745fc4798b8de), UINT64_C(0xe98f353477ad31a7), + UINT64_C(0xa6df411fbfb21ca3), UINT64_C(0xdc0731d78f8795da), + UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), +}; + +uint64_t crc64(const unsigned char *s, uint64_t l) { + uint64_t crc = 0; + uint64_t j; + + for (j = 0; j < l; j++) { + uint8_t byte = s[j]; + crc = crc64_tab[(uint8_t)crc ^ byte] ^ (crc >> 8); + } + return crc; +} + +/* Test main */ +#ifdef TEST_MAIN +#include +int main(void) { + printf("e9c6d914c4b8d9ca == %016llx\n", + (unsigned long long) crc64((unsigned char*)"123456789",9)); + return 0; +} +#endif diff --git a/src/redis.h b/src/redis.h index f3defca47a2..f2229caf930 100644 --- a/src/redis.h +++ b/src/redis.h @@ -694,6 +694,7 @@ extern dictType hashDictType; long long ustime(void); long long mstime(void); void getRandomHexChars(char *p, unsigned int len); +uint64_t crc64(const unsigned char *s, uint64_t l); /* networking.c -- Networking and Client related operations */ redisClient *createClient(int fd); From bff31e129deb7e62b884ccfd941aaa14e8c3e26e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Apr 2012 14:21:59 +0200 Subject: [PATCH 0078/1752] New verions of DUMP, RESTORE, MIGRATE back ported from unstable to 2.6 --- src/migrate.c | 124 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 91 insertions(+), 33 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index f7a5b730998..ae312c4156b 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -1,9 +1,87 @@ #include "redis.h" +#include "endianconv.h" /* ----------------------------------------------------------------------------- - * RESTORE and MIGRATE commands + * DUMP, RESTORE and MIGRATE commands * -------------------------------------------------------------------------- */ +/* Generates a DUMP-format representation of the object 'o', adding it to the + * io stream pointed by 'rio'. This function can't fail. */ +void createDumpPayload(rio *payload, robj *o) { + unsigned char buf[2]; + uint64_t crc; + + /* Serialize the object in a RDB-like format. It consist of an object type + * byte followed by the serialized object. This is understood by RESTORE. */ + rioInitWithBuffer(payload,sdsempty()); + redisAssert(rdbSaveObjectType(payload,o)); + redisAssert(rdbSaveObject(payload,o)); + + /* Write the footer, this is how it looks like: + * ----------------+---------------------+---------------+ + * ... RDB payload | 2 bytes RDB version | 8 bytes CRC64 | + * ----------------+---------------------+---------------+ + * RDB version and CRC are both in little endian. + */ + + /* RDB version */ + buf[0] = REDIS_RDB_VERSION & 0xff; + buf[1] = (REDIS_RDB_VERSION >> 8) & 0xff; + payload->io.buffer.ptr = sdscatlen(payload->io.buffer.ptr,buf,2); + + /* CRC64 */ + crc = crc64((unsigned char*)payload->io.buffer.ptr, + sdslen(payload->io.buffer.ptr)); + memrev64ifbe(&crc); + payload->io.buffer.ptr = sdscatlen(payload->io.buffer.ptr,&crc,8); +} + +/* Verify that the RDB version of the dump payload matches the one of this Redis + * instance and that the checksum is ok. + * If the DUMP payload looks valid REDIS_OK is returned, otherwise REDIS_ERR + * is returned. */ +int verifyDumpPayload(unsigned char *p, size_t len) { + unsigned char *footer; + uint16_t rdbver; + uint64_t crc; + + /* At least 2 bytes of RDB version and 8 of CRC64 should be present. */ + if (len < 10) return REDIS_ERR; + footer = p+(len-10); + + /* Verify RDB version */ + rdbver = (footer[1] << 8) | footer[0]; + if (rdbver != REDIS_RDB_VERSION) return REDIS_ERR; + + /* Verify CRC64 */ + crc = crc64(p,len-8); + memrev64ifbe(&crc); + return (memcmp(&crc,footer+2,8) == 0) ? REDIS_OK : REDIS_ERR; +} + +/* DUMP keyname + * DUMP is actually not used by Redis Cluster but it is the obvious + * complement of RESTORE and can be useful for different applications. */ +void dumpCommand(redisClient *c) { + robj *o, *dumpobj; + rio payload; + + /* Check if the key is here. */ + if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { + addReply(c,shared.nullbulk); + return; + } + + /* Create the DUMP encoded representation. */ + createDumpPayload(&payload,o); + + /* Transfer to the client */ + dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr); + addReplyBulk(c,dumpobj); + decrRefCount(dumpobj); + return; +} + /* RESTORE key ttl serialized-value */ void restoreCommand(redisClient *c) { long ttl; @@ -25,6 +103,12 @@ void restoreCommand(redisClient *c) { return; } + /* Verify RDB version and data checksum. */ + if (verifyDumpPayload(c->argv[3]->ptr,sdslen(c->argv[3]->ptr)) == REDIS_ERR) { + addReplyError(c,"DUMP payload version or checksum are wrong"); + return; + } + rioInitWithBuffer(&payload,c->argv[3]->ptr); if (((type = rdbLoadObjectType(&payload)) == -1) || ((obj = rdbLoadObject(type,&payload)) == NULL)) @@ -35,7 +119,7 @@ void restoreCommand(redisClient *c) { /* Create the key and set the TTL if any */ dbAdd(c->db,c->argv[1],obj); - if (ttl) setExpire(c->db,c->argv[1],time(NULL)+ttl); + if (ttl) setExpire(c->db,c->argv[1],mstime()+ttl); signalModifiedKey(c->db,c->argv[1]); addReply(c,shared.ok); server.dirty++; @@ -78,6 +162,7 @@ void migrateCommand(redisClient *c) { return; } + /* Create RESTORE payload and generate the protocol to call the command. */ rioInitWithBuffer(&cmd,sdsempty()); redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',2)); redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); @@ -91,11 +176,10 @@ void migrateCommand(redisClient *c) { redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl)); /* Finally the last argument that is the serailized object payload - * in the form: . */ - rioInitWithBuffer(&payload,sdsempty()); - redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); - redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o) != -1); - redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,payload.io.buffer.ptr,sdslen(payload.io.buffer.ptr))); + * in the DUMP format. */ + createDumpPayload(&payload,o); + redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,payload.io.buffer.ptr, + sdslen(payload.io.buffer.ptr))); sdsfree(payload.io.buffer.ptr); /* Tranfer the query to the other node in 64K chunks. */ @@ -162,29 +246,3 @@ void migrateCommand(redisClient *c) { close(fd); return; } - -/* DUMP keyname - * DUMP is actually not used by Redis Cluster but it is the obvious - * complement of RESTORE and can be useful for different applications. */ -void dumpCommand(redisClient *c) { - robj *o, *dumpobj; - rio payload; - - /* Check if the key is here. */ - if ((o = lookupKeyRead(c->db,c->argv[1])) == NULL) { - addReply(c,shared.nullbulk); - return; - } - - /* Serialize the object in a RDB-like format. It consist of an object type - * byte followed by the serialized object. This is understood by RESTORE. */ - rioInitWithBuffer(&payload,sdsempty()); - redisAssertWithInfo(c,NULL,rdbSaveObjectType(&payload,o)); - redisAssertWithInfo(c,NULL,rdbSaveObject(&payload,o)); - - /* Transfer to the client */ - dumpobj = createObject(REDIS_STRING,payload.io.buffer.ptr); - addReplyBulk(c,dumpobj); - decrRefCount(dumpobj); - return; -} From 8cf8974a03b894524b65b0bb0aa712cdd1e99512 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Apr 2012 11:44:25 +0200 Subject: [PATCH 0079/1752] DUMP, RESTORE, MIGRATE tests. --- tests/test_helper.tcl | 1 + tests/unit/dump.tcl | 66 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 tests/unit/dump.tcl diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 7978d8a4dee..34d96606d13 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -39,6 +39,7 @@ set ::all_tests { unit/maxmemory unit/introspection unit/obuf-limits + unit/dump } # Index to the next test to run in the ::all_tests list. set ::next_test 0 diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl new file mode 100644 index 00000000000..120638ec631 --- /dev/null +++ b/tests/unit/dump.tcl @@ -0,0 +1,66 @@ +start_server {tags {"dump"}} { + test {DUMP / RESTORE are able to serialize / unserialize a simple key} { + r set foo bar + set encoded [r dump foo] + r del foo + list [r exists foo] [r restore foo 0 $encoded] [r ttl foo] [r get foo] + } {0 OK -1 bar} + + test {RESTORE can set an arbitrary expire to the materialized key} { + r set foo bar + set encoded [r dump foo] + r del foo + r restore foo 5000 $encoded + set ttl [r pttl foo] + assert {$ttl >= 3000 && $ttl <= 5000} + r get foo + } {bar} + + test {RESTORE returns an error of the key already exists} { + r set foo bar + set e {} + catch {r restore foo 0 "..."} e + set e + } {*is busy*} + + test {DUMP of non existing key returns nil} { + r dump nonexisting_key + } {} + + test {MIGRATE is able to migrate a key between two instances} { + set first [srv 0 client] + r set key "Some Value" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + set ret [r -1 migrate $second_host $second_port key 9 5000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second get key] eq {Some Value}} + } + } + + test {MIGRATE timeout actually works} { + set first [srv 0 client] + r set key "Some Value" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + + set rd [redis_deferring_client] + $rd debug sleep 5.0 ; # Make second server unable to reply. + set e {} + catch {r -1 migrate $second_host $second_port key 9 1000} e + assert_match {ERR*} $e + } + } +} From 285df7b0e0c2e3bc175d753326fd2a7391f973ae Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Apr 2012 14:27:04 +0200 Subject: [PATCH 0080/1752] Version bumped 2.4.5 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 63238004f3e..99f6ffb2e22 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.3" +#define REDIS_VERSION "2.5.4" From 37cc07dd418718a06f7309891b1e5c7c272a4b48 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Apr 2012 16:38:24 +0200 Subject: [PATCH 0081/1752] MIGRATE now let the client distinguish I/O errors and timeouts from other erros. --- src/migrate.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index ae312c4156b..90875312164 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -158,7 +158,7 @@ void migrateCommand(redisClient *c) { return; } if ((aeWait(fd,AE_WRITABLE,timeout*1000) & AE_WRITABLE) == 0) { - addReplyError(c,"Timeout connecting to the client"); + addReplySds(c,sdsnew("-IOERR error or timeout connecting to the client\r\n")); return; } @@ -229,19 +229,13 @@ void migrateCommand(redisClient *c) { return; socket_wr_err: - redisLog(REDIS_NOTICE,"Can't write to target node for MIGRATE: %s", - strerror(errno)); - addReplyErrorFormat(c,"MIGRATE failed, writing to target node: %s.", - strerror(errno)); + addReplySds(c,sdsnew("-IOERR error or timeout writing to target instance\r\n")); sdsfree(cmd.io.buffer.ptr); close(fd); return; socket_rd_err: - redisLog(REDIS_NOTICE,"Can't read from target node for MIGRATE: %s", - strerror(errno)); - addReplyErrorFormat(c,"MIGRATE failed, reading from target node: %s.", - strerror(errno)); + addReplySds(c,sdsnew("-IOERR error or timeout reading from target node\r\n")); sdsfree(cmd.io.buffer.ptr); close(fd); return; From b739506936edaa633d7e3c9bfc77c9d67240144b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Apr 2012 16:38:59 +0200 Subject: [PATCH 0082/1752] MIGRATE test modified because the implementation changed. --- tests/unit/dump.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index 120638ec631..6b432377ee9 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -60,7 +60,7 @@ start_server {tags {"dump"}} { $rd debug sleep 5.0 ; # Make second server unable to reply. set e {} catch {r -1 migrate $second_host $second_port key 9 1000} e - assert_match {ERR*} $e + assert_match {IOERR*} $e } } } From 31e2156c4b9a58b88a90c64eecc4bc96cd5de2e4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Apr 2012 10:31:51 +0200 Subject: [PATCH 0083/1752] redis-trib.rb removed from 2.6 branch. --- src/redis-trib.rb | 492 ---------------------------------------------- 1 file changed, 492 deletions(-) delete mode 100755 src/redis-trib.rb diff --git a/src/redis-trib.rb b/src/redis-trib.rb deleted file mode 100755 index 473e4922929..00000000000 --- a/src/redis-trib.rb +++ /dev/null @@ -1,492 +0,0 @@ -#!/usr/bin/env ruby - -# TODO (temporary here, we'll move this into the Github issues once -# redis-trib initial implementation is complted). -# -# - Make sure that if the rehashing fails in the middle redis-trib will try -# to recover. -# - When redis-trib performs a cluster check, if it detects a slot move in -# progress it should prompt the user to continue the move from where it -# stopped. -# - Gracefully handle Ctrl+C in move_slot to prompt the user if really stop -# while rehashing, and performing the best cleanup possible if the user -# forces the quit. -# - When doing "fix" set a global Fix to true, and prompt the user to -# fix the problem if automatically fixable every time there is something -# to fix. For instance: -# 1) If there is a node that pretend to receive a slot, or to migrate a -# slot, but has no entries in that slot, fix it. -# 2) If there is a node having keys in slots that are not owned by it -# fix this condiiton moving the entries in the same node. -# 3) Perform more possibly slow tests about the state of the cluster. -# 4) When aborted slot migration is detected, fix it. - -require 'rubygems' -require 'redis' - -ClusterHashSlots = 4096 - -def xputs(s) - printf s - STDOUT.flush -end - -class ClusterNode - def initialize(addr) - s = addr.split(":") - if s.length != 2 - puts "Invalid node name #{addr}" - exit 1 - end - @r = nil - @info = {} - @info[:host] = s[0] - @info[:port] = s[1] - @info[:slots] = {} - @dirty = false # True if we need to flush slots info into node. - @friends = [] - end - - def friends - @friends - end - - def slots - @info[:slots] - end - - def to_s - "#{@info[:host]}:#{@info[:port]}" - end - - def connect(o={}) - return if @r - xputs "Connecting to node #{self}: " - begin - @r = Redis.new(:host => @info[:host], :port => @info[:port]) - @r.ping - rescue - puts "ERROR" - puts "Sorry, can't connect to node #{self}" - exit 1 if o[:abort] - @r = nil - end - puts "OK" - end - - def assert_cluster - info = @r.info - if !info["cluster_enabled"] || info["cluster_enabled"].to_i == 0 - puts "Error: Node #{self} is not configured as a cluster node." - exit 1 - end - end - - def assert_empty - if !(@r.cluster("info").split("\r\n").index("cluster_known_nodes:1")) || - (@r.info['db0']) - puts "Error: Node #{self} is not empty. Either the node already knows other nodes (check with nodes-info) or contains some key in database 0." - exit 1 - end - end - - def load_info(o={}) - self.connect - nodes = @r.cluster("nodes").split("\n") - nodes.each{|n| - # name addr flags role ping_sent ping_recv link_status slots - split = n.split - name,addr,flags,role,ping_sent,ping_recv,link_status = split[0..6] - slots = split[7..-1] - info = { - :name => name, - :addr => addr, - :flags => flags.split(","), - :role => role, - :ping_sent => ping_sent.to_i, - :ping_recv => ping_recv.to_i, - :link_status => link_status - } - if info[:flags].index("myself") - @info = @info.merge(info) - @info[:slots] = {} - slots.each{|s| - if s[0..0] == '[' - # Fixme: for now skipping migration entries - elsif s.index("-") - start,stop = s.split("-") - self.add_slots((start.to_i)..(stop.to_i)) - else - self.add_slots((s.to_i)..(s.to_i)) - end - } if slots - @dirty = false - @r.cluster("info").split("\n").each{|e| - k,v=e.split(":") - k = k.to_sym - v.chop! - if k != :cluster_state - @info[k] = v.to_i - else - @info[k] = v - end - } - elsif o[:getfriends] - @friends << info - end - } - end - - def add_slots(slots) - slots.each{|s| - @info[:slots][s] = :new - } - @dirty = true - end - - def flush_node_config - return if !@dirty - new = [] - @info[:slots].each{|s,val| - if val == :new - new << s - @info[:slots][s] = true - end - } - @r.cluster("addslots",*new) - @dirty = false - end - - def info_string - # We want to display the hash slots assigned to this node - # as ranges, like in: "1-5,8-9,20-25,30" - # - # Note: this could be easily written without side effects, - # we use 'slots' just to split the computation into steps. - - # First step: we want an increasing array of integers - # for instance: [1,2,3,4,5,8,9,20,21,22,23,24,25,30] - slots = @info[:slots].keys.sort - - # As we want to aggregate adiacent slots we convert all the - # slot integers into ranges (with just one element) - # So we have something like [1..1,2..2, ... and so forth. - slots.map!{|x| x..x} - - # Finally we group ranges with adiacent elements. - slots = slots.reduce([]) {|a,b| - if !a.empty? && b.first == (a[-1].last)+1 - a[0..-2] + [(a[-1].first)..(b.last)] - else - a + [b] - end - } - - # Now our task is easy, we just convert ranges with just one - # element into a number, and a real range into a start-end format. - # Finally we join the array using the comma as separator. - slots = slots.map{|x| - x.count == 1 ? x.first.to_s : "#{x.first}-#{x.last}" - }.join(",") - - "[#{@info[:cluster_state].upcase}] #{self.info[:name]} #{self.to_s} slots:#{slots} (#{self.slots.length} slots)" - end - - def info - @info - end - - def is_dirty? - @dirty - end - - def r - @r - end -end - -class RedisTrib - def initialize - @nodes = [] - end - - def check_arity(req_args, num_args) - if ((req_args > 0 and num_args != req_args) || - (req_args < 0 and num_args < req_args.abs)) - puts "Wrong number of arguments for specified sub command" - exit 1 - end - end - - def add_node(node) - @nodes << node - end - - def get_node_by_name(name) - @nodes.each{|n| - return n if n.info[:name] == name.downcase - } - return nil - end - - def check_cluster - puts "Performing Cluster Check (using node #{@nodes[0]})" - errors = [] - show_nodes - # Check if all the slots are covered - slots = {} - @nodes.each{|n| - slots = slots.merge(n.slots) - } - if slots.length == 4096 - puts "[OK] All 4096 slots covered." - else - errors << "[ERR] Not all 4096 slots are covered by nodes." - puts errors[-1] - end - return errors - end - - def alloc_slots - slots_per_node = ClusterHashSlots/@nodes.length - i = 0 - @nodes.each{|n| - first = i*slots_per_node - last = first+slots_per_node-1 - last = ClusterHashSlots-1 if i == @nodes.length-1 - n.add_slots first..last - i += 1 - } - end - - def flush_nodes_config - @nodes.each{|n| - n.flush_node_config - } - end - - def show_nodes - @nodes.each{|n| - puts n.info_string - } - end - - def join_cluster - # We use a brute force approach to make sure the node will meet - # each other, that is, sending CLUSTER MEET messages to all the nodes - # about the very same node. - # Thanks to gossip this information should propagate across all the - # cluster in a matter of seconds. - first = false - @nodes.each{|n| - if !first then first = n.info; next; end # Skip the first node - n.r.cluster("meet",first[:host],first[:port]) - } - end - - def yes_or_die(msg) - print "#{msg} (type 'yes' to accept): " - STDOUT.flush - if !(STDIN.gets.chomp.downcase == "yes") - puts "Aborting..." - exit 1 - end - end - - def load_cluster_info_from_node(nodeaddr) - node = ClusterNode.new(ARGV[1]) - node.connect(:abort => true) - node.assert_cluster - node.load_info(:getfriends => true) - add_node(node) - node.friends.each{|f| - fnode = ClusterNode.new(f[:addr]) - fnode.connect() - fnode.load_info() - add_node(fnode) - } - end - - # Given a list of source nodes return a "resharding plan" - # with what slots to move in order to move "numslots" slots to another - # instance. - def compute_reshard_table(sources,numslots) - moved = [] - # Sort from bigger to smaller instance, for two reasons: - # 1) If we take less slots than instanes it is better to start getting from - # the biggest instances. - # 2) We take one slot more from the first instance in the case of not perfect - # divisibility. Like we have 3 nodes and need to get 10 slots, we take - # 4 from the first, and 3 from the rest. So the biggest is always the first. - sources = sources.sort{|a,b| b.slots.length <=> a.slots.length} - source_tot_slots = sources.inject(0) {|sum,source| sum+source.slots.length} - sources.each_with_index{|s,i| - # Every node will provide a number of slots proportional to the - # slots it has assigned. - n = (numslots.to_f/source_tot_slots*s.slots.length) - if i == 0 - n = n.ceil - else - n = n.floor - end - s.slots.keys.sort[(0...n)].each{|slot| - if moved.length < numslots - moved << {:source => s, :slot => slot} - end - } - } - return moved - end - - def show_reshard_table(table) - table.each{|e| - puts " Moving slot #{e[:slot]} from #{e[:source].info[:name]}" - } - end - - def move_slot(source,target,slot,o={}) - # We start marking the slot as importing in the destination node, - # and the slot as migrating in the target host. Note that the order of - # the operations is important, as otherwise a client may be redirected to - # the target node that does not yet know it is importing this slot. - print "Moving slot #{slot} from #{source.info_string}: "; STDOUT.flush - target.r.cluster("setslot",slot,"importing",source.info[:name]) - source.r.cluster("setslot",slot,"migrating",source.info[:name]) - # Migrate all the keys from source to target using the MIGRATE command - while true - keys = source.r.cluster("getkeysinslot",slot,10) - break if keys.length == 0 - keys.each{|key| - source.r.migrate(target.info[:host],target.info[:port],key,0,1) - print "." if o[:verbose] - STDOUT.flush - } - end - puts - # Set the new node as the owner of the slot in all the known nodes. - @nodes.each{|n| - n.r.cluster("setslot",slot,"node",target.info[:name]) - } - end - - # redis-trib subcommands implementations - - def check_cluster_cmd - load_cluster_info_from_node(ARGV[1]) - check_cluster - end - - def reshard_cluster_cmd - load_cluster_info_from_node(ARGV[1]) - errors = check_cluster - if errors.length != 0 - puts "Please fix your cluster problems before resharding." - exit 1 - end - numslots = 0 - while numslots <= 0 or numslots > 4096 - print "How many slots do you want to move (from 1 to 4096)? " - numslots = STDIN.gets.to_i - end - target = nil - while not target - print "What is the receiving node ID? " - target = get_node_by_name(STDIN.gets.chop) - if not target - puts "The specified node is not known, please retry." - end - end - sources = [] - puts "Please enter all the source node IDs." - puts " Type 'all' to use all the nodes as source nodes for the hash slots." - puts " Type 'done' once you entered all the source nodes IDs." - while true - print "Source node ##{sources.length+1}:" - line = STDIN.gets.chop - src = get_node_by_name(line) - if line == "done" - if sources.length == 0 - puts "No source nodes given, operation aborted" - exit 1 - else - break - end - elsif line == "all" - @nodes.each{|n| - next if n.info[:name] == target.info[:name] - sources << n - } - break - elsif not src - puts "The specified node is not known, please retry." - elsif src.info[:name] == target.info[:name] - puts "It is not possible to use the target node as source node." - else - sources << src - end - end - puts "\nReady to move #{numslots} slots." - puts " Source nodes:" - sources.each{|s| puts " "+s.info_string} - puts " Destination node:" - puts " #{target.info_string}" - reshard_table = compute_reshard_table(sources,numslots) - puts " Resharding plan:" - show_reshard_table(reshard_table) - print "Do you want to proceed with the proposed reshard plan (yes/no)? " - yesno = STDIN.gets.chop - exit(1) if (yesno != "yes") - reshard_table.each{|e| - move_slot(e[:source],target,e[:slot],:verbose=>true) - } - end - - def create_cluster_cmd - puts "Creating cluster" - ARGV[1..-1].each{|n| - node = ClusterNode.new(n) - node.connect(:abort => true) - node.assert_cluster - node.load_info - node.assert_empty - add_node(node) - } - puts "Performing hash slots allocation on #{@nodes.length} nodes..." - alloc_slots - show_nodes - yes_or_die "Can I set the above configuration?" - flush_nodes_config - puts "** Nodes configuration updated" - puts "** Sending CLUSTER MEET messages to join the cluster" - join_cluster - check_cluster - end -end - -COMMANDS={ - "create" => ["create_cluster_cmd", -2, "host1:port host2:port ... hostN:port"], - "check" => ["check_cluster_cmd", 2, "host:port"], - "reshard" => ["reshard_cluster_cmd", 2, "host:port"] -} - -# Sanity check -if ARGV.length == 0 - puts "Usage: redis-trib " - puts - COMMANDS.each{|k,v| - puts " #{k.ljust(20)} #{v[2]}" - } - puts - exit 1 -end - -rt = RedisTrib.new -cmd_spec = COMMANDS[ARGV[0].downcase] -if !cmd_spec - puts "Unknown redis-trib subcommand '#{ARGV[0]}'" - exit 1 -end -rt.check_arity(cmd_spec[1],ARGV.length) - -# Dispatch -rt.send(cmd_spec[0]) From fbce475270e948c34c6e8e8fb15ac6cb74c41de6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Apr 2012 11:53:45 +0200 Subject: [PATCH 0084/1752] When the user-provided 'maxclients' value is too big for the max number of files we can open, at least try to search the max the OS is allowing (in steps of 256 filedes). --- src/redis.c | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index 557b361a5bb..0bff9339143 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1124,10 +1124,18 @@ void adjustOpenFilesLimit(void) { /* Set the max number of files if the current limit is not enough * for our needs. */ if (oldlimit < maxfiles) { - limit.rlim_cur = maxfiles; - limit.rlim_max = maxfiles; - if (setrlimit(RLIMIT_NOFILE,&limit) == -1) { - server.maxclients = oldlimit-32; + rlim_t f; + + f = maxfiles; + while(f > oldlimit) { + limit.rlim_cur = f; + limit.rlim_max = f; + if (setrlimit(RLIMIT_NOFILE,&limit) != -1) break; + f -= 128; + } + if (f < oldlimit) f = oldlimit; + if (f != maxfiles) { + server.maxclients = f-32; redisLog(REDIS_WARNING,"Unable to set the max number of files limit to %d (%s), setting the max clients configuration to %d.", (int) maxfiles, strerror(errno), (int) server.maxclients); } else { From bde80cb28bb8b74227f710028e3bf0890e62f3a7 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Apr 2012 12:17:40 +0200 Subject: [PATCH 0085/1752] Two fixed for MIGRATE: fix TTL propagation and fix transferring of data bigger than 64k. --- src/migrate.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index 90875312164..1e36a84d3e6 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -130,7 +130,7 @@ void migrateCommand(redisClient *c) { int fd; long timeout; long dbid; - time_t ttl; + long long ttl; robj *o; rio cmd, payload; @@ -168,7 +168,8 @@ void migrateCommand(redisClient *c) { redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); - ttl = getExpire(c->db,c->argv[3]); + ttl = getExpire(c->db,c->argv[3])-mstime(); + if (ttl < 1) ttl = 1; redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',4)); redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); @@ -190,7 +191,7 @@ void migrateCommand(redisClient *c) { while ((towrite = sdslen(buf)-pos) > 0) { towrite = (towrite > (64*1024) ? (64*1024) : towrite); - nwritten = syncWrite(fd,buf+nwritten,towrite,timeout); + nwritten = syncWrite(fd,buf+pos,towrite,timeout); if (nwritten != (signed)towrite) goto socket_wr_err; pos += nwritten; } From fb8409a5c8b46a1663d612c256a1a952c2d8d495 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Apr 2012 15:10:42 +0200 Subject: [PATCH 0086/1752] Another fix for MIGRATE. --- src/migrate.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index 1e36a84d3e6..aeb412b59b2 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -130,7 +130,7 @@ void migrateCommand(redisClient *c) { int fd; long timeout; long dbid; - long long ttl; + long long ttl, expireat; robj *o; rio cmd, payload; @@ -168,13 +168,16 @@ void migrateCommand(redisClient *c) { redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"SELECT",6)); redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,dbid)); - ttl = getExpire(c->db,c->argv[3])-mstime(); - if (ttl < 1) ttl = 1; + expireat = getExpire(c->db,c->argv[3]); + if (expireat != -1) { + ttl = expireat-mstime(); + if (ttl < 1) ttl = 1; + } redisAssertWithInfo(c,NULL,rioWriteBulkCount(&cmd,'*',4)); redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr))); - redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(ttl == -1) ? 0 : ttl)); + redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(expireat==-1) ? 0 : ttl)); /* Finally the last argument that is the serailized object payload * in the DUMP format. */ From e3fd3ccd721dcaeab4d9cd345d04b849cab77e5a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Apr 2012 15:10:51 +0200 Subject: [PATCH 0087/1752] More MIGRATE tests. --- tests/unit/dump.tcl | 46 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index 6b432377ee9..b73cde0c17b 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -42,6 +42,52 @@ start_server {tags {"dump"}} { assert {[$first exists key] == 0} assert {[$second exists key] == 1} assert {[$second get key] eq {Some Value}} + assert {[$second ttl key] == -1} + } + } + + test {MIGRATE propagates TTL correctly} { + set first [srv 0 client] + r set key "Some Value" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + $first expire key 10 + set ret [r -1 migrate $second_host $second_port key 9 5000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second get key] eq {Some Value}} + assert {[$second ttl key] >= 7 && [$second ttl key] <= 10} + } + } + + test {MIGRATE can correctly transfer large values} { + set first [srv 0 client] + r del key + for {set j 0} {$j < 5000} {incr j} { + r rpush key 1 2 3 4 5 6 7 8 9 10 + r rpush key "item 1" "item 2" "item 3" "item 4" "item 5" \ + "item 6" "item 7" "item 8" "item 9" "item 10" + } + assert {[string length [r dump key]] > (1024*64)} + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + set ret [r -1 migrate $second_host $second_port key 9 10000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second ttl key] == -1} + assert {[$second llen key] == 5000*20} } } From 3c413d59c85d222f6519ffe46be2e3210f78f31d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Apr 2012 15:29:47 +0200 Subject: [PATCH 0088/1752] redis-cli help.h updated. Script to generate it updated as well. --- src/help.h | 380 ++++++++++++++++++++------------- utils/generate-command-help.rb | 5 +- 2 files changed, 236 insertions(+), 149 deletions(-) diff --git a/src/help.h b/src/help.h index 51613c9b3b4..cde95c1f841 100644 --- a/src/help.h +++ b/src/help.h @@ -1,4 +1,4 @@ -/* Automatically generated by generate-command-help.rb, do not edit. */ +/* Automatically generated by utils/generate-command-help.rb, do not edit. */ #ifndef __REDIS_HELP_H #define __REDIS_HELP_H @@ -13,7 +13,8 @@ static char *commandGroups[] = { "pubsub", "transactions", "connection", - "server" + "server", + "scripting" }; struct commandHelp { @@ -27,612 +28,697 @@ struct commandHelp { "key value", "Append a value to a key", 1, - "1.3.3" }, + "2.0.0" }, { "AUTH", "password", "Authenticate to the server", 8, - "0.08" }, + "1.0.0" }, { "BGREWRITEAOF", "-", "Asynchronously rewrite the append-only file", 9, - "1.07" }, + "1.0.0" }, { "BGSAVE", "-", "Asynchronously save the dataset to disk", 9, - "0.07" }, + "1.0.0" }, { "BLPOP", "key [key ...] timeout", "Remove and get the first element in a list, or block until one is available", 2, - "1.3.1" }, + "2.0.0" }, { "BRPOP", "key [key ...] timeout", "Remove and get the last element in a list, or block until one is available", 2, - "1.3.1" }, + "2.0.0" }, { "BRPOPLPUSH", "source destination timeout", "Pop a value from a list, push it to another list and return it; or block until one is available", 2, - "2.1.7" }, + "2.2.0" }, { "CONFIG GET", "parameter", "Get the value of a configuration parameter", 9, - "2.0" }, + "2.0.0" }, { "CONFIG RESETSTAT", "-", "Reset the stats returned by INFO", 9, - "2.0" }, + "2.0.0" }, { "CONFIG SET", "parameter value", "Set a configuration parameter to the given value", 9, - "2.0" }, + "2.0.0" }, { "DBSIZE", "-", "Return the number of keys in the selected database", 9, - "0.07" }, + "1.0.0" }, { "DEBUG OBJECT", "key", "Get debugging information about a key", 9, - "0.101" }, + "1.0.0" }, { "DEBUG SEGFAULT", "-", "Make the server crash", 9, - "0.101" }, + "1.0.0" }, { "DECR", "key", "Decrement the integer value of a key by one", 1, - "0.07" }, + "1.0.0" }, { "DECRBY", "key decrement", "Decrement the integer value of a key by the given number", 1, - "0.07" }, + "1.0.0" }, { "DEL", "key [key ...]", "Delete a key", 0, - "0.07" }, + "1.0.0" }, { "DISCARD", "-", "Discard all commands issued after MULTI", 7, - "1.3.3" }, + "2.0.0" }, + { "DUMP", + "key", + "Return a serialized verison of the value stored at the specified key.", + 0, + "2.6.0" }, { "ECHO", "message", "Echo the given string", 8, - "0.07" }, + "1.0.0" }, + { "EVAL", + "script numkeys key [key ...] arg [arg ...]", + "Execute a Lua script server side", + 10, + "2.6.0" }, { "EXEC", "-", "Execute all commands issued after MULTI", 7, - "1.1.95" }, + "1.2.0" }, { "EXISTS", "key", "Determine if a key exists", - 9, - "0.07" }, + 0, + "1.0.0" }, { "EXPIRE", "key seconds", "Set a key's time to live in seconds", 0, - "0.09" }, + "1.0.0" }, { "EXPIREAT", "key timestamp", "Set the expiration for a key as a UNIX timestamp", 0, - "1.1" }, + "1.2.0" }, { "FLUSHALL", "-", "Remove all keys from all databases", 9, - "0.07" }, + "1.0.0" }, { "FLUSHDB", "-", "Remove all keys from the current database", 9, - "0.07" }, + "1.0.0" }, { "GET", "key", "Get the value of a key", 1, - "0.07" }, + "1.0.0" }, { "GETBIT", "key offset", "Returns the bit value at offset in the string value stored at key", 1, - "2.1.8" }, + "2.2.0" }, + { "GETRANGE", + "key start end", + "Get a substring of the string stored at a key", + 1, + "2.4.0" }, { "GETSET", "key value", "Set the string value of a key and return its old value", 1, - "0.091" }, + "1.0.0" }, { "HDEL", - "key field", - "Delete a hash field", + "key field [field ...]", + "Delete one or more hash fields", 5, - "1.3.10" }, + "2.0.0" }, { "HEXISTS", "key field", "Determine if a hash field exists", 5, - "1.3.10" }, + "2.0.0" }, { "HGET", "key field", "Get the value of a hash field", 5, - "1.3.10" }, + "2.0.0" }, { "HGETALL", "key", "Get all the fields and values in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "HINCRBY", "key field increment", "Increment the integer value of a hash field by the given number", 5, - "1.3.10" }, + "2.0.0" }, + { "HINCRBYFLOAT", + "key field increment", + "Increment the float value of a hash field by the given amount", + 5, + "2.6.0" }, { "HKEYS", "key", "Get all the fields in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "HLEN", "key", "Get the number of fields in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "HMGET", "key field [field ...]", "Get the values of all the given hash fields", 5, - "1.3.10" }, + "2.0.0" }, { "HMSET", "key field value [field value ...]", "Set multiple hash fields to multiple values", 5, - "1.3.8" }, + "2.0.0" }, { "HSET", "key field value", "Set the string value of a hash field", 5, - "1.3.10" }, + "2.0.0" }, { "HSETNX", "key field value", "Set the value of a hash field, only if the field does not exist", 5, - "1.3.8" }, + "2.0.0" }, { "HVALS", "key", "Get all the values in a hash", 5, - "1.3.10" }, + "2.0.0" }, { "INCR", "key", "Increment the integer value of a key by one", 1, - "0.07" }, + "1.0.0" }, { "INCRBY", "key increment", - "Increment the integer value of a key by the given number", + "Increment the integer value of a key by the given amount", 1, - "0.07" }, + "1.0.0" }, + { "INCRBYFLOAT", + "key increment", + "Increment the float value of a key by the given amount", + 1, + "2.6.0" }, { "INFO", "-", "Get information and statistics about the server", 9, - "0.07" }, + "1.0.0" }, { "KEYS", "pattern", "Find all keys matching the given pattern", 0, - "0.07" }, + "1.0.0" }, { "LASTSAVE", "-", "Get the UNIX time stamp of the last successful save to disk", 9, - "0.07" }, + "1.0.0" }, { "LINDEX", "key index", "Get an element from a list by its index", 2, - "0.07" }, + "1.0.0" }, { "LINSERT", "key BEFORE|AFTER pivot value", "Insert an element before or after another element in a list", 2, - "2.1.1" }, + "2.2.0" }, { "LLEN", "key", "Get the length of a list", 2, - "0.07" }, + "1.0.0" }, { "LPOP", "key", "Remove and get the first element in a list", 2, - "0.07" }, + "1.0.0" }, { "LPUSH", - "key value", - "Prepend a value to a list", + "key value [value ...]", + "Prepend one or multiple values to a list", 2, - "0.07" }, + "1.0.0" }, { "LPUSHX", "key value", "Prepend a value to a list, only if the list exists", 2, - "2.1.1" }, + "2.2.0" }, { "LRANGE", "key start stop", "Get a range of elements from a list", 2, - "0.07" }, + "1.0.0" }, { "LREM", "key count value", "Remove elements from a list", 2, - "0.07" }, + "1.0.0" }, { "LSET", "key index value", "Set the value of an element in a list by its index", 2, - "0.07" }, + "1.0.0" }, { "LTRIM", "key start stop", "Trim a list to the specified range", 2, - "0.07" }, + "1.0.0" }, { "MGET", "key [key ...]", "Get the values of all the given keys", 1, - "0.07" }, + "1.0.0" }, + { "MIGRATE", + "host port key destination db timeout", + "Atomically transfer a key from a Redis instance to another one.", + 0, + "2.6.0" }, { "MONITOR", "-", "Listen for all requests received by the server in real time", 9, - "0.07" }, + "1.0.0" }, { "MOVE", "key db", "Move a key to another database", 0, - "0.07" }, + "1.0.0" }, { "MSET", "key value [key value ...]", "Set multiple keys to multiple values", 1, - "1.001" }, + "1.0.1" }, { "MSETNX", "key value [key value ...]", "Set multiple keys to multiple values, only if none of the keys exist", 1, - "1.001" }, + "1.0.1" }, { "MULTI", "-", "Mark the start of a transaction block", 7, - "1.1.95" }, + "1.2.0" }, + { "OBJECT", + "subcommand [arguments [arguments ...]]", + "Inspect the internals of Redis objects", + 0, + "2.2.3" }, { "PERSIST", "key", "Remove the expiration from a key", 0, - "2.1.2" }, + "2.2.0" }, + { "PEXPIRE", + "key milliseconds", + "Set a key's time to live in milliseconds", + 0, + "2.6.0" }, + { "PEXPIREAT", + "key milliseconds timestamp", + "Set the expiration for a key as a UNIX timestamp specified in milliseconds", + 0, + "2.6.0" }, { "PING", "-", "Ping the server", 8, - "0.07" }, + "1.0.0" }, + { "PSETEX", + "key milliseconds value", + "Set the value and expiration in milliseconds of a key", + 1, + "2.6.0" }, { "PSUBSCRIBE", - "pattern", + "pattern [pattern ...]", "Listen for messages published to channels matching the given patterns", 6, - "1.3.8" }, + "2.0.0" }, + { "PTTL", + "key", + "Get the time to live for a key in milliseconds", + 0, + "2.6.0" }, { "PUBLISH", "channel message", "Post a message to a channel", 6, - "1.3.8" }, + "2.0.0" }, { "PUNSUBSCRIBE", "[pattern [pattern ...]]", "Stop listening for messages posted to channels matching the given patterns", 6, - "1.3.8" }, + "2.0.0" }, { "QUIT", "-", "Close the connection", 8, - "0.07" }, + "1.0.0" }, { "RANDOMKEY", "-", "Return a random key from the keyspace", 0, - "0.07" }, + "1.0.0" }, { "RENAME", "key newkey", "Rename a key", 0, - "0.07" }, + "1.0.0" }, { "RENAMENX", "key newkey", "Rename a key, only if the new key does not exist", 0, - "0.07" }, + "1.0.0" }, + { "RESTORE", + "key ttl serialized value", + "Create a key using the provided serialized value, previously obtained using DUMP.", + 0, + "2.6.0" }, { "RPOP", "key", "Remove and get the last element in a list", 2, - "0.07" }, + "1.0.0" }, { "RPOPLPUSH", "source destination", "Remove the last element in a list, append it to another list and return it", 2, - "1.1" }, + "1.2.0" }, { "RPUSH", - "key value", - "Append a value to a list", + "key value [value ...]", + "Append one or multiple values to a list", 2, - "0.07" }, + "1.0.0" }, { "RPUSHX", "key value", "Append a value to a list, only if the list exists", 2, - "2.1.1" }, + "2.2.0" }, { "SADD", - "key member", - "Add a member to a set", + "key member [member ...]", + "Add one or more members to a set", 3, - "0.07" }, + "1.0.0" }, { "SAVE", "-", "Synchronously save the dataset to disk", 9, - "0.07" }, + "1.0.0" }, { "SCARD", "key", "Get the number of members in a set", 3, - "0.07" }, + "1.0.0" }, + { "SCRIPT EXISTS", + "script [script ...]", + "Check existence of scripts in the script cache.", + 10, + "2.6.0" }, + { "SCRIPT FLUSH", + "-", + "Remove all the scripts from the script cache.", + 10, + "2.6.0" }, + { "SCRIPT KILL", + "-", + "Kill the script currently in execution.", + 10, + "2.6.0" }, + { "SCRIPT LOAD", + "script", + "Load the specified Lua script into the script cache.", + 10, + "2.6.0" }, { "SDIFF", "key [key ...]", "Subtract multiple sets", 3, - "0.100" }, + "1.0.0" }, { "SDIFFSTORE", "destination key [key ...]", "Subtract multiple sets and store the resulting set in a key", 3, - "0.100" }, + "1.0.0" }, { "SELECT", "index", "Change the selected database for the current connection", 8, - "0.07" }, + "1.0.0" }, { "SET", "key value", "Set the string value of a key", 1, - "0.07" }, + "1.0.0" }, { "SETBIT", "key offset value", "Sets or clears the bit at offset in the string value stored at key", 1, - "2.1.8" }, + "2.2.0" }, { "SETEX", "key seconds value", "Set the value and expiration of a key", 1, - "1.3.10" }, + "2.0.0" }, { "SETNX", "key value", "Set the value of a key, only if the key does not exist", 1, - "0.07" }, + "1.0.0" }, { "SETRANGE", "key offset value", "Overwrite part of a string at key starting at the specified offset", 1, - "2.1.8" }, + "2.2.0" }, { "SHUTDOWN", - "-", + "[NOSAVE] [SAVE]", "Synchronously save the dataset to disk and then shut down the server", 9, - "0.07" }, + "1.0.0" }, { "SINTER", "key [key ...]", "Intersect multiple sets", 3, - "0.07" }, + "1.0.0" }, { "SINTERSTORE", "destination key [key ...]", "Intersect multiple sets and store the resulting set in a key", 3, - "0.07" }, + "1.0.0" }, { "SISMEMBER", "key member", "Determine if a given value is a member of a set", 3, - "0.07" }, + "1.0.0" }, { "SLAVEOF", "host port", "Make the server a slave of another instance, or promote it as master", 9, - "0.100" }, + "1.0.0" }, + { "SLOWLOG", + "subcommand [argument]", + "Manages the Redis slow queries log", + 9, + "2.2.12" }, { "SMEMBERS", "key", "Get all the members in a set", 3, - "0.07" }, + "1.0.0" }, { "SMOVE", "source destination member", "Move a member from one set to another", 3, - "0.091" }, + "1.0.0" }, { "SORT", "key [BY pattern] [LIMIT offset count] [GET pattern [GET pattern ...]] [ASC|DESC] [ALPHA] [STORE destination]", "Sort the elements in a list, set or sorted set", 0, - "0.07" }, + "1.0.0" }, { "SPOP", "key", "Remove and return a random member from a set", 3, - "0.101" }, + "1.0.0" }, { "SRANDMEMBER", "key", "Get a random member from a set", 3, - "1.001" }, + "1.0.0" }, { "SREM", - "key member", - "Remove a member from a set", + "key member [member ...]", + "Remove one or more members from a set", 3, - "0.07" }, + "1.0.0" }, { "STRLEN", "key", "Get the length of the value stored in a key", 1, - "2.1.2" }, + "2.2.0" }, { "SUBSCRIBE", - "channel", + "channel [channel ...]", "Listen for messages published to the given channels", 6, - "1.3.8" }, - { "SUBSTR", - "key start end", - "Get a substring of the string stored at a key", - 1, - "1.3.4" }, + "2.0.0" }, { "SUNION", "key [key ...]", "Add multiple sets", 3, - "0.091" }, + "1.0.0" }, { "SUNIONSTORE", "destination key [key ...]", "Add multiple sets and store the resulting set in a key", 3, - "0.091" }, + "1.0.0" }, { "SYNC", "-", "Internal command used for replication", 9, - "0.07" }, + "1.0.0" }, + { "TIME", + "-", + "Return the current server time", + 9, + "2.6.0" }, { "TTL", "key", "Get the time to live for a key", 0, - "0.100" }, + "1.0.0" }, { "TYPE", "key", "Determine the type stored at key", 0, - "0.07" }, + "1.0.0" }, { "UNSUBSCRIBE", "[channel [channel ...]]", "Stop listening for messages posted to the given channels", 6, - "1.3.8" }, + "2.0.0" }, { "UNWATCH", "-", "Forget about all watched keys", 7, - "2.1.0" }, + "2.2.0" }, { "WATCH", "key [key ...]", "Watch the given keys to determine execution of the MULTI/EXEC block", 7, - "2.1.0" }, + "2.2.0" }, { "ZADD", - "key score member", - "Add a member to a sorted set, or update its score if it already exists", + "key score member [score] [member]", + "Add one or more members to a sorted set, or update its score if it already exists", 4, - "1.1" }, + "1.2.0" }, { "ZCARD", "key", "Get the number of members in a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZCOUNT", "key min max", "Count the members in a sorted set with scores within the given values", 4, - "1.3.3" }, + "2.0.0" }, { "ZINCRBY", "key increment member", "Increment the score of a member in a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZINTERSTORE", "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", "Intersect multiple sorted sets and store the resulting sorted set in a new key", 4, - "1.3.10" }, + "2.0.0" }, { "ZRANGE", "key start stop [WITHSCORES]", "Return a range of members in a sorted set, by index", 4, - "1.1" }, + "1.2.0" }, { "ZRANGEBYSCORE", "key min max [WITHSCORES] [LIMIT offset count]", "Return a range of members in a sorted set, by score", 4, - "1.050" }, + "1.0.5" }, { "ZRANK", "key member", "Determine the index of a member in a sorted set", 4, - "1.3.4" }, + "2.0.0" }, { "ZREM", - "key member", - "Remove a member from a sorted set", + "key member [member ...]", + "Remove one or more members from a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZREMRANGEBYRANK", "key start stop", "Remove all members in a sorted set within the given indexes", 4, - "1.3.4" }, + "2.0.0" }, { "ZREMRANGEBYSCORE", "key min max", "Remove all members in a sorted set within the given scores", 4, - "1.1" }, + "1.2.0" }, { "ZREVRANGE", "key start stop [WITHSCORES]", "Return a range of members in a sorted set, by index, with scores ordered from high to low", 4, - "1.1" }, + "1.2.0" }, { "ZREVRANGEBYSCORE", "key max min [WITHSCORES] [LIMIT offset count]", "Return a range of members in a sorted set, by score, with scores ordered from high to low", 4, - "2.1.6" }, + "2.2.0" }, { "ZREVRANK", "key member", "Determine the index of a member in a sorted set, with scores ordered from high to low", 4, - "1.3.4" }, + "2.0.0" }, { "ZSCORE", "key member", "Get the score associated with the given member in a sorted set", 4, - "1.1" }, + "1.2.0" }, { "ZUNIONSTORE", "destination numkeys key [key ...] [WEIGHTS weight] [AGGREGATE SUM|MIN|MAX]", "Add multiple sorted sets and store the resulting sorted set in a new key", 4, - "1.3.10" } + "2.0.0" } }; #endif diff --git a/utils/generate-command-help.rb b/utils/generate-command-help.rb index 96cccc2bfbc..f6ca8874b4e 100755 --- a/utils/generate-command-help.rb +++ b/utils/generate-command-help.rb @@ -10,7 +10,8 @@ "pubsub", "transactions", "connection", - "server" + "server", + "scripting" ].freeze GROUPS_BY_NAME = Hash[* @@ -48,7 +49,7 @@ def commands require "json" require "uri" - url = URI.parse "https://github.com/antirez/redis-doc/raw/master/commands.json" + url = URI.parse "https://raw.github.com/antirez/redis-doc/master/commands.json" client = Net::HTTP.new url.host, url.port client.use_ssl = true response = client.get url.path From df35d873369d8d8ad35a9c98e05f67b5f558966e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Apr 2012 15:11:17 +0200 Subject: [PATCH 0089/1752] Print milliseconds of the current second in log lines timestamps. Sometimes precise timing is very important for debugging. --- src/redis.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0bff9339143..a7610fa4d7f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -251,7 +251,6 @@ struct redisCommand redisCommandTable[] = { void redisLogRaw(int level, const char *msg) { const int syslogLevelMap[] = { LOG_DEBUG, LOG_INFO, LOG_NOTICE, LOG_WARNING }; const char *c = ".-*#"; - time_t now = time(NULL); FILE *fp; char buf[64]; int rawmode = (level & REDIS_LOG_RAW); @@ -265,7 +264,12 @@ void redisLogRaw(int level, const char *msg) { if (rawmode) { fprintf(fp,"%s",msg); } else { - strftime(buf,sizeof(buf),"%d %b %H:%M:%S",localtime(&now)); + int off; + struct timeval tv; + + gettimeofday(&tv,NULL); + off = strftime(buf,sizeof(buf),"%d %b %H:%M:%S.",localtime(&tv.tv_sec)); + snprintf(buf+off,sizeof(buf)-off,"%03d",(int)tv.tv_usec/1000); fprintf(fp,"[%d] %s %c %s\n",(int)getpid(),buf,c[level],msg); } fflush(fp); From a5b75e9a8649519238cd80405e000f57c0d6c489 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Apr 2012 15:11:30 +0200 Subject: [PATCH 0090/1752] SLAVEOF is not a write command. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index a7610fa4d7f..7befbac2335 100644 --- a/src/redis.c +++ b/src/redis.c @@ -222,7 +222,7 @@ struct redisCommand redisCommandTable[] = { {"ttl",ttlCommand,2,"r",0,NULL,1,1,1,0,0}, {"pttl",pttlCommand,2,"r",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"w",0,NULL,1,1,1,0,0}, - {"slaveof",slaveofCommand,3,"aws",0,NULL,0,0,0,0,0}, + {"slaveof",slaveofCommand,3,"as",0,NULL,0,0,0,0,0}, {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, {"config",configCommand,-2,"ar",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"rps",0,NULL,0,0,0,0,0}, From fa2a27cfeb0de6696aba757fce0d723c4770ffad Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Apr 2012 15:38:13 +0200 Subject: [PATCH 0091/1752] New "os" field in INFO output providing information about the operating system. --- src/redis.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/redis.c b/src/redis.c index 7befbac2335..c73333177e1 100644 --- a/src/redis.c +++ b/src/redis.c @@ -48,6 +48,7 @@ #include #include #include +#include /* Our shared "common" objects */ @@ -1697,12 +1698,16 @@ sds genRedisInfoString(char *section) { /* Server */ if (allsections || defsections || !strcasecmp(section,"server")) { + struct utsname name; + if (sections++) info = sdscat(info,"\r\n"); + uname(&name); info = sdscatprintf(info, "# Server\r\n" "redis_version:%s\r\n" "redis_git_sha1:%s\r\n" "redis_git_dirty:%d\r\n" + "os:%s %s %s\r\n" "arch_bits:%d\r\n" "multiplexing_api:%s\r\n" "gcc_version:%d.%d.%d\r\n" @@ -1715,6 +1720,7 @@ sds genRedisInfoString(char *section) { REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, + name.sysname, name.release, name.machine, server.arch_bits, aeGetApiName(), #ifdef __GNUC__ From 09eb4487468052a7d14c2ac2805ef5a582157e41 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Apr 2012 18:31:37 +0200 Subject: [PATCH 0092/1752] Structure field controlling the INFO field master_link_down_since_seconds initialized correctly to avoid strange INFO output at startup when a slave has yet to connect to its master. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index c73333177e1..8c796c93912 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1066,7 +1066,7 @@ void initServerConfig() { server.repl_syncio_timeout = REDIS_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = 1; server.repl_slave_ro = 1; - server.repl_down_since = -1; + server.repl_down_since = time(NULL); /* Client output buffer limits */ server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].hard_limit_bytes = 0; From 6c52d5ce274dff94587dd66e1f57e973c8dd98e4 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Mon, 2 Apr 2012 11:56:03 +0200 Subject: [PATCH 0093/1752] remove mentions of VM in comments --- src/debug.c | 1 - src/object.c | 12 +----------- src/redis.h | 1 - 3 files changed, 1 insertion(+), 13 deletions(-) diff --git a/src/debug.c b/src/debug.c index e7b3ba407fd..37e6a1f72c1 100644 --- a/src/debug.c +++ b/src/debug.c @@ -105,7 +105,6 @@ void computeDatasetDigest(unsigned char *final) { mixDigest(digest,key,sdslen(key)); - /* Make sure the key is loaded if VM is active */ o = dictGetVal(de); aux = htonl(o->type); diff --git a/src/object.c b/src/object.c index a15ebcfa837..ba7ea323a6f 100644 --- a/src/object.c +++ b/src/object.c @@ -9,18 +9,8 @@ robj *createObject(int type, void *ptr) { o->ptr = ptr; o->refcount = 1; - /* Set the LRU to the current lruclock (minutes resolution). - * We do this regardless of the fact VM is active as LRU is also - * used for the maxmemory directive when Redis is used as cache. - * - * Note that this code may run in the context of an I/O thread - * and accessing server.lruclock in theory is an error - * (no locks). But in practice this is safe, and even if we read - * garbage Redis will not fail. */ + /* Set the LRU to the current lruclock (minutes resolution). */ o->lru = server.lruclock; - /* The following is only needed if VM is active, but since the conditional - * is probably more costly than initializing the field it's better to - * have every field properly initialized anyway. */ return o; } diff --git a/src/redis.h b/src/redis.h index f2229caf930..9d0e19ff2c5 100644 --- a/src/redis.h +++ b/src/redis.h @@ -89,7 +89,6 @@ #define REDIS_SET 2 #define REDIS_ZSET 3 #define REDIS_HASH 4 -#define REDIS_VMPOINTER 8 /* Objects encoding. Some kind of objects like Strings and Hashes can be * internally represented in multiple ways. The 'encoding' field of the object From 4d57e44839f93b048cf0bb79f1b901b9f554bff4 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 3 Apr 2012 13:32:49 +0200 Subject: [PATCH 0094/1752] new option for choosing number of test clients to run --- tests/test_helper.tcl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 34d96606d13..2ec3aad1c16 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -345,6 +345,7 @@ proc print_help_screen {} { "--quiet Don't show individual tests." "--single Just execute the specified unit (see next option)." "--list-tests List all the available test units." + "--clients Number of test clients (16)." "--force-failure Force the execution of a test that always fails." "--help Print this help screen." } "\n"] @@ -390,6 +391,9 @@ for {set j 0} {$j < [llength $argv]} {incr j} { set ::client 1 set ::test_server_port $arg incr j + } elseif {$opt eq {--clients}} { + set ::numclients $arg + incr j } elseif {$opt eq {--help}} { print_help_screen exit 0 From b811b334aeb57b426a03daa0ef3ccacbe0ff33d0 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 3 Apr 2012 14:18:35 +0200 Subject: [PATCH 0095/1752] in kill_server send the signal once, then wait for up to 5sec before sending lethal SIGKILL --- tests/support/server.tcl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 984270adb1b..b1ab38fc108 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -46,11 +46,16 @@ proc kill_server config { } # kill server and wait for the process to be totally exited + catch {exec kill $pid} while {[is_alive $config]} { - if {[incr wait 10] % 1000 == 0} { + incr wait 10 + + if {$wait >= 5000} { + puts "Forcing process $pid to exit..." + catch {exec kill -KILL $pid} + } elseif {$wait % 1000 == 0} { puts "Waiting for process $pid to exit..." } - catch {exec kill $pid} after 10 } From c9adc81e2e4fc526bc1fc96d40b423ace2378cd2 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Tue, 3 Apr 2012 17:40:31 +0200 Subject: [PATCH 0096/1752] allocate alternate signal stack, change of sigaction flags for sigterm --- src/redis.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 8c796c93912..a6931876383 100644 --- a/src/redis.c +++ b/src/redis.c @@ -62,6 +62,9 @@ double R_Zero, R_PosInf, R_NegInf, R_Nan; /*================================= Globals ================================= */ +/* Alternate stack for SIGSEGV/etc handlers */ +char altstack[SIGSTKSZ]; + /* Global vars */ struct redisServer server; /* server global state */ struct redisCommand *commandTable; @@ -2278,15 +2281,24 @@ static void sigtermHandler(int sig) { void setupSignalHandlers(void) { struct sigaction act; + stack_t stack; + + stack.ss_sp = altstack; + stack.ss_flags = 0; + stack.ss_size = SIGSTKSZ; + + sigaltstack(&stack, NULL); /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction is used. * Otherwise, sa_handler is used. */ sigemptyset(&act.sa_mask); - act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND; + act.sa_flags = 0; act.sa_handler = sigtermHandler; sigaction(SIGTERM, &act, NULL); #ifdef HAVE_BACKTRACE + /* Use alternate stack so we don't clobber stack in case of segv, or when we run out of stack .. + * also resethand & nodefer so we can get interrupted (and killed) if we cause SEGV during SEGV handler */ sigemptyset(&act.sa_mask); act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND | SA_SIGINFO; act.sa_sigaction = sigsegvHandler; From c537980d9893d033407506c3843dd62d5b439d52 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Apr 2012 19:54:23 +0200 Subject: [PATCH 0097/1752] On slow computers, 10 seconds are not enough for this heavy replication test. --- tests/integration/replication.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 2c7d98deaa0..7c1edb550a2 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -97,7 +97,7 @@ start_server {tags {"repl"}} { [lindex $slaves 2] slaveof $master_host $master_port # Wait for all the three slaves to reach the "online" state - set retry 100 + set retry 500 while {$retry} { set info [r -3 info] if {[string match {*slave0:*,online*slave1:*,online*slave2:*,online*} $info]} { From 937abebbe10ec3b25b40bb20bcd79fc0eeec0cba Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Thu, 5 Apr 2012 08:25:22 +0200 Subject: [PATCH 0098/1752] future-proof version comparison --- src/zmalloc.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zmalloc.h b/src/zmalloc.h index 8079fb5319a..ff555619e5f 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -38,7 +38,7 @@ #if defined(USE_TCMALLOC) #define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR)) #include -#if TC_VERSION_MAJOR >= 1 && TC_VERSION_MINOR >= 6 +#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1) #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) tc_malloc_size(p) #else @@ -49,7 +49,7 @@ #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) #define JEMALLOC_MANGLE #include -#if JEMALLOC_VERSION_MAJOR >= 2 && JEMALLOC_VERSION_MINOR >= 1 +#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) #define HAVE_MALLOC_SIZE 1 #define zmalloc_size(p) JEMALLOC_P(malloc_usable_size)(p) #else From 9f899440d2d33c0a6a54ae85e987460d551e20d9 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Wed, 4 Apr 2012 19:17:32 +0200 Subject: [PATCH 0099/1752] add support for generation of lcov coverage reports --- src/Makefile | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 34ef7a7cea2..11f270b9d1e 100644 --- a/src/Makefile +++ b/src/Makefile @@ -226,10 +226,10 @@ redis-check-aof: .make-prerequisites $(CHECKAOFOBJ) %.o: %.c .make-prerequisites $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< -.PHONY: all clean distclean +.PHONY: all clean distclean lcov clean: - rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov + rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html distclean: clean -(cd ../deps && $(MAKE) distclean) @@ -241,6 +241,12 @@ dep: test: redis-server redis-check-aof @(cd ..; ./runtest) +lcov: + $(MAKE) clean gcov + @(set -e; cd ..; ./runtest --clients 1) + @geninfo -o redis.info . + @genhtml --legend -o lcov-html redis.info + bench: ./redis-benchmark From b518a83525e7407d38c413f73a338996bdac6b64 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Wed, 28 Mar 2012 12:03:32 +0200 Subject: [PATCH 0100/1752] fix mstime() ommited while comparing if key is already expired --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 73ee9166a00..b8c87b4f095 100644 --- a/src/db.c +++ b/src/db.c @@ -529,7 +529,7 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { * * Instead we take the other branch of the IF statement setting an expire * (possibly in the past) and wait for an explicit DEL from the master. */ - if (milliseconds <= 0 && !server.loading && !server.masterhost) { + if (milliseconds <= mstime() && !server.loading && !server.masterhost) { robj *aux; redisAssertWithInfo(c,key,dbDelete(c->db,key)); From 81a28fe131adbffb96563c89ba67a50f1b352722 Mon Sep 17 00:00:00 2001 From: Premysl Hruby Date: Wed, 28 Mar 2012 11:48:36 +0200 Subject: [PATCH 0101/1752] for (p)expireat use absolute time, without double recomputation --- src/db.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/db.c b/src/db.c index b8c87b4f095..2cc70d2c3c0 100644 --- a/src/db.c +++ b/src/db.c @@ -516,7 +516,7 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { return; if (unit == UNIT_SECONDS) milliseconds *= 1000; - milliseconds -= offset; + milliseconds += offset; de = dictFind(c->db->dict,key->ptr); if (de == NULL) { @@ -543,8 +543,7 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { addReply(c, shared.cone); return; } else { - long long when = mstime()+milliseconds; - setExpire(c->db,key,when); + setExpire(c->db,key,milliseconds); addReply(c,shared.cone); signalModifiedKey(c->db,key); server.dirty++; @@ -553,19 +552,19 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { } void expireCommand(redisClient *c) { - expireGenericCommand(c,0,UNIT_SECONDS); + expireGenericCommand(c,mstime(),UNIT_SECONDS); } void expireatCommand(redisClient *c) { - expireGenericCommand(c,mstime(),UNIT_SECONDS); + expireGenericCommand(c,0,UNIT_SECONDS); } void pexpireCommand(redisClient *c) { - expireGenericCommand(c,0,UNIT_MILLISECONDS); + expireGenericCommand(c,mstime(),UNIT_MILLISECONDS); } void pexpireatCommand(redisClient *c) { - expireGenericCommand(c,mstime(),UNIT_MILLISECONDS); + expireGenericCommand(c,0,UNIT_MILLISECONDS); } void ttlGenericCommand(redisClient *c, int output_ms) { From 3aad0de2e9f7c1f6f3d1259995fcfe0eade91c5d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Apr 2012 15:52:08 +0200 Subject: [PATCH 0102/1752] expireGenericCommand(): better variable names and a top-comment that describes the function's behavior. --- src/db.c | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/db.c b/src/db.c index 2cc70d2c3c0..943878b4074 100644 --- a/src/db.c +++ b/src/db.c @@ -507,16 +507,23 @@ int stringObjectEqualsMs(robj *a) { return tolower(arg[0]) == 'm' && tolower(arg[1]) == 's' && arg[2] == '\0'; } -void expireGenericCommand(redisClient *c, long long offset, int unit) { +/* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT + * and PEXPIREAT. Because the commad second argument may be relative or absolute + * the "basetime" argument is used to signal what the base time is (either 0 + * for *AT variants of the command, or the current time for relative expires). + * + * unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for + * the argv[2] parameter. The basetime is always specified in milliesconds. */ +void expireGenericCommand(redisClient *c, long long basetime, int unit) { dictEntry *de; robj *key = c->argv[1], *param = c->argv[2]; - long long milliseconds; + long long when; /* unix time in milliseconds when the key will expire. */ - if (getLongLongFromObjectOrReply(c, param, &milliseconds, NULL) != REDIS_OK) + if (getLongLongFromObjectOrReply(c, param, &when, NULL) != REDIS_OK) return; - if (unit == UNIT_SECONDS) milliseconds *= 1000; - milliseconds += offset; + if (unit == UNIT_SECONDS) when *= 1000; + when += basetime; de = dictFind(c->db->dict,key->ptr); if (de == NULL) { @@ -529,7 +536,7 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { * * Instead we take the other branch of the IF statement setting an expire * (possibly in the past) and wait for an explicit DEL from the master. */ - if (milliseconds <= mstime() && !server.loading && !server.masterhost) { + if (when <= mstime() && !server.loading && !server.masterhost) { robj *aux; redisAssertWithInfo(c,key,dbDelete(c->db,key)); @@ -543,7 +550,7 @@ void expireGenericCommand(redisClient *c, long long offset, int unit) { addReply(c, shared.cone); return; } else { - setExpire(c->db,key,milliseconds); + setExpire(c->db,key,when); addReply(c,shared.cone); signalModifiedKey(c->db,key); server.dirty++; From a3ec16f0c2f4400bc9da1f47184ff3fd83ee5f0e Mon Sep 17 00:00:00 2001 From: jokea Date: Fri, 6 Jan 2012 18:56:07 +0800 Subject: [PATCH 0103/1752] implement aeWait using poll(2). Fixes issue #267. --- src/ae.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/ae.c b/src/ae.c index 4099b12598f..668277a7808 100644 --- a/src/ae.c +++ b/src/ae.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include "ae.h" @@ -369,21 +370,17 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) /* Wait for millseconds until the given file descriptor becomes * writable/readable/exception */ int aeWait(int fd, int mask, long long milliseconds) { - struct timeval tv; - fd_set rfds, wfds, efds; + struct pollfd pfd; int retmask = 0, retval; - tv.tv_sec = milliseconds/1000; - tv.tv_usec = (milliseconds%1000)*1000; - FD_ZERO(&rfds); - FD_ZERO(&wfds); - FD_ZERO(&efds); - - if (mask & AE_READABLE) FD_SET(fd,&rfds); - if (mask & AE_WRITABLE) FD_SET(fd,&wfds); - if ((retval = select(fd+1, &rfds, &wfds, &efds, &tv)) > 0) { - if (FD_ISSET(fd,&rfds)) retmask |= AE_READABLE; - if (FD_ISSET(fd,&wfds)) retmask |= AE_WRITABLE; + memset(&pfd, 0, sizeof(pfd)); + pfd.fd = fd; + if (mask & AE_READABLE) pfd.events |= POLLIN; + if (mask & AE_WRITABLE) pfd.events |= POLLOUT; + + if ((retval = poll(&pfd, 1, milliseconds))== 1) { + if (pfd.revents & POLLIN) retmask |= AE_READABLE; + if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; return retmask; } else { return retval; From 8382453ecfa3a3e98a420363d8962590223d16ed Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Apr 2012 12:27:31 +0200 Subject: [PATCH 0104/1752] version bumped to 2.5.5 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 99f6ffb2e22..f1646ec79e7 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.4" +#define REDIS_VERSION "2.5.5" From 8d3f5ce9dbb7d5e349f457f5909830a35318923f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Apr 2012 21:12:50 +0200 Subject: [PATCH 0105/1752] New INFO field in persistence section: bgrewriteaof_scheduled. --- src/redis.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index a6931876383..14812cf0832 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1793,14 +1793,16 @@ sds genRedisInfoString(char *section) { "bgsave_in_progress:%d\r\n" "last_save_time:%ld\r\n" "last_bgsave_status:%s\r\n" - "bgrewriteaof_in_progress:%d\r\n", + "bgrewriteaof_in_progress:%d\r\n" + "bgrewriteaof_scheduled:%d\r\n", server.loading, server.aof_state != REDIS_AOF_OFF, server.dirty, server.rdb_child_pid != -1, server.lastsave, server.lastbgsave_status == REDIS_OK ? "ok" : "err", - server.aof_child_pid != -1); + server.aof_child_pid != -1, + server.aof_rewrite_scheduled); if (server.aof_state != REDIS_AOF_OFF) { info = sdscatprintf(info, From 3984108474e308994778c8b046cbfd9882fe1fa9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Apr 2012 23:52:28 +0200 Subject: [PATCH 0106/1752] redis.tcl: no longer leave unread replies if an error happens during a MULTI/EXEC block. --- tests/support/redis.tcl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index ca6cf34b601..99415b6409e 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -142,9 +142,15 @@ proc ::redis::redis_multi_bulk_read fd { set count [redis_read_line $fd] if {$count == -1} return {} set l {} + set err {} for {set i 0} {$i < $count} {incr i} { - lappend l [redis_read_reply $fd] + if {[catch { + lappend l [redis_read_reply $fd] + } e] && $err eq {}} { + set err $e + } } + if {$err ne {}} {return -code error $err} return $l } From eb6bc2e0475f66012f744a04d0957f0cf8fd7641 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Apr 2012 23:52:53 +0200 Subject: [PATCH 0107/1752] Two new tests for BGREWRTIEAOF. Check for scheduled rewrite if a BGSAVAE is in progress. Check for error if a rewrite is already in progress. --- tests/unit/aofrw.tcl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index 358266ef780..44192ac1eb4 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -104,4 +104,30 @@ start_server {tags {"aofrw"}} { } } } + + test {BGREWRITEAOF is delayed if BGSAVE is in progress} { + r multi + r bgsave + r bgrewriteaof + r info persistence + set res [r exec] + assert_match {*scheduled*} [lindex $res 1] + assert_match {*bgrewriteaof_scheduled:1*} [lindex $res 2] + while {[string match {*bgrewriteaof_scheduled:1*} [r info persistence]]} { + after 100 + } + } + + test {BGREWRITEAOF is refused if already in progress} { + catch { + r multi + r bgrewriteaof + r bgrewriteaof + r exec + } e + assert_match {*ERR*already*} $e + while {[string match {*bgrewriteaof_scheduled:1*} [r info persistence]]} { + after 100 + } + } } From 7dc1d2ba17588ede178e44af716ef2933f260719 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Apr 2012 02:03:29 +0200 Subject: [PATCH 0108/1752] Removed dead code: function rdbSaveTime() is no longer used since RDB now saves expires in milliseconds. --- src/rdb.c | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 481efe9de8b..6736d8fcb10 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -26,11 +26,6 @@ int rdbLoadType(rio *rdb) { return type; } -int rdbSaveTime(rio *rdb, time_t t) { - int32_t t32 = (int32_t) t; - return rdbWriteRaw(rdb,&t32,4); -} - time_t rdbLoadTime(rio *rdb) { int32_t t32; if (rioRead(rdb,&t32,4) == 0) return -1; From 4f0bd607d95b130bf8d6b3906a91827a82afd7f9 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Apr 2012 02:10:48 +0200 Subject: [PATCH 0109/1752] Never used function stringObjectEqualsMs() removed. --- src/db.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/db.c b/src/db.c index 943878b4074..3c75f6bad61 100644 --- a/src/db.c +++ b/src/db.c @@ -499,14 +499,6 @@ int expireIfNeeded(redisDb *db, robj *key) { * Expires Commands *----------------------------------------------------------------------------*/ -/* Given an string object return true if it contains exactly the "ms" - * or "MS" string. This is used in order to check if the last argument - * of EXPIRE, EXPIREAT or TTL is "ms" to switch into millisecond input/output */ -int stringObjectEqualsMs(robj *a) { - char *arg = a->ptr; - return tolower(arg[0]) == 'm' && tolower(arg[1]) == 's' && arg[2] == '\0'; -} - /* This is the generic command implementation for EXPIRE, PEXPIRE, EXPIREAT * and PEXPIREAT. Because the commad second argument may be relative or absolute * the "basetime" argument is used to signal what the base time is (either 0 From b162e6f133d66f7b051badfc6d4c163ad9f97fe5 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Apr 2012 11:14:52 +0200 Subject: [PATCH 0110/1752] New client info field added to CLIENT LIST output: multi, containing the length of the current pipeline. Test modified accordingly. --- src/networking.c | 3 ++- tests/unit/introspection.tcl | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 75f6cf2213c..f922e297513 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1085,7 +1085,7 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatprintf(sdsempty(), - "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", + "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", ip,port,client->fd, (long)(server.unixtime - client->ctime), (long)(server.unixtime - client->lastinteraction), @@ -1093,6 +1093,7 @@ sds getClientInfoString(redisClient *client) { client->db->id, (int) dictSize(client->pubsub_channels), (int) listLength(client->pubsub_patterns), + (client->flags & REDIS_MULTI) ? client->mstate.count : -1, (unsigned long) sdslen(client->querybuf), (unsigned long) sdsavail(client->querybuf), (unsigned long) client->bufpos, diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index a768e2ab98b..773df112700 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -1,5 +1,5 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list - } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} + } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} } From 2cf3f071a5362f1c2271ba87652c7d9980f9774d Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Apr 2012 11:26:24 +0200 Subject: [PATCH 0111/1752] Tests for MONITOR. --- tests/unit/introspection.tcl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 773df112700..9db0395a2bc 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -2,4 +2,21 @@ start_server {tags {"introspection"}} { test {CLIENT LIST} { r client list } {*addr=*:* fd=* age=* idle=* flags=N db=9 sub=0 psub=0 multi=-1 qbuf=0 qbuf-free=* obl=0 oll=0 omem=0 events=r cmd=client*} + + test {MONITOR can log executed commands} { + set rd [redis_deferring_client] + $rd monitor + r set foo bar + r get foo + list [$rd read] [$rd read] [$rd read] + } {*OK*"set" "foo"*"get" "foo"*} + + test {MONITOR can log commands issued by the scripting engine} { + set rd [redis_deferring_client] + $rd monitor + r eval {redis.call('set',KEYS[1],ARGV[1])} 1 foo bar + $rd read ;# Discard the OK + assert_match {*eval*} [$rd read] + assert_match {*lua*"set"*"foo"*"bar"*} [$rd read] + } } From 55951f900590cc637c67cc820400fd94e8e5bf7f Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Apr 2012 12:11:23 +0200 Subject: [PATCH 0112/1752] For coverage testing use exit() instead of _exit() when termiating saving children. --- src/Makefile | 2 +- src/aof.c | 4 ++-- src/rdb.c | 2 +- src/redis.c | 12 ++++++++++++ src/redis.h | 1 + 5 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index 11f270b9d1e..8116ee2be06 100644 --- a/src/Makefile +++ b/src/Makefile @@ -263,7 +263,7 @@ gprof: $(MAKE) PROF="-pg" gcov: - $(MAKE) PROF="-fprofile-arcs -ftest-coverage" + $(MAKE) PROF="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" noopt: $(MAKE) OPTIMIZATION="" diff --git a/src/aof.c b/src/aof.c index 3f7cd10ffb8..115da29bae2 100644 --- a/src/aof.c +++ b/src/aof.c @@ -803,9 +803,9 @@ int rewriteAppendOnlyFileBackground(void) { if (server.sofd > 0) close(server.sofd); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) { - _exit(0); + exitFromChild(0); } else { - _exit(1); + exitFromChild(1); } } else { /* Parent */ diff --git a/src/rdb.c b/src/rdb.c index 6736d8fcb10..abb966e0281 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -686,7 +686,7 @@ int rdbSaveBackground(char *filename) { if (server.ipfd > 0) close(server.ipfd); if (server.sofd > 0) close(server.sofd); retval = rdbSave(filename); - _exit((retval == REDIS_OK) ? 0 : 1); + exitFromChild((retval == REDIS_OK) ? 0 : 1); } else { /* Parent */ server.stat_fork_time = ustime()-start; diff --git a/src/redis.c b/src/redis.c index 14812cf0832..c79f49d41a2 100644 --- a/src/redis.c +++ b/src/redis.c @@ -354,6 +354,18 @@ long long mstime(void) { return ustime()/1000; } +/* After an RDB dump or AOF rewrite we exit from children using _exit() instead of + * exit(), because the latter may interact with the same file objects used by + * the parent process. However if we are testing the coverage normal exit() is + * used in order to obtain the right coverage information. */ +void exitFromChild(int retcode) { +#ifdef COVERAGE_TEST + exit(retcode); +#else + _exit(retcode); +#endif +} + /*====================== Hash table type implementation ==================== */ /* This is an hash table type that uses the SDS dynamic strings libary as diff --git a/src/redis.h b/src/redis.h index 9d0e19ff2c5..e86823db835 100644 --- a/src/redis.h +++ b/src/redis.h @@ -694,6 +694,7 @@ long long ustime(void); long long mstime(void); void getRandomHexChars(char *p, unsigned int len); uint64_t crc64(const unsigned char *s, uint64_t l); +void exitFromChild(int retcode); /* networking.c -- Networking and Client related operations */ redisClient *createClient(int fd); From 08211b25d3ddda62a0c542668ad404a0eff4b29f Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Apr 2012 13:22:04 +0200 Subject: [PATCH 0113/1752] Added new test to check that "CONFIG appendonly no" actually kills the background AOF operation in progress if any. --- tests/unit/aofrw.tcl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index 44192ac1eb4..e341d77b215 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -1,4 +1,15 @@ start_server {tags {"aofrw"}} { + + test {Turning off AOF kills the background writing child if any} { + r config set appendonly yes + waitForBgrewriteaof r + r multi + r bgrewriteaof + r config set appendonly no + r exec + set result [exec cat [srv 0 stdout] | tail -n1] + } {*Killing*AOF*child*} + foreach d {string int} { foreach e {ziplist linkedlist} { test "AOF rewrite of list with $e encoding, $d data" { From 9ba4d5a3bb48bbd7ef8aebe05343035ad2063a88 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Apr 2012 11:11:00 +0200 Subject: [PATCH 0114/1752] rio.c file somewhat documented so that the casual reader can understand what's going on without reading the code. --- src/rio.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/rio.c b/src/rio.c index 95b1ee7e604..5a745840609 100644 --- a/src/rio.c +++ b/src/rio.c @@ -1,3 +1,17 @@ +/* rio.c is a simple stream-oriented I/O abstraction that provides an interface + * to write code that can consume/produce data using different concrete input + * and output devices. For instance the same rdb.c code using the rio abstraction + * can be used to read and write the RDB format using in-memory buffers or files. + * + * A rio object provides the following methods: + * read: read from stream. + * write: write to stream. + * tell: get the current offset. + * + * It is also possible to set a 'checksum' method that is used by rio.c in order + * to compute a checksum of the data written or read, or to query the rio object + * for the current checksum. */ + #include "fmacros.h" #include #include @@ -65,6 +79,10 @@ void rioInitWithBuffer(rio *r, sds s) { r->io.buffer.pos = 0; } +/* ------------------------------ Higher level interface --------------------------- + * The following higher level functions use lower level rio.c functions to help + * generating the Redis protocol for the Append Only File. */ + /* Write multi bulk count in the format: "*\r\n". */ size_t rioWriteBulkCount(rio *r, char prefix, int count) { char cbuf[128]; From bb99f42596bb0f5c90416c54f6b9f1960cda71ed Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Apr 2012 12:20:47 +0200 Subject: [PATCH 0115/1752] crc64.c modified for incremental computation. --- src/crc64.c | 5 ++--- src/redis.h | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/crc64.c b/src/crc64.c index f2ea8d4ae4c..ecdba90e045 100644 --- a/src/crc64.c +++ b/src/crc64.c @@ -170,8 +170,7 @@ static const uint64_t crc64_tab[256] = { UINT64_C(0x536fa08fdfd90e51), UINT64_C(0x29b7d047efec8728), }; -uint64_t crc64(const unsigned char *s, uint64_t l) { - uint64_t crc = 0; +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l) { uint64_t j; for (j = 0; j < l; j++) { @@ -186,7 +185,7 @@ uint64_t crc64(const unsigned char *s, uint64_t l) { #include int main(void) { printf("e9c6d914c4b8d9ca == %016llx\n", - (unsigned long long) crc64((unsigned char*)"123456789",9)); + (unsigned long long) crc64(0,(unsigned char*)"123456789",9)); return 0; } #endif diff --git a/src/redis.h b/src/redis.h index e86823db835..d8595fe2468 100644 --- a/src/redis.h +++ b/src/redis.h @@ -693,7 +693,7 @@ extern dictType hashDictType; long long ustime(void); long long mstime(void); void getRandomHexChars(char *p, unsigned int len); -uint64_t crc64(const unsigned char *s, uint64_t l); +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); void exitFromChild(int retcode); /* networking.c -- Networking and Client related operations */ From b4b923b04bb06e1026ad93389856b47e5eb25de7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Apr 2012 12:33:09 +0200 Subject: [PATCH 0116/1752] Add checksum computation to rio.c --- src/rdb.c | 4 ++-- src/rio.c | 10 ++++++++++ src/rio.h | 33 +++++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index abb966e0281..3f1e225208a 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -717,7 +717,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { size_t len; unsigned int i; - redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",rdbtype,rdb->tell(rdb)); + redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",rdbtype,rdbTell(rdb)); if (rdbtype == REDIS_RDB_TYPE_STRING) { /* Read string value */ if ((o = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; @@ -1039,7 +1039,7 @@ int rdbLoad(char *filename) { /* Serve the clients from time to time */ if (!(loops++ % 1000)) { - loadingProgress(rdb.tell(&rdb)); + loadingProgress(rdbTell(&rdb)); aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); } diff --git a/src/rio.c b/src/rio.c index 5a745840609..bb977c740cc 100644 --- a/src/rio.c +++ b/src/rio.c @@ -58,6 +58,8 @@ static const rio rioBufferIO = { rioBufferRead, rioBufferWrite, rioBufferTell, + NULL, /* update_checksum */ + 0, /* current checksum */ { { NULL, 0 } } /* union for io-specific vars */ }; @@ -65,6 +67,8 @@ static const rio rioFileIO = { rioFileRead, rioFileWrite, rioFileTell, + NULL, /* update_checksum */ + 0, /* current checksum */ { { NULL, 0 } } /* union for io-specific vars */ }; @@ -79,6 +83,12 @@ void rioInitWithBuffer(rio *r, sds s) { r->io.buffer.pos = 0; } +/* This function can be installed both in memory and file streams when checksum + * computation is needed. */ +void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len) { + r->checksum = crc64(r->checksum,buf,len); +} + /* ------------------------------ Higher level interface --------------------------- * The following higher level functions use lower level rio.c functions to help * generating the Redis protocol for the Append Only File. */ diff --git a/src/rio.h b/src/rio.h index 2a830eb575a..31746303ca8 100644 --- a/src/rio.h +++ b/src/rio.h @@ -2,6 +2,7 @@ #define __REDIS_RIO_H #include +#include #include "sds.h" struct _rio { @@ -11,6 +12,14 @@ struct _rio { size_t (*read)(struct _rio *, void *buf, size_t len); size_t (*write)(struct _rio *, const void *buf, size_t len); off_t (*tell)(struct _rio *); + /* The update_cksum method if not NULL is used to compute the checksum of all the + * data that was read or written so far. The method should be designed so that + * can be called with the current checksum, and the buf and len fields pointing + * to the new block of data to add to the checksum computation. */ + void (*update_cksum)(struct _rio *, void *buf, size_t len); + + /* The current checksum */ + uint64_t cksum; /* Backend-specific vars. */ union { @@ -26,8 +35,26 @@ struct _rio { typedef struct _rio rio; -#define rioWrite(rio,buf,len) ((rio)->write((rio),(buf),(len))) -#define rioRead(rio,buf,len) ((rio)->read((rio),(buf),(len))) +/* The following functions are our interface with the stream. They'll call the + * actual implementation of read / write / tell, and will update the checksum + * if needed. */ + +inline size_t rioWrite(rio *r, const void *buf, size_t len) { + if (r->udpate_cksum) r->update_cksum(r,buf,len); + return r->write(r,buf,len); +} + +inline size_t rioRead(rio *r, void *buf, size_t len) { + if (r->read(r,buf,len) == 1) { + if (r->udpate_cksum) r->update_cksum(r,buf,len); + return 1; + } + return 0; +} + +inline off_t rioTell(rio *r) { + return r->tell(r); +} void rioInitWithFile(rio *r, FILE *fp); void rioInitWithBuffer(rio *r, sds s); @@ -37,4 +64,6 @@ size_t rioWriteBulkString(rio *r, const char *buf, size_t len); size_t rioWriteBulkLongLong(rio *r, long long l); size_t rioWriteBulkDouble(rio *r, double d); +void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len); + #endif From 1bcb45d1187bbe601b9e14eb5d4cdbd4b5ae3961 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Apr 2012 12:36:44 +0200 Subject: [PATCH 0117/1752] Fixed compilation of new rio.c changes (typos and so forth.) --- src/rdb.c | 4 ++-- src/rio.c | 4 +++- src/rio.h | 6 +++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 3f1e225208a..3667f279efe 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -717,7 +717,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { size_t len; unsigned int i; - redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",rdbtype,rdbTell(rdb)); + redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",rdbtype,rioTell(rdb)); if (rdbtype == REDIS_RDB_TYPE_STRING) { /* Read string value */ if ((o = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; @@ -1039,7 +1039,7 @@ int rdbLoad(char *filename) { /* Serve the clients from time to time */ if (!(loops++ % 1000)) { - loadingProgress(rdbTell(&rdb)); + loadingProgress(rioTell(&rdb)); aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); } diff --git a/src/rio.c b/src/rio.c index bb977c740cc..44165d71565 100644 --- a/src/rio.c +++ b/src/rio.c @@ -18,6 +18,8 @@ #include "rio.h" #include "util.h" +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); + /* Returns 1 or 0 for success/failure. */ static size_t rioBufferWrite(rio *r, const void *buf, size_t len) { r->io.buffer.ptr = sdscatlen(r->io.buffer.ptr,(char*)buf,len); @@ -86,7 +88,7 @@ void rioInitWithBuffer(rio *r, sds s) { /* This function can be installed both in memory and file streams when checksum * computation is needed. */ void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len) { - r->checksum = crc64(r->checksum,buf,len); + r->cksum = crc64(r->cksum,buf,len); } /* ------------------------------ Higher level interface --------------------------- diff --git a/src/rio.h b/src/rio.h index 31746303ca8..9012856ffa9 100644 --- a/src/rio.h +++ b/src/rio.h @@ -16,7 +16,7 @@ struct _rio { * data that was read or written so far. The method should be designed so that * can be called with the current checksum, and the buf and len fields pointing * to the new block of data to add to the checksum computation. */ - void (*update_cksum)(struct _rio *, void *buf, size_t len); + void (*update_cksum)(struct _rio *, const void *buf, size_t len); /* The current checksum */ uint64_t cksum; @@ -40,13 +40,13 @@ typedef struct _rio rio; * if needed. */ inline size_t rioWrite(rio *r, const void *buf, size_t len) { - if (r->udpate_cksum) r->update_cksum(r,buf,len); + if (r->update_cksum) r->update_cksum(r,buf,len); return r->write(r,buf,len); } inline size_t rioRead(rio *r, void *buf, size_t len) { if (r->read(r,buf,len) == 1) { - if (r->udpate_cksum) r->update_cksum(r,buf,len); + if (r->update_cksum) r->update_cksum(r,buf,len); return 1; } return 0; From 7f4f86f427fa81260708cc3e9298046abced5ade Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Apr 2012 22:40:41 +0200 Subject: [PATCH 0118/1752] RDB files now embed a crc64 checksum. Version of RDB bumped to 5. --- src/rdb.c | 25 ++++++++++++++++++++++++- src/rdb.h | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 3667f279efe..1815dc24332 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1,6 +1,7 @@ #include "redis.h" #include "lzf.h" /* LZF compression library */ #include "zipmap.h" +#include "endianconv.h" #include #include @@ -602,6 +603,7 @@ int rdbSave(char *filename) { long long now = mstime(); FILE *fp; rio rdb; + uint64_t cksum; snprintf(tmpfile,256,"temp-%d.rdb", (int) getpid()); fp = fopen(tmpfile,"w"); @@ -612,6 +614,7 @@ int rdbSave(char *filename) { } rioInitWithFile(&rdb,fp); + rdb.update_cksum = rioGenericUpdateChecksum; snprintf(magic,sizeof(magic),"REDIS%04d",REDIS_RDB_VERSION); if (rdbWriteRaw(&rdb,magic,9) == -1) goto werr; @@ -641,9 +644,16 @@ int rdbSave(char *filename) { } dictReleaseIterator(di); } + di = NULL; /* So that we don't release it again on error. */ + /* EOF opcode */ if (rdbSaveType(&rdb,REDIS_RDB_OPCODE_EOF) == -1) goto werr; + /* CRC64 checksum */ + cksum = rdb.cksum; + memrev64ifbe(&cksum); + rioWrite(&rdb,&cksum,8); + /* Make sure data will not remain on the OS's output buffers */ fflush(fp); fsync(fileno(fp)); @@ -1016,6 +1026,7 @@ int rdbLoad(char *filename) { return REDIS_ERR; } rioInitWithFile(&rdb,fp); + rdb.update_cksum = rioGenericUpdateChecksum; if (rioRead(&rdb,buf,9) == 0) goto eoferr; buf[9] = '\0'; if (memcmp(buf,"REDIS",5) != 0) { @@ -1025,7 +1036,7 @@ int rdbLoad(char *filename) { return REDIS_ERR; } rdbver = atoi(buf+5); - if (rdbver < 1 || rdbver > 4) { + if (rdbver < 1 || rdbver > 5) { fclose(fp); redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); errno = EINVAL; @@ -1096,6 +1107,18 @@ int rdbLoad(char *filename) { decrRefCount(key); } + /* Verify the checksum if RDB version is >= 5 */ + if (rdbver >= 5) { + uint64_t cksum, expected = rdb.cksum; + + if (rioRead(&rdb,&cksum,8) == 0) goto eoferr; + memrev64ifbe(&cksum); + if (cksum != expected) { + redisLog(REDIS_WARNING,"Wrong RDB checksum. Aborting now."); + exit(1); + } + } + fclose(fp); stopLoading(); return REDIS_OK; diff --git a/src/rdb.h b/src/rdb.h index 60157ad8740..2be5d9cd11b 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -9,7 +9,7 @@ /* The current RDB version. When the format changes in a way that is no longer * backward compatible this number gets incremented. */ -#define REDIS_RDB_VERSION 4 +#define REDIS_RDB_VERSION 5 /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of From 39d1e350d9fe07babfe0758ce539b085e33206aa Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Apr 2012 15:47:10 +0200 Subject: [PATCH 0119/1752] It is now possible to enable/disable RDB checksum computation from redis.conf or via CONFIG SET/GET. Also CONFIG SET support added for rdbcompression as well. --- redis.conf | 9 +++++++++ src/config.c | 15 +++++++++++++++ src/rdb.c | 15 ++++++++++----- src/redis.c | 1 + src/redis.h | 1 + 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/redis.conf b/redis.conf index 5c16143c7c8..85220b80deb 100644 --- a/redis.conf +++ b/redis.conf @@ -114,6 +114,15 @@ stop-writes-on-bgsave-error yes # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes +# Since verison 5 of RDB a CRC64 checksum is placed at the end of the file. +# This makes the format more resistant to corruption but there is a performance +# hit to pay (around 10%) when saving and loading RDB files, so you can disable it +# for maximum performances. +# +# RDB files created with checksum disabled have a checksum of zero that will +# tell the loading code to skip the check. +rdbchecksum yes + # The filename where to dump the DB dbfilename dump.rdb diff --git a/src/config.c b/src/config.c index 804e251493c..4d35521d742 100644 --- a/src/config.c +++ b/src/config.c @@ -210,6 +210,10 @@ void loadServerConfigFromString(char *config) { if ((server.rdb_compression = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"rdbchecksum") && argc == 2) { + if ((server.rdb_checksum = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"activerehashing") && argc == 2) { if ((server.activerehashing = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; @@ -626,6 +630,16 @@ void configSetCommand(redisClient *c) { enableWatchdog(ll); else disableWatchdog(); + } else if (!strcasecmp(c->argv[2]->ptr,"rdbcompression")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.rdb_compression = yn; + } else if (!strcasecmp(c->argv[2]->ptr,"rdbchecksum")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.rdb_checksum = yn; } else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -727,6 +741,7 @@ void configGetCommand(redisClient *c) { server.stop_writes_on_bgsave_err); config_get_bool_field("daemonize", server.daemonize); config_get_bool_field("rdbcompression", server.rdb_compression); + config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); /* Everything we can't handle with macros follows. */ diff --git a/src/rdb.c b/src/rdb.c index 1815dc24332..8ffd2c28f9d 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -614,7 +614,8 @@ int rdbSave(char *filename) { } rioInitWithFile(&rdb,fp); - rdb.update_cksum = rioGenericUpdateChecksum; + if (server.rdb_checksum) + rdb.update_cksum = rioGenericUpdateChecksum; snprintf(magic,sizeof(magic),"REDIS%04d",REDIS_RDB_VERSION); if (rdbWriteRaw(&rdb,magic,9) == -1) goto werr; @@ -649,7 +650,8 @@ int rdbSave(char *filename) { /* EOF opcode */ if (rdbSaveType(&rdb,REDIS_RDB_OPCODE_EOF) == -1) goto werr; - /* CRC64 checksum */ + /* CRC64 checksum. It will be zero if checksum computation is disabled, the + * loading code skips the check in this case. */ cksum = rdb.cksum; memrev64ifbe(&cksum); rioWrite(&rdb,&cksum,8); @@ -1026,7 +1028,8 @@ int rdbLoad(char *filename) { return REDIS_ERR; } rioInitWithFile(&rdb,fp); - rdb.update_cksum = rioGenericUpdateChecksum; + if (server.rdb_checksum) + rdb.update_cksum = rioGenericUpdateChecksum; if (rioRead(&rdb,buf,9) == 0) goto eoferr; buf[9] = '\0'; if (memcmp(buf,"REDIS",5) != 0) { @@ -1108,12 +1111,14 @@ int rdbLoad(char *filename) { decrRefCount(key); } /* Verify the checksum if RDB version is >= 5 */ - if (rdbver >= 5) { + if (rdbver >= 5 && server.rdb_checksum) { uint64_t cksum, expected = rdb.cksum; if (rioRead(&rdb,&cksum,8) == 0) goto eoferr; memrev64ifbe(&cksum); - if (cksum != expected) { + if (cksum == 0) { + redisLog(REDIS_WARNING,"RDB file was saved with checksum disabled: no check performed."); + } else if (cksum != expected) { redisLog(REDIS_WARNING,"Wrong RDB checksum. Aborting now."); exit(1); } diff --git a/src/redis.c b/src/redis.c index c79f49d41a2..43487bc8ee3 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1045,6 +1045,7 @@ void initServerConfig() { server.aof_filename = zstrdup("appendonly.aof"); server.requirepass = NULL; server.rdb_compression = 1; + server.rdb_checksum = 1; server.activerehashing = 1; server.maxclients = REDIS_MAX_CLIENTS; server.bpop_blocked_clients = 0; diff --git a/src/redis.h b/src/redis.h index d8595fe2468..9702010e2de 100644 --- a/src/redis.h +++ b/src/redis.h @@ -517,6 +517,7 @@ struct redisServer { int saveparamslen; /* Number of saving points */ char *rdb_filename; /* Name of RDB file */ int rdb_compression; /* Use compression in RDB? */ + int rdb_checksum; /* Use RDB checksum? */ time_t lastsave; /* Unix time of last save succeeede */ int lastbgsave_status; /* REDIS_OK or REDIS_ERR */ int stop_writes_on_bgsave_err; /* Don't allow writes if can't BGSAVE */ From e95740392b54635ec83994261af021015afc6551 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Apr 2012 12:33:57 +0200 Subject: [PATCH 0120/1752] dump/restore fixed to use the new crc64 API. --- src/migrate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index aeb412b59b2..d3b92859cfe 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -30,7 +30,7 @@ void createDumpPayload(rio *payload, robj *o) { payload->io.buffer.ptr = sdscatlen(payload->io.buffer.ptr,buf,2); /* CRC64 */ - crc = crc64((unsigned char*)payload->io.buffer.ptr, + crc = crc64(0,(unsigned char*)payload->io.buffer.ptr, sdslen(payload->io.buffer.ptr)); memrev64ifbe(&crc); payload->io.buffer.ptr = sdscatlen(payload->io.buffer.ptr,&crc,8); @@ -54,7 +54,7 @@ int verifyDumpPayload(unsigned char *p, size_t len) { if (rdbver != REDIS_RDB_VERSION) return REDIS_ERR; /* Verify CRC64 */ - crc = crc64(p,len-8); + crc = crc64(0,p,len-8); memrev64ifbe(&crc); return (memcmp(&crc,footer+2,8) == 0) ? REDIS_OK : REDIS_ERR; } From fdf8bd4025980b2202cdfc7e19dcd238bc104a7d Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 8 Apr 2012 11:16:40 +0200 Subject: [PATCH 0121/1752] Test for maxclients. --- tests/test_helper.tcl | 1 + tests/unit/limits.tcl | 13 +++++++++++++ 2 files changed, 14 insertions(+) create mode 100644 tests/unit/limits.tcl diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 2ec3aad1c16..598a3929165 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -38,6 +38,7 @@ set ::all_tests { unit/scripting unit/maxmemory unit/introspection + unit/limits unit/obuf-limits unit/dump } diff --git a/tests/unit/limits.tcl b/tests/unit/limits.tcl new file mode 100644 index 00000000000..9988983fad7 --- /dev/null +++ b/tests/unit/limits.tcl @@ -0,0 +1,13 @@ +start_server {tags {"limits"} overrides {maxclients 10}} { + test {Check if maxclients works refusing connections} { + set c 0 + catch { + while 1 { + incr c + redis_deferring_client + } + } e + assert {$c > 8 && $c <= 10} + set e + } {*ERR max*reached*} +} From 3f64694e7161075bbb48135379da205619c9b034 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Apr 2012 16:34:51 +0200 Subject: [PATCH 0122/1752] Version 2.5.6. --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index f1646ec79e7..d25a2e1d8a5 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.5" +#define REDIS_VERSION "2.5.6" From b9aa332843416653e314a18a00e1150906ce3230 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Apr 2012 16:48:28 +0200 Subject: [PATCH 0123/1752] Check write(2) return value to avoid warnings, because in this context failing write is not critical. --- src/redis.c | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/redis.c b/src/redis.c index 43487bc8ee3..bfac279f490 100644 --- a/src/redis.c +++ b/src/redis.c @@ -316,14 +316,15 @@ void redisLogFromHandler(int level, const char *msg) { STDOUT_FILENO; if (fd == -1) return; ll2string(buf,sizeof(buf),getpid()); - write(fd,"[",1); - write(fd,buf,strlen(buf)); - write(fd," | signal handler] (",20); + if (write(fd,"[",1) == -1) goto err; + if (write(fd,buf,strlen(buf)) == -1) goto err; + if (write(fd," | signal handler] (",20) == -1) goto err; ll2string(buf,sizeof(buf),time(NULL)); - write(fd,buf,strlen(buf)); - write(fd,") ",2); - write(fd,msg,strlen(msg)); - write(fd,"\n",1); + if (write(fd,buf,strlen(buf)) == -1) goto err; + if (write(fd,") ",2) == -1) goto err; + if (write(fd,msg,strlen(msg)) == -1) goto err; + if (write(fd,"\n",1) == -1) goto err; +err: if (server.logfile) close(fd); } From 3ba5eab774b151ef838130e4f24f095bf5ac0208 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Apr 2012 16:46:29 +0200 Subject: [PATCH 0124/1752] Minor MIGRATE implementation simplification about ttl handling. --- src/migrate.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index d3b92859cfe..94aa1e40db6 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -130,7 +130,7 @@ void migrateCommand(redisClient *c) { int fd; long timeout; long dbid; - long long ttl, expireat; + long long ttl = 0, expireat; robj *o; rio cmd, payload; @@ -177,7 +177,7 @@ void migrateCommand(redisClient *c) { redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,"RESTORE",7)); redisAssertWithInfo(c,NULL,c->argv[3]->encoding == REDIS_ENCODING_RAW); redisAssertWithInfo(c,NULL,rioWriteBulkString(&cmd,c->argv[3]->ptr,sdslen(c->argv[3]->ptr))); - redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,(expireat==-1) ? 0 : ttl)); + redisAssertWithInfo(c,NULL,rioWriteBulkLongLong(&cmd,ttl)); /* Finally the last argument that is the serailized object payload * in the DUMP format. */ From e74bec56fab1c8e5cbf86c61a70aa153f85f1bc8 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Sat, 24 Mar 2012 19:33:08 -0700 Subject: [PATCH 0125/1752] Everything x86 is little endian --- src/config.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/config.h b/src/config.h index 6a69364a971..136fd40c4c8 100644 --- a/src/config.h +++ b/src/config.h @@ -59,10 +59,11 @@ #define BIG_ENDIAN 4321 /* most-significant byte first (IBM, net) */ #define PDP_ENDIAN 3412 /* LSB first in word, MSW first in long (pdp)*/ -#if defined(vax) || defined(ns32000) || defined(sun386) || defined(__i386__) || \ - defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ - defined(__alpha__) || defined(__alpha) -#define BYTE_ORDER LITTLE_ENDIAN +#if defined(__i386__) || defined(__x86_64__) || defined(__amd64__) || \ + defined(vax) || defined(ns32000) || defined(sun386) || \ + defined(MIPSEL) || defined(_MIPSEL) || defined(BIT_ZERO_ON_RIGHT) || \ + defined(__alpha__) || defined(__alpha) +#define BYTE_ORDER LITTLE_ENDIAN #endif #if defined(sel) || defined(pyr) || defined(mc68000) || defined(sparc) || \ From de07849e0d5e467a00eb47ea5c4c15202e64d46b Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Sat, 24 Mar 2012 19:25:03 -0700 Subject: [PATCH 0126/1752] Clean up Makefiles Remove unused variables. Instead of overriding non-standard variables such as ARCH and PROF, use standard variables CFLAGS and LDFLAGS to override Makefile settings. Move dependencies generated by `make dep` to a separate file. --- deps/Makefile | 85 ++++++++----- deps/linenoise/Makefile | 17 ++- src/Makefile | 274 +++++++++++++++------------------------- src/Makefile.dep | 83 ++++++++++++ 4 files changed, 253 insertions(+), 206 deletions(-) create mode 100644 src/Makefile.dep diff --git a/deps/Makefile b/deps/Makefile index b881c814e7d..7cd9c0f6445 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -1,17 +1,6 @@ # Redis dependency Makefile -UNAME_S:=$(shell sh -c 'uname -s 2> /dev/null || echo not') - -LUA_CFLAGS=-O2 -Wall $(ARCH) -ifeq ($(UNAME_S),SunOS) - # Make isinf() available - LUA_CFLAGS+= -D__C99FEATURES__=1 -endif - -JEMALLOC_CFLAGS= -ifeq ($(ARCH),-m32) - JEMALLOC_CFLAGS+=CFLAGS="-std=gnu99 -Wall -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -m32" -endif +uname_S:= $(shell sh -c 'uname -s 2>/dev/null || echo not') CCCOLOR="\033[34m" LINKCOLOR="\033[34;1m" @@ -23,37 +12,67 @@ ENDCOLOR="\033[0m" default: @echo "Explicit target required" -# Clean everything when ARCH is different -ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) -.make-arch: distclean -else -.make-arch: +.PHONY: default + +# Prerequisites target +.make-prerequisites: + @touch $@ + +# Clean everything when CFLAGS is different +ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(CFLAGS)) +.make-cflags: distclean + -(echo "$(CFLAGS)" > .make-cflags) +.make-prerequisites: .make-cflags endif -.make-arch: - -(echo $(ARCH) > .make-arch) +# Clean everything when LDFLAGS is different +ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(LDFLAGS)) +.make-ldflags: distclean + -(echo "$(LDFLAGS)" > .make-ldflags) +.make-prerequisites: .make-ldflags +endif distclean: -(cd hiredis && $(MAKE) clean) > /dev/null || true -(cd linenoise && $(MAKE) clean) > /dev/null || true -(cd lua && $(MAKE) clean) > /dev/null || true -(cd jemalloc && [ -f Makefile ] && $(MAKE) distclean) > /dev/null || true - -(rm -f .make-arch) + -(rm -f .make-*) + +.PHONY: distclean + +hiredis: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd hiredis && $(MAKE) static + +.PHONY: hiredis + +linenoise: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd linenoise && $(MAKE) + +.PHONY: linenoise + +ifeq ($(uname_S),SunOS) + # Make isinf() available + LUA_CFLAGS= -D__C99FEATURES__=1 +endif + +LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI $(CFLAGS) +LUA_LDFLAGS+= $(LDFLAGS) -hiredis: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)hiredis$(ENDCOLOR) - cd hiredis && $(MAKE) static ARCH="$(ARCH)" +lua: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd lua/src && $(MAKE) all CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(LUA_LDFLAGS)" -linenoise: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)linenoise$(ENDCOLOR) - cd linenoise && $(MAKE) ARCH="$(ARCH)" +.PHONY: lua -lua: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)lua$(ENDCOLOR) - cd lua && $(MAKE) CFLAGS="$(LUA_CFLAGS)" MYLDFLAGS="$(ARCH)" ansi +JEMALLOC_CFLAGS= -std=gnu99 -Wall -pipe -g3 -O3 -funroll-loops $(CFLAGS) +JEMALLOC_LDFLAGS= $(LDFLAGS) -jemalloc: .make-arch - @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)jemalloc$(ENDCOLOR) - cd jemalloc && ./configure $(JEMALLOC_CFLAGS) --with-jemalloc-prefix=je_ --enable-cc-silence && $(MAKE) lib/libjemalloc.a +jemalloc: .make-prerequisites + @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) + cd jemalloc && ./configure --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" + cd jemalloc && $(MAKE) lib/libjemalloc.a -.PHONY: default conditional_clean hiredis linenoise lua jemalloc +.PHONY: jemalloc diff --git a/deps/linenoise/Makefile b/deps/linenoise/Makefile index 841f39072ff..1dd894b49e9 100644 --- a/deps/linenoise/Makefile +++ b/deps/linenoise/Makefile @@ -1,10 +1,21 @@ -linenoise_example: linenoise.h linenoise.c +STD= +WARN= -Wall +OPT= -Os + +R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) +R_LDFLAGS= $(LDFLAGS) +DEBUG= -g + +R_CC=$(CC) $(R_CFLAGS) +R_LD=$(CC) $(R_LDFLAGS) + +linenoise.o: linenoise.h linenoise.c linenoise_example: linenoise.o example.o - $(CC) $(ARCH) -Wall -W -Os -g -o linenoise_example linenoise.o example.o + $(R_LD) -o $@ $^ .c.o: - $(CC) $(ARCH) -c -Wall -W -Os -g $< + $(R_CC) -c $< clean: rm -f linenoise_example *.o diff --git a/src/Makefile b/src/Makefile index 8116ee2be06..616842343b7 100644 --- a/src/Makefile +++ b/src/Makefile @@ -7,16 +7,25 @@ uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') OPTIMIZATION?=-O2 DEPENDENCY_TARGETS=hiredis linenoise lua +STD= -std=c99 -pedantic +WARN= -Wall +OPT= $(OPTIMIZATION) + ifeq ($(uname_S),SunOS) - CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W -D__EXTENSIONS__ -D_XPG6 - CCLINK?=-ldl -lnsl -lsocket -lm -lpthread - DEBUG?=-g -ggdb + R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) -D__EXTENSIONS__ -D_XPG6 + R_LDFLAGS= $(LDFLAGS) + R_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread + DEBUG= -g -ggdb else - CFLAGS?=-std=c99 -pedantic $(OPTIMIZATION) -Wall -W $(ARCH) $(PROF) - CCLINK?=-lm -pthread - DEBUG?=-g -rdynamic -ggdb + R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) + R_LDFLAGS= $(LDFLAGS) + R_LIBS= $(LIBS) -lm -pthread + DEBUG= -g -rdynamic -ggdb endif +# Include paths to dependencies +R_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src + # Default allocator ifeq ($(uname_S),Linux) MALLOC?=jemalloc @@ -38,24 +47,23 @@ ifeq ($(USE_JEMALLOC),yes) endif ifeq ($(MALLOC),tcmalloc) - ALLOC_LINK=-ltcmalloc - ALLOC_FLAGS=-DUSE_TCMALLOC + R_CFLAGS+= -DUSE_TCMALLOC + R_LIBS+= -ltcmalloc endif ifeq ($(MALLOC),tcmalloc_minimal) - ALLOC_LINK=-ltcmalloc_minimal - ALLOC_FLAGS=-DUSE_TCMALLOC + R_CFLAGS+= -DUSE_TCMALLOC + R_LIBS+= -ltcmalloc_minimal endif ifeq ($(MALLOC),jemalloc) - ALLOC_LINK=../deps/jemalloc/lib/libjemalloc.a -ldl - ALLOC_FLAGS=-DUSE_JEMALLOC -I../deps/jemalloc/include DEPENDENCY_TARGETS+= jemalloc + R_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include + R_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl endif -CCLINK+= $(ALLOC_LINK) -CFLAGS+= $(ALLOC_FLAGS) -CCOPT= $(CFLAGS) $(ARCH) $(PROF) +R_CC=$(QUIET_CC)$(CC) $(R_CFLAGS) +R_LD=$(QUIET_LINK)$(CC) $(R_LDFLAGS) PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin @@ -69,176 +77,100 @@ MAKECOLOR="\033[32;1m" ENDCOLOR="\033[0m" ifndef V -QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR); -QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR); +QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR) 1>&2; +QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -OBJ = adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o -BENCHOBJ = ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o -CLIOBJ = anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o -CHECKDUMPOBJ = redis-check-dump.o lzf_c.o lzf_d.o -CHECKAOFOBJ = redis-check-aof.o - -PRGNAME = redis-server -BENCHPRGNAME = redis-benchmark -CLIPRGNAME = redis-cli -CHECKDUMPPRGNAME = redis-check-dump -CHECKAOFPRGNAME = redis-check-aof - -all: redis-benchmark redis-cli redis-check-dump redis-check-aof redis-server +R_SERVER_NAME= redis-server +R_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o +R_CLI_NAME= redis-cli +R_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o +R_BENCHMARK_NAME= redis-benchmark +R_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o +R_CHECK_DUMP_NAME= redis-check-dump +R_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o +R_CHECK_AOF_NAME= redis-check-aof +R_CHECK_AOF_OBJ= redis-check-aof.o + +all: $(R_SERVER_NAME) $(R_CLI_NAME) $(R_BENCHMARK_NAME) $(R_CHECK_DUMP_NAME) $(R_CHECK_AOF_NAME) @echo "" @echo "Hint: To run 'make test' is a good idea ;)" @echo "" -# Deps (use make dep to generate this) -adlist.o: adlist.c adlist.h zmalloc.h -ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c -ae_epoll.o: ae_epoll.c -ae_kqueue.o: ae_kqueue.c -ae_select.o: ae_select.c -anet.o: anet.c fmacros.h anet.h -aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h -bio.o: bio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h -config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h -dict.o: dict.c fmacros.h dict.h zmalloc.h -endianconv.o: endianconv.c -intset.o: intset.c intset.h zmalloc.h endianconv.h -lzf_c.o: lzf_c.c lzfP.h -lzf_d.o: lzf_d.c lzfP.h -migrate.o: migrate.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h -multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h -object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -pqsort.o: pqsort.c -pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -rand.o: rand.c -rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h lzf.h \ - zipmap.h -redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ - ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h -redis-check-aof.o: redis-check-aof.c fmacros.h config.h -redis-check-dump.o: redis-check-dump.c lzf.h -redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ - sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h -redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ - slowlog.h bio.h asciilogo.h -release.o: release.c release.h -replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h -rio.o: rio.c fmacros.h rio.h sds.h util.h -scripting.o: scripting.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h sha1.h rand.h -sds.o: sds.c sds.h zmalloc.h -sha1.o: sha1.c sha1.h config.h -slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h slowlog.h -sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ - pqsort.h -syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h -t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -util.o: util.c fmacros.h util.h -ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h -zipmap.o: zipmap.c zmalloc.h endianconv.h -zmalloc.o: zmalloc.c config.h zmalloc.h - -# Clean local objects when ARCH is different -ifneq ($(shell sh -c '[ -f .make-arch ] && cat .make-arch'), $(ARCH)) -.make-arch: clean -else -.make-arch: -endif +.PHONY: all -.make-arch: - -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS) ARCH="$(ARCH)") - -(echo $(ARCH) > .make-arch) +# Deps (use make dep to generate this) +include Makefile.dep -# Clean local objects when allocator changes -ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc'), $(MALLOC)) -.make-malloc: clean -else -.make-malloc: -endif +dep: + $(R_CC) -MM *.c -.make-malloc: - -(echo $(MALLOC) > .make-malloc) +.PHONY: dep -# Union of make-prerequisites -.make-prerequisites: .make-arch .make-malloc +# Prerequisites target +.make-prerequisites: @touch $@ -redis-server: .make-prerequisites $(OBJ) - $(QUIET_LINK)$(CC) -o $(PRGNAME) $(CCOPT) $(DEBUG) $(OBJ) ../deps/lua/src/liblua.a $(CCLINK) +# Clean local objects and build dependencies when R_CFLAGS is different +ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(R_CFLAGS)) +.make-cflags: clean + -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS)) + -(echo "$(R_CFLAGS)" > .make-cflags) +.make-prerequisites: .make-cflags +endif -redis-benchmark: .make-prerequisites $(BENCHOBJ) - $(QUIET_LINK)$(CC) -o $(BENCHPRGNAME) $(CCOPT) $(DEBUG) $(BENCHOBJ) ../deps/hiredis/libhiredis.a $(CCLINK) +# Clean local objects when R_LDFLAGS is different +ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(R_LDFLAGS)) +.make-ldflags: clean + -(echo "$(R_LDFLAGS)" > .make-ldflags) +.make-prerequisites: .make-ldflags +endif + +# Clean local objects when MALLOC is different +ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc || echo none'), $(MALLOC)) +.make-malloc: clean + -(echo "$(MALLOC)" > .make-malloc) +.make-prerequisites: .make-malloc +endif -redis-benchmark.o: redis-benchmark.c .make-prerequisites - $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis $(DEBUG) $(COMPILE_TIME) $< +# redis-server +$(R_SERVER_NAME): $(R_SERVER_OBJ) + $(R_LD) -o $@ $^ ../deps/lua/src/liblua.a $(R_LIBS) -redis-cli: .make-prerequisites $(CLIOBJ) - $(QUIET_LINK)$(CC) -o $(CLIPRGNAME) $(CCOPT) $(DEBUG) $(CLIOBJ) ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(CCLINK) +# redis-cli +$(R_CLI_NAME): $(R_CLI_OBJ) + $(R_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(R_LIBS) -redis-cli.o: redis-cli.c .make-prerequisites - $(QUIET_CC)$(CC) -c $(CFLAGS) -I../deps/hiredis -I../deps/linenoise $(DEBUG) $(COMPILE_TIME) $< +# redis-benchmark +$(R_BENCHMARK_NAME): $(R_BENCHMARK_OBJ) + $(R_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(R_LIBS) -redis-check-dump: .make-prerequisites $(CHECKDUMPOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKDUMPPRGNAME) $(CCOPT) $(DEBUG) $(CHECKDUMPOBJ) $(CCLINK) +# redis-check-dump +$(R_CHECK_DUMP_NAME): $(R_CHECK_DUMP_OBJ) + $(R_LD) -o $@ $^ $(R_LIBS) -redis-check-aof: .make-prerequisites $(CHECKAOFOBJ) - $(QUIET_LINK)$(CC) -o $(CHECKAOFPRGNAME) $(CCOPT) $(DEBUG) $(CHECKAOFOBJ) $(CCLINK) +# redis-check-aof +$(R_CHECK_AOF_NAME): $(R_CHECK_AOF_OBJ) + $(R_LD) -o $@ $^ $(R_LIBS) -# Because the jemalloc.h header is generated as a part of the jemalloc build -# process, building it should complete before building any other object. Instead of -# depending on a single artifact, simply build all dependencies first. +# Because the jemalloc.h header is generated as a part of the jemalloc build, +# building it should complete before building any other object. Instead of +# depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(QUIET_CC)$(CC) -c $(CFLAGS) $(DEBUG) $(COMPILE_TIME) -I../deps/lua/src $< - -.PHONY: all clean distclean lcov + $(R_CC) -c $< clean: - rm -rf $(PRGNAME) $(BENCHPRGNAME) $(CLIPRGNAME) $(CHECKDUMPPRGNAME) $(CHECKAOFPRGNAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html + rm -rf $(R_SERVER_NAME) $(R_CLI_NAME) $(R_BENCHMARK_NAME) $(R_CHECK_DUMP_NAME) $(R_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html + +.PHONY: clean distclean: clean -(cd ../deps && $(MAKE) distclean) - -(rm -f .make-arch .make-malloc) + -(rm -f .make-*) -dep: - $(CC) -MM *.c -I ../deps/hiredis -I ../deps/linenoise +.PHONY: distclean -test: redis-server redis-check-aof +test: $(R_SERVER_NAME) $(R_CHECK_AOF_NAME) @(cd ..; ./runtest) lcov: @@ -247,8 +179,10 @@ lcov: @geninfo -o redis.info . @genhtml --legend -o lcov-html redis.info -bench: - ./redis-benchmark +.PHONY: lcov + +bench: $(R_BENCHMARK_NAME) + ./$(R_BENCHMARK_NAME) log: git log '--pretty=format:%ad %s (%cn)' --date=short > ../Changelog @@ -257,27 +191,27 @@ log: @echo "" @echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386" @echo "" - $(MAKE) ARCH="-m32" JEMALLOC_CFLAGS='CFLAGS="-std=gnu99 -Wall -pipe -g3 -fvisibility=hidden -O3 -funroll-loops -m32"' + $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" gprof: - $(MAKE) PROF="-pg" + $(MAKE) CFLAGS="$(CFLAGS) -pg" LDFLAGS="$(LDFLAGS) -pg" gcov: - $(MAKE) PROF="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" + $(MAKE) CFLAGS="$(CFLAGS) -fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" noopt: - $(MAKE) OPTIMIZATION="" + $(MAKE) OPT="-O0" 32bitgprof: - $(MAKE) PROF="-pg" ARCH="-arch i386" + $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" gprof src/help.h: @../utils/generate-command-help.rb > help.h install: all mkdir -p $(INSTALL_BIN) - $(INSTALL) $(PRGNAME) $(INSTALL_BIN) - $(INSTALL) $(BENCHPRGNAME) $(INSTALL_BIN) - $(INSTALL) $(CLIPRGNAME) $(INSTALL_BIN) - $(INSTALL) $(CHECKDUMPPRGNAME) $(INSTALL_BIN) - $(INSTALL) $(CHECKAOFPRGNAME) $(INSTALL_BIN) + $(INSTALL) $(R_SERVER_NAME) $(INSTALL_BIN) + $(INSTALL) $(R_BENCHMARK_NAME) $(INSTALL_BIN) + $(INSTALL) $(R_CLI_NAME) $(INSTALL_BIN) + $(INSTALL) $(R_CHECK_DUMP_NAME) $(INSTALL_BIN) + $(INSTALL) $(R_CHECK_AOF_NAME) $(INSTALL_BIN) diff --git a/src/Makefile.dep b/src/Makefile.dep new file mode 100644 index 00000000000..83de32106ab --- /dev/null +++ b/src/Makefile.dep @@ -0,0 +1,83 @@ +adlist.o: adlist.c adlist.h zmalloc.h +ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c +ae_epoll.o: ae_epoll.c +ae_kqueue.o: ae_kqueue.c +ae_select.o: ae_select.c +anet.o: anet.c fmacros.h anet.h +aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h +bio.o: bio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h +config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h +dict.o: dict.c fmacros.h dict.h zmalloc.h +endianconv.o: endianconv.c +intset.o: intset.c intset.h zmalloc.h endianconv.h +lzf_c.o: lzf_c.c lzfP.h +lzf_d.o: lzf_d.c lzfP.h +memtest.o: memtest.c +migrate.o: migrate.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h +multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h +object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +pqsort.o: pqsort.c +pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +rand.o: rand.c +rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h lzf.h \ + zipmap.h +redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ + ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h +redis-check-aof.o: redis-check-aof.c fmacros.h config.h +redis-check-dump.o: redis-check-dump.c lzf.h +redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ + sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h +redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ + slowlog.h bio.h asciilogo.h +release.o: release.c release.h +replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h +rio.o: rio.c fmacros.h rio.h sds.h util.h +scripting.o: scripting.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h sha1.h rand.h +sds.o: sds.c sds.h zmalloc.h +sha1.o: sha1.c sha1.h config.h +slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h slowlog.h +sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ + pqsort.h +syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \ + adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ + rio.h +t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +util.o: util.c fmacros.h util.h +ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h +zipmap.o: zipmap.c zmalloc.h endianconv.h +zmalloc.o: zmalloc.c config.h zmalloc.h From ef278d110f2be68457b1c5e04ee12b556365123f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Apr 2012 11:32:22 +0200 Subject: [PATCH 0127/1752] Macros ULONG_ONEZERO / ULONG_ZEROONE were inverted in #ifdef to test 32/64 bit arch. --- src/memtest.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/memtest.c b/src/memtest.c index 272ec502255..c8b47d489a3 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -16,11 +16,11 @@ #endif #ifdef MEMTEST_32BIT -#define ULONG_ONEZERO 0xaaaaaaaaaaaaaaaaUL -#define ULONG_ZEROONE 0x5555555555555555UL -#else #define ULONG_ONEZERO 0xaaaaaaaaUL #define ULONG_ZEROONE 0x55555555UL +#else +#define ULONG_ONEZERO 0xaaaaaaaaaaaaaaaaUL +#define ULONG_ZEROONE 0x5555555555555555UL #endif static struct winsize ws; From 336ba6a152042245748eb42dae577eef2b1c88d9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Apr 2012 11:58:32 +0200 Subject: [PATCH 0128/1752] Make inline functions rioRead/Write/Tell static. This fixes issue #447. --- src/rio.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/rio.h b/src/rio.h index 9012856ffa9..8befe6676e7 100644 --- a/src/rio.h +++ b/src/rio.h @@ -39,12 +39,12 @@ typedef struct _rio rio; * actual implementation of read / write / tell, and will update the checksum * if needed. */ -inline size_t rioWrite(rio *r, const void *buf, size_t len) { +static inline size_t rioWrite(rio *r, const void *buf, size_t len) { if (r->update_cksum) r->update_cksum(r,buf,len); return r->write(r,buf,len); } -inline size_t rioRead(rio *r, void *buf, size_t len) { +static inline size_t rioRead(rio *r, void *buf, size_t len) { if (r->read(r,buf,len) == 1) { if (r->update_cksum) r->update_cksum(r,buf,len); return 1; @@ -52,7 +52,7 @@ inline size_t rioRead(rio *r, void *buf, size_t len) { return 0; } -inline off_t rioTell(rio *r) { +static inline off_t rioTell(rio *r) { return r->tell(r); } From d5ec38958542890d6d47f1eb4ca202973fb818d8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Apr 2012 12:12:05 +0200 Subject: [PATCH 0129/1752] make dep: redirect output to Makefile.dep. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 616842343b7..a32aa5a382b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -103,7 +103,7 @@ all: $(R_SERVER_NAME) $(R_CLI_NAME) $(R_BENCHMARK_NAME) $(R_CHECK_DUMP_NAME) $(R include Makefile.dep dep: - $(R_CC) -MM *.c + $(R_CC) -MM *.c > Makefile.dep .PHONY: dep From 79e3df9d72cc18ccca2494b4a68cffc28552f6d8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Apr 2012 12:12:30 +0200 Subject: [PATCH 0130/1752] Makefile.dep updated. --- src/Makefile.dep | 123 +++++++++++++++++++++++++++-------------------- 1 file changed, 72 insertions(+), 51 deletions(-) diff --git a/src/Makefile.dep b/src/Makefile.dep index 83de32106ab..a098154ae88 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -4,79 +4,100 @@ ae_epoll.o: ae_epoll.c ae_kqueue.o: ae_kqueue.c ae_select.o: ae_select.c anet.o: anet.c fmacros.h anet.h -aof.o: aof.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h -bio.o: bio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h bio.h -config.o: config.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -crc16.o: crc16.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -db.o: db.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -debug.o: debug.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h +aof.o: aof.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h bio.h +bio.o: bio.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h bio.h +config.o: config.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +crc16.o: crc16.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +crc64.o: crc64.c +db.o: db.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +debug.o: debug.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h dict.o: dict.c fmacros.h dict.h zmalloc.h endianconv.o: endianconv.c intset.o: intset.c intset.h zmalloc.h endianconv.h lzf_c.o: lzf_c.c lzfP.h lzf_d.o: lzf_d.c lzfP.h memtest.o: memtest.c -migrate.o: migrate.c redis.h fmacros.h config.h ae.h sds.h dict.h \ +migrate.o: migrate.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h endianconv.h +multi.o: multi.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +networking.o: networking.c redis.h fmacros.h config.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ rio.h -multi.o: multi.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -networking.o: networking.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h -object.o: object.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +object.o: object.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h pqsort.o: pqsort.c -pubsub.o: pubsub.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +pubsub.o: pubsub.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h rand.o: rand.c -rdb.o: rdb.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h lzf.h \ - zipmap.h +rdb.o: rdb.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h lzf.h zipmap.h \ + endianconv.h redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h redis-check-aof.o: redis-check-aof.c fmacros.h config.h redis-check-dump.o: redis-check-dump.c lzf.h redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h -redis.o: redis.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ - slowlog.h bio.h asciilogo.h +redis.o: redis.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h slowlog.h bio.h \ + asciilogo.h release.o: release.c release.h -replication.o: replication.c redis.h fmacros.h config.h ae.h sds.h dict.h \ +replication.o: replication.c redis.h fmacros.h config.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ rio.h rio.o: rio.c fmacros.h rio.h sds.h util.h -scripting.o: scripting.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h sha1.h rand.h +scripting.o: scripting.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h rand.h \ + ../deps/lua/src/lauxlib.h ../deps/lua/src/lua.h \ + ../deps/lua/src/lualib.h sds.o: sds.c sds.h zmalloc.h sha1.o: sha1.c sha1.h config.h -slowlog.o: slowlog.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h slowlog.h -sort.o: sort.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h \ - pqsort.h -syncio.o: syncio.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_hash.o: t_hash.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_list.o: t_list.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_set.o: t_set.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h -t_string.o: t_string.c redis.h fmacros.h config.h ae.h sds.h dict.h \ - adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ - rio.h -t_zset.o: t_zset.c redis.h fmacros.h config.h ae.h sds.h dict.h adlist.h \ - zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h rio.h +slowlog.o: slowlog.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h slowlog.h +sort.o: sort.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h pqsort.h +syncio.o: syncio.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +t_hash.o: t_hash.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +t_list.o: t_list.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +t_set.o: t_set.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +t_string.o: t_string.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h +t_zset.o: t_zset.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h util.o: util.c fmacros.h util.h ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h zipmap.o: zipmap.c zmalloc.h endianconv.h From b9cd703b5b16e350c0783061ce99c79089bcfc0a Mon Sep 17 00:00:00 2001 From: Erik Dubbelboer Date: Wed, 11 Apr 2012 17:04:31 +0200 Subject: [PATCH 0131/1752] added explanation for the magic 511 backlog number --- src/anet.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/anet.c b/src/anet.c index ba4e6cce89b..434d945c7d8 100644 --- a/src/anet.c +++ b/src/anet.c @@ -262,7 +262,11 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { close(s); return ANET_ERR; } - if (listen(s, 511) == -1) { /* the magic 511 constant is from nginx */ + + /* Use a backlog of 512 entries. We pass 511 to the listen() call because + * the kernel does: backlogsize = roundup_pow_of_two(backlogsize + 1); + * which will thus give us a backlog of 512 entries */ + if (listen(s, 511) == -1) { anetSetError(err, "listen: %s", strerror(errno)); close(s); return ANET_ERR; From 69ac4d063dc91cf78597b64746a39300bc250b9b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Apr 2012 11:09:38 +0200 Subject: [PATCH 0132/1752] Makefile now introduces Redis-specific CFLAGS / LDFLAGS. Gcov target fixed. Added comments to describe how it works. --- src/Makefile | 129 ++++++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/src/Makefile b/src/Makefile index a32aa5a382b..a0e03f3e7b2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -1,6 +1,16 @@ # Redis Makefile # Copyright (C) 2009 Salvatore Sanfilippo # This file is released under the BSD license, see the COPYING file +# +# The Makefile composes the final REDIS_CFLAGS and REDIS_LDFLAGS using +# what is needed for Redis plus the standard CFLAGS and LDFLAGS passed. +# However when building the dependencies (Jemalloc, Lua, Hiredis, ...) +# CFLAGS and LDFLAGS are propagated to the dependencies, so to pass +# flags only to be used when compiling / linking Redis itself ADD_CFLAGS +# and ADD_LDFLAGS are used instead (this is the case of 'make gcov'). +# +# Dependencies are stored in the Makefile.dep file. To rebuild this file +# Just use 'make dep', but this is only needed by developers. release_hdr := $(shell sh -c './mkreleasehdr.sh') uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') @@ -12,19 +22,19 @@ WARN= -Wall OPT= $(OPTIMIZATION) ifeq ($(uname_S),SunOS) - R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) -D__EXTENSIONS__ -D_XPG6 - R_LDFLAGS= $(LDFLAGS) - R_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread + REDIS_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(ADD_CFLAGS) -D__EXTENSIONS__ -D_XPG6 + REDIS_LDFLAGS= $(LDFLAGS) $(ADD_LDFLAGS) + REDIS_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread DEBUG= -g -ggdb else - R_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) - R_LDFLAGS= $(LDFLAGS) - R_LIBS= $(LIBS) -lm -pthread + REDIS_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(ADD_CFLAGS) + REDIS_LDFLAGS= $(LDFLAGS) $(ADD_LDFLAGS) + REDIS_LIBS= $(LIBS) -lm -pthread DEBUG= -g -rdynamic -ggdb endif # Include paths to dependencies -R_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +REDIS_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src # Default allocator ifeq ($(uname_S),Linux) @@ -47,23 +57,23 @@ ifeq ($(USE_JEMALLOC),yes) endif ifeq ($(MALLOC),tcmalloc) - R_CFLAGS+= -DUSE_TCMALLOC - R_LIBS+= -ltcmalloc + REDIS_CFLAGS+= -DUSE_TCMALLOC + REDIS_LIBS+= -ltcmalloc endif ifeq ($(MALLOC),tcmalloc_minimal) - R_CFLAGS+= -DUSE_TCMALLOC - R_LIBS+= -ltcmalloc_minimal + REDIS_CFLAGS+= -DUSE_TCMALLOC + REDIS_LIBS+= -ltcmalloc_minimal endif ifeq ($(MALLOC),jemalloc) DEPENDENCY_TARGETS+= jemalloc - R_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include - R_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl + REDIS_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include + REDIS_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl endif -R_CC=$(QUIET_CC)$(CC) $(R_CFLAGS) -R_LD=$(QUIET_LINK)$(CC) $(R_LDFLAGS) +REDIS_CC=$(QUIET_CC)$(CC) $(REDIS_CFLAGS) +REDIS_LD=$(QUIET_LINK)$(CC) $(REDIS_LDFLAGS) PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin @@ -81,18 +91,18 @@ QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -R_SERVER_NAME= redis-server -R_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o -R_CLI_NAME= redis-cli -R_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o -R_BENCHMARK_NAME= redis-benchmark -R_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o -R_CHECK_DUMP_NAME= redis-check-dump -R_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o -R_CHECK_AOF_NAME= redis-check-aof -R_CHECK_AOF_OBJ= redis-check-aof.o - -all: $(R_SERVER_NAME) $(R_CLI_NAME) $(R_BENCHMARK_NAME) $(R_CHECK_DUMP_NAME) $(R_CHECK_AOF_NAME) +REDIS_SERVEREDIS_NAME= redis-server +REDIS_SERVEREDIS_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o +REDIS_CLI_NAME= redis-cli +REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o +REDIS_BENCHMARK_NAME= redis-benchmark +REDIS_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o +REDIS_CHECK_DUMP_NAME= redis-check-dump +REDIS_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o +REDIS_CHECK_AOF_NAME= redis-check-aof +REDIS_CHECK_AOF_OBJ= redis-check-aof.o + +all: $(REDIS_SERVEREDIS_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" @echo "Hint: To run 'make test' is a good idea ;)" @echo "" @@ -103,7 +113,7 @@ all: $(R_SERVER_NAME) $(R_CLI_NAME) $(R_BENCHMARK_NAME) $(R_CHECK_DUMP_NAME) $(R include Makefile.dep dep: - $(R_CC) -MM *.c > Makefile.dep + $(REDIS_CC) -MM *.c > Makefile.dep .PHONY: dep @@ -111,18 +121,18 @@ dep: .make-prerequisites: @touch $@ -# Clean local objects and build dependencies when R_CFLAGS is different -ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(R_CFLAGS)) +# Clean local objects and build dependencies when REDIS_CFLAGS is different +ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(REDIS_CFLAGS)) .make-cflags: clean -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS)) - -(echo "$(R_CFLAGS)" > .make-cflags) + -(echo "$(REDIS_CFLAGS)" > .make-cflags) .make-prerequisites: .make-cflags endif -# Clean local objects when R_LDFLAGS is different -ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(R_LDFLAGS)) +# Clean local objects when REDIS_LDFLAGS is different +ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(REDIS_LDFLAGS)) .make-ldflags: clean - -(echo "$(R_LDFLAGS)" > .make-ldflags) + -(echo "$(REDIS_LDFLAGS)" > .make-ldflags) .make-prerequisites: .make-ldflags endif @@ -134,33 +144,33 @@ ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc || echo none'), $( endif # redis-server -$(R_SERVER_NAME): $(R_SERVER_OBJ) - $(R_LD) -o $@ $^ ../deps/lua/src/liblua.a $(R_LIBS) +$(REDIS_SERVEREDIS_NAME): $(REDIS_SERVEREDIS_OBJ) + $(REDIS_LD) -o $@ $^ ../deps/lua/src/liblua.a $(REDIS_LIBS) # redis-cli -$(R_CLI_NAME): $(R_CLI_OBJ) - $(R_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(R_LIBS) +$(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(REDIS_LIBS) # redis-benchmark -$(R_BENCHMARK_NAME): $(R_BENCHMARK_OBJ) - $(R_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(R_LIBS) +$(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(REDIS_LIBS) # redis-check-dump -$(R_CHECK_DUMP_NAME): $(R_CHECK_DUMP_OBJ) - $(R_LD) -o $@ $^ $(R_LIBS) +$(REDIS_CHECK_DUMP_NAME): $(REDIS_CHECK_DUMP_OBJ) + $(REDIS_LD) -o $@ $^ $(REDIS_LIBS) # redis-check-aof -$(R_CHECK_AOF_NAME): $(R_CHECK_AOF_OBJ) - $(R_LD) -o $@ $^ $(R_LIBS) +$(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ) + $(REDIS_LD) -o $@ $^ $(REDIS_LIBS) # Because the jemalloc.h header is generated as a part of the jemalloc build, # building it should complete before building any other object. Instead of # depending on a single artifact, build all dependencies first. %.o: %.c .make-prerequisites - $(R_CC) -c $< + $(REDIS_CC) -c $< clean: - rm -rf $(R_SERVER_NAME) $(R_CLI_NAME) $(R_BENCHMARK_NAME) $(R_CHECK_DUMP_NAME) $(R_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html + rm -rf $(REDIS_SERVEREDIS_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html .PHONY: clean @@ -170,7 +180,7 @@ distclean: clean .PHONY: distclean -test: $(R_SERVER_NAME) $(R_CHECK_AOF_NAME) +test: $(REDIS_SERVEREDIS_NAME) $(REDIS_CHECK_AOF_NAME) @(cd ..; ./runtest) lcov: @@ -181,11 +191,8 @@ lcov: .PHONY: lcov -bench: $(R_BENCHMARK_NAME) - ./$(R_BENCHMARK_NAME) - -log: - git log '--pretty=format:%ad %s (%cn)' --date=short > ../Changelog +bench: $(REDIS_BENCHMARK_NAME) + ./$(REDIS_BENCHMARK_NAME) 32bit: @echo "" @@ -193,25 +200,19 @@ log: @echo "" $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" -gprof: - $(MAKE) CFLAGS="$(CFLAGS) -pg" LDFLAGS="$(LDFLAGS) -pg" - gcov: - $(MAKE) CFLAGS="$(CFLAGS) -fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" + $(MAKE) ADD_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" noopt: $(MAKE) OPT="-O0" -32bitgprof: - $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" gprof - src/help.h: @../utils/generate-command-help.rb > help.h install: all mkdir -p $(INSTALL_BIN) - $(INSTALL) $(R_SERVER_NAME) $(INSTALL_BIN) - $(INSTALL) $(R_BENCHMARK_NAME) $(INSTALL_BIN) - $(INSTALL) $(R_CLI_NAME) $(INSTALL_BIN) - $(INSTALL) $(R_CHECK_DUMP_NAME) $(INSTALL_BIN) - $(INSTALL) $(R_CHECK_AOF_NAME) $(INSTALL_BIN) + $(INSTALL) $(REDIS_SERVEREDIS_NAME) $(INSTALL_BIN) + $(INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) + $(INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) + $(INSTALL) $(REDIS_CHECK_DUMP_NAME) $(INSTALL_BIN) + $(INSTALL) $(REDIS_CHECK_AOF_NAME) $(INSTALL_BIN) From 50fb330399a377016ceede10819c042b76ae6903 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Apr 2012 11:51:58 +0200 Subject: [PATCH 0133/1752] Make gcov fixed. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index a0e03f3e7b2..ff4212f2843 100644 --- a/src/Makefile +++ b/src/Makefile @@ -201,7 +201,7 @@ bench: $(REDIS_BENCHMARK_NAME) $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" gcov: - $(MAKE) ADD_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" + $(MAKE) ADD_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" ADD_LDFLAGS="-fprofile-arcs -ftest-coverage" noopt: $(MAKE) OPT="-O0" From 206568257a6df9effd66f292904d0e75c3c928ad Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Apr 2012 11:49:52 +0200 Subject: [PATCH 0134/1752] memtest.c: integer overflow fixed. --- src/memtest.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/memtest.c b/src/memtest.c index c8b47d489a3..09b4d831b1e 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -47,7 +47,7 @@ void memtest_progress_end(void) { } void memtest_progress_step(size_t curr, size_t size, char c) { - size_t chars = (curr*progress_full)/size, j; + size_t chars = ((unsigned long long)curr*progress_full)/size, j; for (j = 0; j < chars-progress_printed; j++) { printf("%c",c); From d298825803ee32513ad08f415a93f7f0a7c362c2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Apr 2012 11:50:18 +0200 Subject: [PATCH 0135/1752] Print arch bits with redis-server -v --- src/redis.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index bfac279f490..86cf0d957e7 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2251,8 +2251,12 @@ void daemonize(void) { } void version() { - printf("Redis server v=%s sha=%s:%d malloc=%s\n", REDIS_VERSION, - redisGitSHA1(), atoi(redisGitDirty()) > 0, ZMALLOC_LIB); + printf("Redis server v=%s sha=%s:%d malloc=%s bits=%d\n", + REDIS_VERSION, + redisGitSHA1(), + atoi(redisGitDirty()) > 0, + ZMALLOC_LIB, + sizeof(long) == 4 ? 32 : 64); exit(0); } From 236adc28091d1a6714b8a3c62338a81943db76f9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 16:13:56 +0200 Subject: [PATCH 0136/1752] A few var names fixed in Makefile. I modified it for error in a previous commit doing search & replace. --- src/Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Makefile b/src/Makefile index ff4212f2843..3a6cdc66362 100644 --- a/src/Makefile +++ b/src/Makefile @@ -91,8 +91,8 @@ QUIET_CC = @printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$@$(ENDCOLOR QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -REDIS_SERVEREDIS_NAME= redis-server -REDIS_SERVEREDIS_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o +REDIS_SERVER_NAME= redis-server +REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o REDIS_CLI_NAME= redis-cli REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o REDIS_BENCHMARK_NAME= redis-benchmark @@ -102,7 +102,7 @@ REDIS_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o REDIS_CHECK_AOF_NAME= redis-check-aof REDIS_CHECK_AOF_OBJ= redis-check-aof.o -all: $(REDIS_SERVEREDIS_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) +all: $(REDIS_SERVER_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" @echo "Hint: To run 'make test' is a good idea ;)" @echo "" @@ -144,7 +144,7 @@ ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc || echo none'), $( endif # redis-server -$(REDIS_SERVEREDIS_NAME): $(REDIS_SERVEREDIS_OBJ) +$(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) $(REDIS_LD) -o $@ $^ ../deps/lua/src/liblua.a $(REDIS_LIBS) # redis-cli @@ -170,7 +170,7 @@ $(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ) $(REDIS_CC) -c $< clean: - rm -rf $(REDIS_SERVEREDIS_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html + rm -rf $(REDIS_SERVER_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) *.o *.gcda *.gcno *.gcov redis.info lcov-html .PHONY: clean @@ -180,7 +180,7 @@ distclean: clean .PHONY: distclean -test: $(REDIS_SERVEREDIS_NAME) $(REDIS_CHECK_AOF_NAME) +test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) @(cd ..; ./runtest) lcov: @@ -211,7 +211,7 @@ src/help.h: install: all mkdir -p $(INSTALL_BIN) - $(INSTALL) $(REDIS_SERVEREDIS_NAME) $(INSTALL_BIN) + $(INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) $(INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) $(INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) $(INSTALL) $(REDIS_CHECK_DUMP_NAME) $(INSTALL_BIN) From 430602b26cb2b67d168479fd6171dab7357314ce Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Mar 2012 12:02:28 +0200 Subject: [PATCH 0137/1752] Protect globals access in Lua scripting. --- src/scripting.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 0f876961a22..03938edd5d6 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -412,6 +412,45 @@ void luaLoadLibraries(lua_State *lua) { #endif } +void scriptingProtectGlobals(lua_State *lua) { + char *s[26]; + sds code = sdsempty(); + int j; + + /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html */ + s[0]="local mt = getmetatable(_G)\n"; + s[1]="if mt == nil then\n"; + s[2]=" mt = {}\n"; + s[3]=" setmetatable(_G, mt)\n"; + s[4]="end\n"; + s[5]="__STRICT = true\n"; + s[6]="mt.__declared = {}\n"; + s[7]="mt.__newindex = function (t, n, v)\n"; + s[8]=" if __STRICT and not mt.__declared[n] and debug.getinfo(2) then\n"; + s[9]=" local w = debug.getinfo(2, \"S\").what\n"; + s[10]=" if w ~= \"main\" and w ~= \"C\" then\n"; + s[11]=" error(\"assign to undeclared global var '\"..n..\"'\", 2)\n"; + s[12]=" end\n"; + s[13]=" mt.__declared[n] = true\n"; + s[14]=" end\n"; + s[15]=" rawset(t, n, v)\n"; + s[16]="end\n"; + s[17]="mt.__index = function (t, n)\n"; + s[18]=" if debug.getinfo(2) and not mt.__declared[n] and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; + s[19]=" error(\"global var '\"..n..\"' is not declared\", 2)\n"; + s[20]=" end\n"; + s[21]=" return rawget(t, n)\n"; + s[22]="end\n"; + s[23]="function global(...)\n"; + s[24]=" for _, v in ipairs{...} do mt.__declared[v] = true end\n"; + s[25]="end\n"; + + for (j = 0; j < 26; j++) code = sdscatlen(code,s[j],strlen(s[j])); + luaL_loadbuffer(lua,code,sdslen(code),"strict_lua"); + lua_pcall(lua,0,0,0); + sdsfree(code); +} + /* Initialize the scripting environment. * It is possible to call this function to reset the scripting environment * assuming that we call scriptingRelease() before. @@ -501,6 +540,11 @@ void scriptingInit(void) { server.lua_client->flags |= REDIS_LUA_CLIENT; } + /* Lua beginners ofter don't use "local", this is likely to introduce + * subtle bugs in their code. To prevent problems we protect accesses + * to global variables. */ + scriptingProtectGlobals(lua); + server.lua = lua; } From 3e6a4463e02dc4a39cf4f73704dbcabacd7db8ff Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 11:16:50 +0200 Subject: [PATCH 0138/1752] Scripting: globals protection can now be switched on/off. --- redis.conf | 17 +++++++++++ src/config.c | 15 +++++++++ src/redis.c | 1 + src/redis.h | 3 ++ src/scripting.c | 81 ++++++++++++++++++++++++++++--------------------- 5 files changed, 83 insertions(+), 34 deletions(-) diff --git a/redis.conf b/redis.conf index 85220b80deb..d48101db3f8 100644 --- a/redis.conf +++ b/redis.conf @@ -401,6 +401,23 @@ auto-aof-rewrite-min-size 64mb # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 +# By default variables in a Lua script are global, this means that a correct +# script must declare all the local variables explicitly using the 'local' +# keyword. Lua beginners are known to violate this rule, polluting the global +# namespace, or creating scripts that may fail under certain conditions, for +# this reason by default Redis installs a protection that will raise an error +# every time a script attempts to access a global variable that was not +# explicitly declared via global(). +# +# It's worth to note that normal Redis scripts should never use globals, but +# we don't entirely disable the possibility because from time to time crazy +# things in the right hands can be pretty powerful. +# +# Globals protection may result into a minor performance hint, so it is +# possible to disable the feature in production environments using the +# following configuration directive, or at runtime using CONFIG SET. +lua-protect-globals yes + ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified diff --git a/src/config.c b/src/config.c index 4d35521d742..a45e5108e1c 100644 --- a/src/config.c +++ b/src/config.c @@ -308,6 +308,10 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); + } else if (!strcasecmp(argv[0],"lua-protect-globals") && argc == 2) { + if ((server.lua_protect_globals = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { @@ -549,6 +553,16 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"lua-time-limit")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.lua_time_limit = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"lua-protect-globals")) { + int enable = yesnotoi(o->ptr); + + if (enable == -1) goto badfmt; + if (enable == 0 && server.lua_protect_globals == 1) { + scriptingDisableGlobalsProtection(server.lua); + } else if (enable && server.lua_protect_globals == 0) { + scriptingEnableGlobalsProtection(server.lua); + } + server.lua_protect_globals = enable; } else if (!strcasecmp(c->argv[2]->ptr,"slowlog-log-slower-than")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR) goto badfmt; server.slowlog_log_slower_than = ll; @@ -743,6 +757,7 @@ void configGetCommand(redisClient *c) { config_get_bool_field("rdbcompression", server.rdb_compression); config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); + config_get_bool_field("lua-protect-globals", server.lua_protect_globals); /* Everything we can't handle with macros follows. */ diff --git a/src/redis.c b/src/redis.c index 86cf0d957e7..9966f945a86 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1067,6 +1067,7 @@ void initServerConfig() { server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; server.lua_timedout = 0; + server.lua_protect_globals = 1; updateLRUClock(); resetServerSaveParams(); diff --git a/src/redis.h b/src/redis.h index 9702010e2de..7d1cc7ddf86 100644 --- a/src/redis.h +++ b/src/redis.h @@ -585,6 +585,7 @@ struct redisServer { int lua_timedout; /* True if we reached the time limit for script execution. */ int lua_kill; /* Kill the script if true. */ + int lua_protect_globals; /* If true globals must be declared */ /* Assert & bug reportign */ char *assert_failed; char *assert_file; @@ -960,6 +961,8 @@ int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *num /* Scripting */ void scriptingInit(void); +void scriptingEnableGlobalsProtection(lua_State *lua); +void scriptingDisableGlobalsProtection(lua_State *lua); /* Git SHA1 */ char *redisGitSHA1(void); diff --git a/src/scripting.c b/src/scripting.c index 03938edd5d6..138437cf5e6 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -412,45 +412,57 @@ void luaLoadLibraries(lua_State *lua) { #endif } -void scriptingProtectGlobals(lua_State *lua) { - char *s[26]; +/* This function installs metamethods in the global table _G that prevent + * the creation of globals accidentally. + * + * It should be the last to be called in the scripting engine initialization + * sequence, because it may interact with creation of globals. + * Note that the function is designed to be called multiple times if needed + * without issues, because it is possible to enabled/disable globals protection + * at runtime with CONFIG SET. */ +void scriptingEnableGlobalsProtection(lua_State *lua) { + char *s[32]; sds code = sdsempty(); - int j; + int j = 0; - /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html */ - s[0]="local mt = getmetatable(_G)\n"; - s[1]="if mt == nil then\n"; - s[2]=" mt = {}\n"; - s[3]=" setmetatable(_G, mt)\n"; - s[4]="end\n"; - s[5]="__STRICT = true\n"; - s[6]="mt.__declared = {}\n"; - s[7]="mt.__newindex = function (t, n, v)\n"; - s[8]=" if __STRICT and not mt.__declared[n] and debug.getinfo(2) then\n"; - s[9]=" local w = debug.getinfo(2, \"S\").what\n"; - s[10]=" if w ~= \"main\" and w ~= \"C\" then\n"; - s[11]=" error(\"assign to undeclared global var '\"..n..\"'\", 2)\n"; - s[12]=" end\n"; - s[13]=" mt.__declared[n] = true\n"; - s[14]=" end\n"; - s[15]=" rawset(t, n, v)\n"; - s[16]="end\n"; - s[17]="mt.__index = function (t, n)\n"; - s[18]=" if debug.getinfo(2) and not mt.__declared[n] and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; - s[19]=" error(\"global var '\"..n..\"' is not declared\", 2)\n"; - s[20]=" end\n"; - s[21]=" return rawget(t, n)\n"; - s[22]="end\n"; - s[23]="function global(...)\n"; - s[24]=" for _, v in ipairs{...} do mt.__declared[v] = true end\n"; - s[25]="end\n"; - - for (j = 0; j < 26; j++) code = sdscatlen(code,s[j],strlen(s[j])); - luaL_loadbuffer(lua,code,sdslen(code),"strict_lua"); + /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html. + * Modified to be adapted to Redis. */ + s[j++]="mt = {}\n"; + s[j++]="setmetatable(_G, mt)\n"; + s[j++]="mt.declared = {}\n"; + s[j++]="mt.__newindex = function (t, n, v)\n"; + s[j++]=" if not mt.declared[n] and debug.getinfo(2) then\n"; + s[j++]=" local w = debug.getinfo(2, \"S\").what\n"; + s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; + s[j++]=" error(\"assignment to undeclared global variable '\"..n..\"'\", 2)\n"; + s[j++]=" end\n"; + s[j++]=" mt.declared[n] = true\n"; + s[j++]=" end\n"; + s[j++]=" rawset(t, n, v)\n"; + s[j++]="end\n"; + s[j++]="mt.__index = function (t, n)\n"; + s[j++]=" if debug.getinfo(2) and not mt.declared[n] and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; + s[j++]=" error(\"global variable '\"..n..\"' is not declared\", 2)\n"; + s[j++]=" end\n"; + s[j++]=" return rawget(t, n)\n"; + s[j++]="end\n"; + s[j++]="function global(...)\n"; + s[j++]=" for _, v in ipairs{...} do mt.declared[v] = true end\n"; + s[j++]="end\n"; + s[j++]=NULL; + + for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j])); + luaL_loadbuffer(lua,code,sdslen(code),"enable_strict_lua"); lua_pcall(lua,0,0,0); sdsfree(code); } +void scriptingDisableGlobalsProtection(lua_State *lua) { + char *s = "setmetatable(_G, nil)\n"; + luaL_loadbuffer(lua,s,strlen(s),"disable_strict_lua"); + lua_pcall(lua,0,0,0); +} + /* Initialize the scripting environment. * It is possible to call this function to reset the scripting environment * assuming that we call scriptingRelease() before. @@ -543,7 +555,8 @@ void scriptingInit(void) { /* Lua beginners ofter don't use "local", this is likely to introduce * subtle bugs in their code. To prevent problems we protect accesses * to global variables. */ - scriptingProtectGlobals(lua); + if (server.lua_protect_globals) + scriptingEnableGlobalsProtection(lua); server.lua = lua; } From e387dc52a03c863eaa4ec2e4d049b92b278efe64 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 11:48:45 +0200 Subject: [PATCH 0139/1752] Tests for lua globals protection. --- tests/unit/scripting.tcl | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index b9307cc1930..091b4c430df 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -219,6 +219,43 @@ start_server {tags {"scripting"}} { list [r eval {return redis.sha1hex('')} 0] \ [r eval {return redis.sha1hex('Pizza & Mandolino')} 0] } {da39a3ee5e6b4b0d3255bfef95601890afd80709 74822d82031af7493c20eefa13bd07ec4fada82f} + + test {Globals protection reading an undeclared global variable} { + catch {r eval {return a} 0} e + set e + } {*ERR*global variable*not declared*} + + test {Globals protection setting an undeclared global variable} { + catch {r eval {a=10} 0} e + set e + } {*ERR*assignment to undeclared*} + + test {Globals protection bypassed using 'global' function} { + catch {r eval {global("a"); a=10; return a} 0} e + set e + } {10} + + test {Globals protection can be disabled} { + r config set lua-protect-globals no + catch {r eval {b=20; return b} 0} e + set e + } {20} + + test {Globals protection can be re-enabled} { + r config set lua-protect-globals yes + catch {r eval {c=30; return c} 0} e + set e + } {*ERR*assignment to undeclared*} + + test {Globals protection 'global' function works with mutliple args} { + catch {r eval { + global("var1","var2") + var1=10 + var2=20 + return {var1,var2} + } 0 } e + set e + } {10 20} } start_server {tags {"scripting repl"}} { From 6255a5ae668a0aaf13d73f051face138cfbd78a4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 12:13:02 +0200 Subject: [PATCH 0140/1752] Globals protection global() function modified for speed and correctness. --- src/scripting.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 138437cf5e6..44ceb1e284c 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -427,7 +427,7 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { /* strict.lua from: http://metalua.luaforge.net/src/lib/strict.lua.html. * Modified to be adapted to Redis. */ - s[j++]="mt = {}\n"; + s[j++]="local mt = {}\n"; s[j++]="setmetatable(_G, mt)\n"; s[j++]="mt.declared = {}\n"; s[j++]="mt.__newindex = function (t, n, v)\n"; @@ -447,7 +447,11 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { s[j++]=" return rawget(t, n)\n"; s[j++]="end\n"; s[j++]="function global(...)\n"; - s[j++]=" for _, v in ipairs{...} do mt.declared[v] = true end\n"; + s[j++]=" local nargs = select(\"#\",...)\n"; + s[j++]=" for i = 1, nargs do\n"; + s[j++]=" local v = select(i,...)\n"; + s[j++]=" mt.declared[v] = true\n"; + s[j++]=" end\n"; s[j++]="end\n"; s[j++]=NULL; From 97cab30993d71633d6d9947bf0405d278b239651 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 13:26:59 +0200 Subject: [PATCH 0141/1752] Stop access to global vars. Not configurable. After considering the interaction between ability to delcare globals in scripts using the 'global' function, and the complexities related to hanlding replication and AOF in a sane way with globals AND ability to turn protection On and Off, we reconsidered the design. The new design makes clear that there is only one good way to write Redis scripts, that is not using globals. In the rare cases state must be retained across calls a Redis key can be used. --- redis.conf | 17 ----------------- src/config.c | 15 --------------- src/redis.c | 1 - src/redis.h | 3 --- src/scripting.c | 25 ++++--------------------- 5 files changed, 4 insertions(+), 57 deletions(-) diff --git a/redis.conf b/redis.conf index d48101db3f8..85220b80deb 100644 --- a/redis.conf +++ b/redis.conf @@ -401,23 +401,6 @@ auto-aof-rewrite-min-size 64mb # Set it to 0 or a negative value for unlimited execution without warnings. lua-time-limit 5000 -# By default variables in a Lua script are global, this means that a correct -# script must declare all the local variables explicitly using the 'local' -# keyword. Lua beginners are known to violate this rule, polluting the global -# namespace, or creating scripts that may fail under certain conditions, for -# this reason by default Redis installs a protection that will raise an error -# every time a script attempts to access a global variable that was not -# explicitly declared via global(). -# -# It's worth to note that normal Redis scripts should never use globals, but -# we don't entirely disable the possibility because from time to time crazy -# things in the right hands can be pretty powerful. -# -# Globals protection may result into a minor performance hint, so it is -# possible to disable the feature in production environments using the -# following configuration directive, or at runtime using CONFIG SET. -lua-protect-globals yes - ################################## SLOW LOG ################################### # The Redis Slow Log is a system to log queries that exceeded a specified diff --git a/src/config.c b/src/config.c index a45e5108e1c..4d35521d742 100644 --- a/src/config.c +++ b/src/config.c @@ -308,10 +308,6 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"lua-time-limit") && argc == 2) { server.lua_time_limit = strtoll(argv[1],NULL,10); - } else if (!strcasecmp(argv[0],"lua-protect-globals") && argc == 2) { - if ((server.lua_protect_globals = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; - } } else if (!strcasecmp(argv[0],"slowlog-log-slower-than") && argc == 2) { @@ -553,16 +549,6 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"lua-time-limit")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.lua_time_limit = ll; - } else if (!strcasecmp(c->argv[2]->ptr,"lua-protect-globals")) { - int enable = yesnotoi(o->ptr); - - if (enable == -1) goto badfmt; - if (enable == 0 && server.lua_protect_globals == 1) { - scriptingDisableGlobalsProtection(server.lua); - } else if (enable && server.lua_protect_globals == 0) { - scriptingEnableGlobalsProtection(server.lua); - } - server.lua_protect_globals = enable; } else if (!strcasecmp(c->argv[2]->ptr,"slowlog-log-slower-than")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR) goto badfmt; server.slowlog_log_slower_than = ll; @@ -757,7 +743,6 @@ void configGetCommand(redisClient *c) { config_get_bool_field("rdbcompression", server.rdb_compression); config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); - config_get_bool_field("lua-protect-globals", server.lua_protect_globals); /* Everything we can't handle with macros follows. */ diff --git a/src/redis.c b/src/redis.c index 9966f945a86..86cf0d957e7 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1067,7 +1067,6 @@ void initServerConfig() { server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; server.lua_timedout = 0; - server.lua_protect_globals = 1; updateLRUClock(); resetServerSaveParams(); diff --git a/src/redis.h b/src/redis.h index 7d1cc7ddf86..9702010e2de 100644 --- a/src/redis.h +++ b/src/redis.h @@ -585,7 +585,6 @@ struct redisServer { int lua_timedout; /* True if we reached the time limit for script execution. */ int lua_kill; /* Kill the script if true. */ - int lua_protect_globals; /* If true globals must be declared */ /* Assert & bug reportign */ char *assert_failed; char *assert_file; @@ -961,8 +960,6 @@ int *zunionInterGetKeys(struct redisCommand *cmd,robj **argv, int argc, int *num /* Scripting */ void scriptingInit(void); -void scriptingEnableGlobalsProtection(lua_State *lua); -void scriptingDisableGlobalsProtection(lua_State *lua); /* Git SHA1 */ char *redisGitSHA1(void); diff --git a/src/scripting.c b/src/scripting.c index 44ceb1e284c..ae906c68cc4 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -416,10 +416,7 @@ void luaLoadLibraries(lua_State *lua) { * the creation of globals accidentally. * * It should be the last to be called in the scripting engine initialization - * sequence, because it may interact with creation of globals. - * Note that the function is designed to be called multiple times if needed - * without issues, because it is possible to enabled/disable globals protection - * at runtime with CONFIG SET. */ + * sequence, because it may interact with creation of globals. */ void scriptingEnableGlobalsProtection(lua_State *lua) { char *s[32]; sds code = sdsempty(); @@ -434,7 +431,7 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { s[j++]=" if not mt.declared[n] and debug.getinfo(2) then\n"; s[j++]=" local w = debug.getinfo(2, \"S\").what\n"; s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; - s[j++]=" error(\"assignment to undeclared global variable '\"..n..\"'\", 2)\n"; + s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"; s[j++]=" end\n"; s[j++]=" mt.declared[n] = true\n"; s[j++]=" end\n"; @@ -442,17 +439,10 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { s[j++]="end\n"; s[j++]="mt.__index = function (t, n)\n"; s[j++]=" if debug.getinfo(2) and not mt.declared[n] and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; - s[j++]=" error(\"global variable '\"..n..\"' is not declared\", 2)\n"; + s[j++]=" error(\"Script attempted to access unexisting global variable '\"..n..\"'\", 2)\n"; s[j++]=" end\n"; s[j++]=" return rawget(t, n)\n"; s[j++]="end\n"; - s[j++]="function global(...)\n"; - s[j++]=" local nargs = select(\"#\",...)\n"; - s[j++]=" for i = 1, nargs do\n"; - s[j++]=" local v = select(i,...)\n"; - s[j++]=" mt.declared[v] = true\n"; - s[j++]=" end\n"; - s[j++]="end\n"; s[j++]=NULL; for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j])); @@ -461,12 +451,6 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { sdsfree(code); } -void scriptingDisableGlobalsProtection(lua_State *lua) { - char *s = "setmetatable(_G, nil)\n"; - luaL_loadbuffer(lua,s,strlen(s),"disable_strict_lua"); - lua_pcall(lua,0,0,0); -} - /* Initialize the scripting environment. * It is possible to call this function to reset the scripting environment * assuming that we call scriptingRelease() before. @@ -559,8 +543,7 @@ void scriptingInit(void) { /* Lua beginners ofter don't use "local", this is likely to introduce * subtle bugs in their code. To prevent problems we protect accesses * to global variables. */ - if (server.lua_protect_globals) - scriptingEnableGlobalsProtection(lua); + scriptingEnableGlobalsProtection(lua); server.lua = lua; } From d2906893e8c5aaab000cdd150c2be6ce4f453190 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 13:36:08 +0200 Subject: [PATCH 0142/1752] mt.declared is no longer needed. Lua global protection can now be simpified becuase we no longer have the global() function. It's useless to occupy memory with this table, it is also not faster because the metamethods we use are only called when a global object does not exist or we are trying to create it from a script. --- src/scripting.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index ae906c68cc4..76b25ad2334 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -426,19 +426,17 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { * Modified to be adapted to Redis. */ s[j++]="local mt = {}\n"; s[j++]="setmetatable(_G, mt)\n"; - s[j++]="mt.declared = {}\n"; s[j++]="mt.__newindex = function (t, n, v)\n"; - s[j++]=" if not mt.declared[n] and debug.getinfo(2) then\n"; + s[j++]=" if debug.getinfo(2) then\n"; s[j++]=" local w = debug.getinfo(2, \"S\").what\n"; s[j++]=" if w ~= \"main\" and w ~= \"C\" then\n"; s[j++]=" error(\"Script attempted to create global variable '\"..tostring(n)..\"'\", 2)\n"; s[j++]=" end\n"; - s[j++]=" mt.declared[n] = true\n"; s[j++]=" end\n"; s[j++]=" rawset(t, n, v)\n"; s[j++]="end\n"; s[j++]="mt.__index = function (t, n)\n"; - s[j++]=" if debug.getinfo(2) and not mt.declared[n] and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; + s[j++]=" if debug.getinfo(2) and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; s[j++]=" error(\"Script attempted to access unexisting global variable '\"..n..\"'\", 2)\n"; s[j++]=" end\n"; s[j++]=" return rawget(t, n)\n"; From d63a1716ebc8b6164287ee3ceed7c4bae0c49751 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 13:40:57 +0200 Subject: [PATCH 0143/1752] Tests modified to match the new global protection implementation. --- tests/unit/scripting.tcl | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 091b4c430df..a9aae7b8c65 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -223,39 +223,12 @@ start_server {tags {"scripting"}} { test {Globals protection reading an undeclared global variable} { catch {r eval {return a} 0} e set e - } {*ERR*global variable*not declared*} + } {*ERR*attempted to access unexisting global*} - test {Globals protection setting an undeclared global variable} { + test {Globals protection setting an undeclared global*} { catch {r eval {a=10} 0} e set e - } {*ERR*assignment to undeclared*} - - test {Globals protection bypassed using 'global' function} { - catch {r eval {global("a"); a=10; return a} 0} e - set e - } {10} - - test {Globals protection can be disabled} { - r config set lua-protect-globals no - catch {r eval {b=20; return b} 0} e - set e - } {20} - - test {Globals protection can be re-enabled} { - r config set lua-protect-globals yes - catch {r eval {c=30; return c} 0} e - set e - } {*ERR*assignment to undeclared*} - - test {Globals protection 'global' function works with mutliple args} { - catch {r eval { - global("var1","var2") - var1=10 - var2=20 - return {var1,var2} - } 0 } e - set e - } {10 20} + } {*ERR*attempted to create global*} } start_server {tags {"scripting repl"}} { From c9853f537b80971bfc82f1df9fedf5c1abf004d5 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 14:54:49 +0200 Subject: [PATCH 0144/1752] Use Lua tostring() before concatenation. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 76b25ad2334..c449714f5cc 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -437,7 +437,7 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { s[j++]="end\n"; s[j++]="mt.__index = function (t, n)\n"; s[j++]=" if debug.getinfo(2) and debug.getinfo(2, \"S\").what ~= \"C\" then\n"; - s[j++]=" error(\"Script attempted to access unexisting global variable '\"..n..\"'\", 2)\n"; + s[j++]=" error(\"Script attempted to access unexisting global variable '\"..tostring(n)..\"'\", 2)\n"; s[j++]=" end\n"; s[j++]=" return rawget(t, n)\n"; s[j++]="end\n"; From 9a2dd1eff9ed1e7fdd24ad52da0f03b72c86415b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 15:12:16 +0200 Subject: [PATCH 0145/1752] EVAL errors are more clear now. --- src/scripting.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index c449714f5cc..4c7de33bf1d 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -444,7 +444,7 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { s[j++]=NULL; for (j = 0; s[j] != NULL; j++) code = sdscatlen(code,s[j],strlen(s[j])); - luaL_loadbuffer(lua,code,sdslen(code),"enable_strict_lua"); + luaL_loadbuffer(lua,code,sdslen(code),"@enable_strict_lua"); lua_pcall(lua,0,0,0); sdsfree(code); } @@ -525,7 +525,7 @@ void scriptingInit(void) { " if b == false then b = '' end\n" " return aptr,sdslen(body->ptr)); funcdef = sdscatlen(funcdef," end",4); - if (luaL_loadbuffer(lua,funcdef,sdslen(funcdef),"func definition")) { + if (luaL_loadbuffer(lua,funcdef,sdslen(funcdef),"@user_script")) { addReplyErrorFormat(c,"Error compiling script (new function): %s\n", lua_tostring(lua,-1)); lua_pop(lua,1); From 59333ffd37f666222d80921f854a3bedd2e3d11a Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Apr 2012 15:23:32 +0200 Subject: [PATCH 0146/1752] New test for scripting engine: DECR_IF_GT. --- tests/unit/scripting.tcl | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index a9aae7b8c65..009c1347d38 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -229,6 +229,28 @@ start_server {tags {"scripting"}} { catch {r eval {a=10} 0} e set e } {*ERR*attempted to create global*} + + test {Test an example script DECR_IF_GT} { + set decr_if_gt { + local current + + current = redis.call('get',KEYS[1]) + if not current then return nil end + if current > ARGV[1] then + return redis.call('decr',KEYS[1]) + else + return redis.call('get',KEYS[1]) + end + } + r set foo 5 + set res {} + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + lappend res [r eval $decr_if_gt 1 foo 2] + set res + } {4 3 2 2 2} } start_server {tags {"scripting repl"}} { From 6cedb4d4893b02e4b23efc4d0aaeb684ed6d0935 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:34:31 -0700 Subject: [PATCH 0147/1752] Rename ADD_*FLAGS -> REDIS_*FLAGS, REDIS_*FLAGS -> FINAL_*FLAGS This reflects that REDIS_*FLAGS will only be used for compilation of Redis and not for its dependencies. Similarly, that FINAL_*FLAGS are composed of other variables and holds the options that are finally passed to the compiler and linker. --- src/Makefile | 60 ++++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3a6cdc66362..2dcd7e80ce7 100644 --- a/src/Makefile +++ b/src/Makefile @@ -2,12 +2,12 @@ # Copyright (C) 2009 Salvatore Sanfilippo # This file is released under the BSD license, see the COPYING file # -# The Makefile composes the final REDIS_CFLAGS and REDIS_LDFLAGS using +# The Makefile composes the final FINAL_CFLAGS and FINAL_LDFLAGS using # what is needed for Redis plus the standard CFLAGS and LDFLAGS passed. # However when building the dependencies (Jemalloc, Lua, Hiredis, ...) # CFLAGS and LDFLAGS are propagated to the dependencies, so to pass -# flags only to be used when compiling / linking Redis itself ADD_CFLAGS -# and ADD_LDFLAGS are used instead (this is the case of 'make gcov'). +# flags only to be used when compiling / linking Redis itself REDIS_CFLAGS +# and REDIS_LDFLAGS are used instead (this is the case of 'make gcov'). # # Dependencies are stored in the Makefile.dep file. To rebuild this file # Just use 'make dep', but this is only needed by developers. @@ -22,19 +22,19 @@ WARN= -Wall OPT= $(OPTIMIZATION) ifeq ($(uname_S),SunOS) - REDIS_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(ADD_CFLAGS) -D__EXTENSIONS__ -D_XPG6 - REDIS_LDFLAGS= $(LDFLAGS) $(ADD_LDFLAGS) - REDIS_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread + FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 + FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) + FINAL_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread DEBUG= -g -ggdb else - REDIS_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(ADD_CFLAGS) - REDIS_LDFLAGS= $(LDFLAGS) $(ADD_LDFLAGS) - REDIS_LIBS= $(LIBS) -lm -pthread + FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) + FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) + FINAL_LIBS= $(LIBS) -lm -pthread DEBUG= -g -rdynamic -ggdb endif # Include paths to dependencies -REDIS_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src # Default allocator ifeq ($(uname_S),Linux) @@ -57,23 +57,23 @@ ifeq ($(USE_JEMALLOC),yes) endif ifeq ($(MALLOC),tcmalloc) - REDIS_CFLAGS+= -DUSE_TCMALLOC - REDIS_LIBS+= -ltcmalloc + FINAL_CFLAGS+= -DUSE_TCMALLOC + FINAL_LIBS+= -ltcmalloc endif ifeq ($(MALLOC),tcmalloc_minimal) - REDIS_CFLAGS+= -DUSE_TCMALLOC - REDIS_LIBS+= -ltcmalloc_minimal + FINAL_CFLAGS+= -DUSE_TCMALLOC + FINAL_LIBS+= -ltcmalloc_minimal endif ifeq ($(MALLOC),jemalloc) DEPENDENCY_TARGETS+= jemalloc - REDIS_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include - REDIS_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl + FINAL_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include + FINAL_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl endif -REDIS_CC=$(QUIET_CC)$(CC) $(REDIS_CFLAGS) -REDIS_LD=$(QUIET_LINK)$(CC) $(REDIS_LDFLAGS) +REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) +REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) PREFIX= /usr/local INSTALL_BIN= $(PREFIX)/bin @@ -121,18 +121,18 @@ dep: .make-prerequisites: @touch $@ -# Clean local objects and build dependencies when REDIS_CFLAGS is different -ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(REDIS_CFLAGS)) +# Clean local objects and build dependencies when FINAL_CFLAGS is different +ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(FINAL_CFLAGS)) .make-cflags: clean -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS)) - -(echo "$(REDIS_CFLAGS)" > .make-cflags) + -(echo "$(FINAL_CFLAGS)" > .make-cflags) .make-prerequisites: .make-cflags endif -# Clean local objects when REDIS_LDFLAGS is different -ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(REDIS_LDFLAGS)) +# Clean local objects when FINAL_LDFLAGS is different +ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(FINAL_LDFLAGS)) .make-ldflags: clean - -(echo "$(REDIS_LDFLAGS)" > .make-ldflags) + -(echo "$(FINAL_LDFLAGS)" > .make-ldflags) .make-prerequisites: .make-ldflags endif @@ -145,23 +145,23 @@ endif # redis-server $(REDIS_SERVER_NAME): $(REDIS_SERVER_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/lua/src/liblua.a $(REDIS_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/lua/src/liblua.a $(FINAL_LIBS) # redis-cli $(REDIS_CLI_NAME): $(REDIS_CLI_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(REDIS_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a ../deps/linenoise/linenoise.o $(FINAL_LIBS) # redis-benchmark $(REDIS_BENCHMARK_NAME): $(REDIS_BENCHMARK_OBJ) - $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(REDIS_LIBS) + $(REDIS_LD) -o $@ $^ ../deps/hiredis/libhiredis.a $(FINAL_LIBS) # redis-check-dump $(REDIS_CHECK_DUMP_NAME): $(REDIS_CHECK_DUMP_OBJ) - $(REDIS_LD) -o $@ $^ $(REDIS_LIBS) + $(REDIS_LD) -o $@ $^ $(FINAL_LIBS) # redis-check-aof $(REDIS_CHECK_AOF_NAME): $(REDIS_CHECK_AOF_OBJ) - $(REDIS_LD) -o $@ $^ $(REDIS_LIBS) + $(REDIS_LD) -o $@ $^ $(FINAL_LIBS) # Because the jemalloc.h header is generated as a part of the jemalloc build, # building it should complete before building any other object. Instead of @@ -201,7 +201,7 @@ bench: $(REDIS_BENCHMARK_NAME) $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" gcov: - $(MAKE) ADD_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" ADD_LDFLAGS="-fprofile-arcs -ftest-coverage" + $(MAKE) REDIS_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_LDFLAGS="-fprofile-arcs -ftest-coverage" noopt: $(MAKE) OPT="-O0" From 0b27a55f307a84d92c87d7bb587b8ec04cf9515a Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:38:12 -0700 Subject: [PATCH 0148/1752] The lcov target shouldn't clean This is not needed because every change in compiler/linker flags triggers a cleanup. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 2dcd7e80ce7..e6972640259 100644 --- a/src/Makefile +++ b/src/Makefile @@ -184,7 +184,7 @@ test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) @(cd ..; ./runtest) lcov: - $(MAKE) clean gcov + $(MAKE) gcov @(set -e; cd ..; ./runtest --clients 1) @geninfo -o redis.info . @genhtml --legend -o lcov-html redis.info From cb481f432af0bcd2850c5cd0ef1a93906594976f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:38:29 -0700 Subject: [PATCH 0149/1752] Ignore gcov/lcov artifacts --- src/.gitignore | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 src/.gitignore diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 00000000000..aee7aacf09f --- /dev/null +++ b/src/.gitignore @@ -0,0 +1,5 @@ +*.gcda +*.gcno +*.gcov +redis.info +lcov-html From 95bc195158d2d9951a8534a993182399f2bb49b8 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:41:40 -0700 Subject: [PATCH 0150/1752] Question mark assignment is not needed --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index e6972640259..b3ada6ffa55 100644 --- a/src/Makefile +++ b/src/Makefile @@ -38,9 +38,9 @@ FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src # Default allocator ifeq ($(uname_S),Linux) - MALLOC?=jemalloc + MALLOC=jemalloc else - MALLOC?=libc + MALLOC=libc endif # Backwards compatibility for selecting an allocator From 61e8825e89fb4ffdf0f2bd81cb2333c176cc4fdf Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:43:06 -0700 Subject: [PATCH 0151/1752] First set defaults, then do composition --- src/Makefile | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/src/Makefile b/src/Makefile index b3ada6ffa55..8b24fd9ccd2 100644 --- a/src/Makefile +++ b/src/Makefile @@ -17,25 +17,11 @@ uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') OPTIMIZATION?=-O2 DEPENDENCY_TARGETS=hiredis linenoise lua +# Default settings STD= -std=c99 -pedantic WARN= -Wall OPT= $(OPTIMIZATION) -ifeq ($(uname_S),SunOS) - FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 - FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) - FINAL_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread - DEBUG= -g -ggdb -else - FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) - FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) - FINAL_LIBS= $(LIBS) -lm -pthread - DEBUG= -g -rdynamic -ggdb -endif - -# Include paths to dependencies -FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src - # Default allocator ifeq ($(uname_S),Linux) MALLOC=jemalloc @@ -56,6 +42,21 @@ ifeq ($(USE_JEMALLOC),yes) MALLOC=jemalloc endif +ifeq ($(uname_S),SunOS) + FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 + FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) + FINAL_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread + DEBUG= -g -ggdb +else + FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) + FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) + FINAL_LIBS= $(LIBS) -lm -pthread + DEBUG= -g -rdynamic -ggdb +endif + +# Include paths to dependencies +FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src + ifeq ($(MALLOC),tcmalloc) FINAL_CFLAGS+= -DUSE_TCMALLOC FINAL_LIBS+= -ltcmalloc From 3126e087572077a28ccd5ec9d97e55f44dc52547 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:44:34 -0700 Subject: [PATCH 0152/1752] Remove unused LIBS variable --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 8b24fd9ccd2..bc3c97db83b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -45,12 +45,12 @@ endif ifeq ($(uname_S),SunOS) FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) - FINAL_LIBS= $(LIBS) -ldl -lnsl -lsocket -lm -lpthread + FINAL_LIBS= -ldl -lnsl -lsocket -lm -lpthread DEBUG= -g -ggdb else FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) - FINAL_LIBS= $(LIBS) -lm -pthread + FINAL_LIBS= -lm -pthread DEBUG= -g -rdynamic -ggdb endif From 864229585a407431c8bb4b156bfb1a828aeeff9f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:46:28 -0700 Subject: [PATCH 0153/1752] Don't set flags recursively --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index bc3c97db83b..38a680646ca 100644 --- a/src/Makefile +++ b/src/Makefile @@ -199,7 +199,7 @@ bench: $(REDIS_BENCHMARK_NAME) @echo "" @echo "WARNING: if it fails under Linux you probably need to install libc6-dev-i386" @echo "" - $(MAKE) CFLAGS="$(CFLAGS) -m32" LDFLAGS="$(LDFLAGS) -m32" + $(MAKE) CFLAGS="-m32" LDFLAGS="-m32" gcov: $(MAKE) REDIS_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_LDFLAGS="-fprofile-arcs -ftest-coverage" From d0cd262fdfb0372ab9206b1381002636aee91756 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 13 Apr 2012 17:50:38 -0700 Subject: [PATCH 0154/1752] Persist `make` settings and trigger rebuild if anything changes --- src/Makefile | 41 +++++++++++++++++++++++------------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/Makefile b/src/Makefile index 38a680646ca..f6432278713 100644 --- a/src/Makefile +++ b/src/Makefile @@ -42,6 +42,9 @@ ifeq ($(USE_JEMALLOC),yes) MALLOC=jemalloc endif +# Override default settings if possible +-include .make-settings + ifeq ($(uname_S),SunOS) FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) @@ -118,30 +121,32 @@ dep: .PHONY: dep +persist-settings: distclean + echo STD=$(STD) >> .make-settings + echo WARN=$(WARN) >> .make-settings + echo OPT=$(OPT) >> .make-settings + echo MALLOC=$(MALLOC) >> .make-settings + echo CFLAGS=$(CFLAGS) >> .make-settings + echo LDFLAGS=$(LDFLAGS) >> .make-settings + echo REDIS_CFLAGS=$(REDIS_CFLAGS) >> .make-settings + echo REDIS_LDFLAGS=$(REDIS_LDFLAGS) >> .make-settings + echo PREV_FINAL_CFLAGS=$(FINAL_CFLAGS) >> .make-settings + echo PREV_FINAL_LDFLAGS=$(FINAL_LDFLAGS) >> .make-settings + -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS)) + +.PHONY: persist-settings + # Prerequisites target .make-prerequisites: @touch $@ -# Clean local objects and build dependencies when FINAL_CFLAGS is different -ifneq ($(shell sh -c '[ -f .make-cflags ] && cat .make-cflags || echo none'), $(FINAL_CFLAGS)) -.make-cflags: clean - -(cd ../deps && $(MAKE) $(DEPENDENCY_TARGETS)) - -(echo "$(FINAL_CFLAGS)" > .make-cflags) -.make-prerequisites: .make-cflags -endif - -# Clean local objects when FINAL_LDFLAGS is different -ifneq ($(shell sh -c '[ -f .make-ldflags ] && cat .make-ldflags || echo none'), $(FINAL_LDFLAGS)) -.make-ldflags: clean - -(echo "$(FINAL_LDFLAGS)" > .make-ldflags) -.make-prerequisites: .make-ldflags +# Clean everything, persist settings and build dependencies if anything changed +ifneq ($(strip $(PREV_FINAL_CFLAGS)), $(strip $(FINAL_CFLAGS))) +.make-prerequisites: persist-settings endif -# Clean local objects when MALLOC is different -ifneq ($(shell sh -c '[ -f .make-malloc ] && cat .make-malloc || echo none'), $(MALLOC)) -.make-malloc: clean - -(echo "$(MALLOC)" > .make-malloc) -.make-prerequisites: .make-malloc +ifneq ($(strip $(PREV_FINAL_LDFLAGS)), $(strip $(FINAL_LDFLAGS))) +.make-prerequisites: persist-settings endif # redis-server From 96aeca4b9d084ceff6bc8ee56a2bf58bab61d3db Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Apr 2012 10:04:42 +0200 Subject: [PATCH 0155/1752] Less false positives in maxclients test, hopefully. --- tests/unit/limits.tcl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/limits.tcl b/tests/unit/limits.tcl index 9988983fad7..f622e1b9155 100644 --- a/tests/unit/limits.tcl +++ b/tests/unit/limits.tcl @@ -2,9 +2,10 @@ start_server {tags {"limits"} overrides {maxclients 10}} { test {Check if maxclients works refusing connections} { set c 0 catch { - while 1 { + while {$c < 50} { incr c redis_deferring_client + after 100 } } e assert {$c > 8 && $c <= 10} From 68ee18558ab23066b0273ae59b3d6ab63db2dd15 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Apr 2012 13:05:09 +0200 Subject: [PATCH 0156/1752] lookupKeyByPattern() used by SORT GET/BY rewritten. Fixes issue #460. lookupKeyByPattern() was implemented with a trick to speedup the lookup process allocating two fake Redis obejcts on the stack. However now that we propagate expires to the slave as DEL operations the lookup of the key may result into a call to expireIfNeeded() having the stack allocated object as argument, that may in turn use it to create the protocol to send to the slave. But since this fake obejcts are inherently read-only this is a problem. As a side effect of this fix there are no longer size limits in the pattern to be used with GET/BY option of SORT. See https://github.com/antirez/redis/issues/460 for bug details. --- src/sort.c | 74 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 42 insertions(+), 32 deletions(-) diff --git a/src/sort.c b/src/sort.c index 3f02e49a709..ff655c7e71c 100644 --- a/src/sort.c +++ b/src/sort.c @@ -9,21 +9,27 @@ redisSortOperation *createSortOperation(int type, robj *pattern) { return so; } -/* Return the value associated to the key with a name obtained - * substituting the first occurence of '*' in 'pattern' with 'subst'. +/* Return the value associated to the key with a name obtained using + * the following rules: + * + * 1) The first occurence of '*' in 'pattern' is substituted with 'subst'. + * + * 2) If 'pattern' matches the "->" string, everything on the left of + * the arrow is treated as the name of an hash field, and the part on the + * left as the key name containing an hash. The value of the specified + * field is returned. + * + * 3) If 'pattern' equals "#", the function simply returns 'subst' itself so + * that the SORT command can be used like: SORT key GET # to retrieve + * the Set/List elements directly. + * * The returned object will always have its refcount increased by 1 * when it is non-NULL. */ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { - char *p, *f; + char *p, *f, *k; sds spat, ssub; - robj keyobj, fieldobj, *o; + robj *keyobj, *fieldobj, *o; int prefixlen, sublen, postfixlen, fieldlen; - /* Expoit the internal sds representation to create a sds string allocated on the stack in order to make this function faster */ - struct { - int len; - int free; - char buf[REDIS_SORTKEY_MAX+1]; - } keyname, fieldname; /* If the pattern is "#" return the substitution object itself in order * to implement the "SORT ... GET #" feature. */ @@ -37,9 +43,10 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { * a decoded object on the fly. Otherwise getDecodedObject will just * increment the ref count, that we'll decrement later. */ subst = getDecodedObject(subst); - ssub = subst->ptr; - if (sdslen(spat)+sdslen(ssub)-1 > REDIS_SORTKEY_MAX) return NULL; + + /* If we can't find '*' in the pattern we return NULL as to GET a + * fixed key does not make sense. */ p = strchr(spat,'*'); if (!p) { decrRefCount(subst); @@ -47,46 +54,49 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { } /* Find out if we're dealing with a hash dereference. */ - if ((f = strstr(p+1, "->")) != NULL) { - fieldlen = sdslen(spat)-(f-spat); - /* this also copies \0 character */ - memcpy(fieldname.buf,f+2,fieldlen-1); - fieldname.len = fieldlen-2; + if ((f = strstr(p+1, "->")) != NULL && *(f+2) != '\0') { + fieldlen = sdslen(spat)-(f-spat)-2; + fieldobj = createStringObject(f+2,fieldlen); } else { fieldlen = 0; } + /* Perform the '*' substitution. */ prefixlen = p-spat; sublen = sdslen(ssub); - postfixlen = sdslen(spat)-(prefixlen+1)-fieldlen; - memcpy(keyname.buf,spat,prefixlen); - memcpy(keyname.buf+prefixlen,ssub,sublen); - memcpy(keyname.buf+prefixlen+sublen,p+1,postfixlen); - keyname.buf[prefixlen+sublen+postfixlen] = '\0'; - keyname.len = prefixlen+sublen+postfixlen; - decrRefCount(subst); + postfixlen = sdslen(spat)-(prefixlen+1)-(fieldlen ? fieldlen+2 : 0); + keyobj = createStringObject(NULL,prefixlen+sublen+postfixlen); + k = keyobj->ptr; + memcpy(k,spat,prefixlen); + memcpy(k+prefixlen,ssub,sublen); + memcpy(k+prefixlen+sublen,p+1,postfixlen); + decrRefCount(subst); /* Incremented by decodeObject() */ /* Lookup substituted key */ - initStaticStringObject(keyobj,((char*)&keyname)+(sizeof(struct sdshdr))); - o = lookupKeyRead(db,&keyobj); - if (o == NULL) return NULL; + o = lookupKeyRead(db,keyobj); + if (o == NULL) goto noobj; if (fieldlen > 0) { - if (o->type != REDIS_HASH || fieldname.len < 1) return NULL; + if (o->type != REDIS_HASH) goto noobj; /* Retrieve value from hash by the field name. This operation * already increases the refcount of the returned object. */ - initStaticStringObject(fieldobj,((char*)&fieldname)+(sizeof(struct sdshdr))); - o = hashTypeGetObject(o, &fieldobj); + o = hashTypeGetObject(o, fieldobj); } else { - if (o->type != REDIS_STRING) return NULL; + if (o->type != REDIS_STRING) goto noobj; /* Every object that this function returns needs to have its refcount * increased. sortCommand decreases it again. */ incrRefCount(o); } - + decrRefCount(keyobj); + if (fieldlen) decrRefCount(fieldobj); return o; + +noobj: + decrRefCount(keyobj); + if (fieldlen) decrRefCount(fieldobj); + return NULL; } /* sortCompare() is used by qsort in sortCommand(). Given that qsort_r with From a1090c1193c6b3162d166c8c85aca86a39fb6c3e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Apr 2012 16:28:50 +0200 Subject: [PATCH 0157/1752] Added test for SORT corner case: pattern ending with just "->". --- tests/unit/sort.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/sort.tcl b/tests/unit/sort.tcl index ba4122540e4..5a181641cff 100644 --- a/tests/unit/sort.tcl +++ b/tests/unit/sort.tcl @@ -190,6 +190,13 @@ start_server { r sort myset by score:* } {a aa aaa azz b c d e f g h i l m n o p q r s t u v z} + test "SORT GET with pattern ending with just -> does not get hash field" { + r del mylist + r lpush mylist a + r set x:a-> 100 + r sort mylist by num get x:*-> + } {100} + tags {"slow"} { set num 100 set res [create_random_dataset $num lpush] From 727d6dd52a3e778403d2ef66a516498ff25f3c02 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 11:31:24 +0200 Subject: [PATCH 0158/1752] Two small fixes to maxclients handling. 1) Don't accept maxclients set to < 0 2) Allow maxclients < 1024, it is useful for testing. --- src/config.c | 3 +++ src/redis.c | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 4d35521d742..c2ea5b76816 100644 --- a/src/config.c +++ b/src/config.c @@ -155,6 +155,9 @@ void loadServerConfigFromString(char *config) { loadServerConfig(argv[1],NULL); } else if (!strcasecmp(argv[0],"maxclients") && argc == 2) { server.maxclients = atoi(argv[1]); + if (server.maxclients < 1) { + err = "Invalid max clients limit"; goto loaderr; + } } else if (!strcasecmp(argv[0],"maxmemory") && argc == 2) { server.maxmemory = memtoll(argv[1],NULL); } else if (!strcasecmp(argv[0],"maxmemory-policy") && argc == 2) { diff --git a/src/redis.c b/src/redis.c index 86cf0d957e7..46ae3ffa6a9 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1135,7 +1135,6 @@ void adjustOpenFilesLimit(void) { rlim_t maxfiles = server.maxclients+32; struct rlimit limit; - if (maxfiles < 1024) maxfiles = 1024; if (getrlimit(RLIMIT_NOFILE,&limit) == -1) { redisLog(REDIS_WARNING,"Unable to obtain the current NOFILE limit (%s), assuming 1024 and setting the max clients configuration accordingly.", strerror(errno)); From 212bb9ca2e62019b3fc53ca080ab1d9884f81bec Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 11:34:18 +0200 Subject: [PATCH 0159/1752] More robust maxclients test. --- tests/unit/limits.tcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/limits.tcl b/tests/unit/limits.tcl index f622e1b9155..b37ea9b0f5b 100644 --- a/tests/unit/limits.tcl +++ b/tests/unit/limits.tcl @@ -4,7 +4,9 @@ start_server {tags {"limits"} overrides {maxclients 10}} { catch { while {$c < 50} { incr c - redis_deferring_client + set rd [redis_deferring_client] + $rd ping + $rd read after 100 } } e From ae55245d1c88361bb05934da1ad62469885ccca7 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 11:37:14 +0200 Subject: [PATCH 0160/1752] Marginally cleaner lookupKeyByPattern() implementation. just fieldobj itself as sentinel of the fact a field object is used or not, instead of using the filed length, that may be confusing both for people and for the compiler emitting a warning. --- src/sort.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/sort.c b/src/sort.c index ff655c7e71c..c1ed5517f2f 100644 --- a/src/sort.c +++ b/src/sort.c @@ -28,7 +28,7 @@ redisSortOperation *createSortOperation(int type, robj *pattern) { robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { char *p, *f, *k; sds spat, ssub; - robj *keyobj, *fieldobj, *o; + robj *keyobj, *fieldobj = NULL, *o; int prefixlen, sublen, postfixlen, fieldlen; /* If the pattern is "#" return the substitution object itself in order @@ -76,7 +76,7 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { o = lookupKeyRead(db,keyobj); if (o == NULL) goto noobj; - if (fieldlen > 0) { + if (fieldobj) { if (o->type != REDIS_HASH) goto noobj; /* Retrieve value from hash by the field name. This operation @@ -90,7 +90,7 @@ robj *lookupKeyByPattern(redisDb *db, robj *pattern, robj *subst) { incrRefCount(o); } decrRefCount(keyobj); - if (fieldlen) decrRefCount(fieldobj); + if (fieldobj) decrRefCount(fieldobj); return o; noobj: From 8d12645569f931849bfe05ee145962d4d334e61f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 17:17:53 +0200 Subject: [PATCH 0161/1752] Test LINDEX out of range index. --- tests/unit/type/list.tcl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 970e3ee7fb8..9a63d45137f 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -26,6 +26,7 @@ start_server { assert_equal c [r lindex myziplist2 0] assert_equal b [r lindex myziplist2 1] assert_equal a [r lindex myziplist2 2] + assert_equal {} [r lindex myziplist2 3] assert_encoding ziplist myziplist2 } @@ -39,6 +40,7 @@ start_server { assert_equal $largevalue(linkedlist) [r lindex mylist1 0] assert_equal b [r lindex mylist1 1] assert_equal c [r lindex mylist1 2] + assert_equal {} [r lindex mylist1 3] # first rpush then lpush assert_equal 1 [r rpush mylist2 $largevalue(linkedlist)] @@ -49,6 +51,7 @@ start_server { assert_equal c [r lindex mylist2 0] assert_equal b [r lindex mylist2 1] assert_equal $largevalue(linkedlist) [r lindex mylist2 2] + assert_equal {} [r lindex mylist2 3] } test {Variadic RPUSH/LPUSH} { From a00fcaa671d23235697ba25bbc73dc845bc07fe5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 17:22:14 +0200 Subject: [PATCH 0162/1752] Test LINSERT syntax error. --- tests/unit/type/list.tcl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 9a63d45137f..035cf3de5dc 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -399,6 +399,11 @@ start_server { } } + test {LINSERT raise error on bad syntax} { + catch {[r linsert xlist aft3r aa 42]} e + set e + } {*ERR*syntax*error*} + test {LPUSHX, RPUSHX convert from ziplist to list} { set large $largevalue(linkedlist) From bec200ec399690fb7203257732cd2a6b47bdb962 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 17:32:48 +0200 Subject: [PATCH 0163/1752] Explicit RPOP/LPOP tests. --- tests/unit/type/list.tcl | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 035cf3de5dc..85dde5690a4 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -7,7 +7,7 @@ start_server { } { source "tests/unit/type/list-common.tcl" - test {LPUSH, RPUSH, LLENGTH, LINDEX - ziplist} { + test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - ziplist} { # first lpush then rpush assert_equal 1 [r lpush myziplist1 a] assert_equal 2 [r rpush myziplist1 b] @@ -16,6 +16,9 @@ start_server { assert_equal a [r lindex myziplist1 0] assert_equal b [r lindex myziplist1 1] assert_equal c [r lindex myziplist1 2] + assert_equal {} [r lindex myziplist2 3] + assert_equal c [r rpop myziplist1] + assert_equal a [r lpop myziplist1] assert_encoding ziplist myziplist1 # first rpush then lpush @@ -27,10 +30,12 @@ start_server { assert_equal b [r lindex myziplist2 1] assert_equal a [r lindex myziplist2 2] assert_equal {} [r lindex myziplist2 3] + assert_equal a [r rpop myziplist2] + assert_equal c [r lpop myziplist2] assert_encoding ziplist myziplist2 } - test {LPUSH, RPUSH, LLENGTH, LINDEX - regular list} { + test {LPUSH, RPUSH, LLENGTH, LINDEX, LPOP - regular list} { # first lpush then rpush assert_equal 1 [r lpush mylist1 $largevalue(linkedlist)] assert_encoding linkedlist mylist1 @@ -41,6 +46,8 @@ start_server { assert_equal b [r lindex mylist1 1] assert_equal c [r lindex mylist1 2] assert_equal {} [r lindex mylist1 3] + assert_equal c [r rpop mylist1] + assert_equal $largevalue(linkedlist) [r lpop mylist1] # first rpush then lpush assert_equal 1 [r rpush mylist2 $largevalue(linkedlist)] @@ -52,8 +59,14 @@ start_server { assert_equal b [r lindex mylist2 1] assert_equal $largevalue(linkedlist) [r lindex mylist2 2] assert_equal {} [r lindex mylist2 3] + assert_equal $largevalue(linkedlist) [r rpop mylist2] + assert_equal c [r lpop mylist2] } + test {R/LPOP against empty list} { + r lpop non-existing-list + } {} + test {Variadic RPUSH/LPUSH} { r del mylist assert_equal 4 [r lpush mylist a b c d] From 60ef787efad8be46afa29ec751532e3ea7ebd70c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 17:38:02 +0200 Subject: [PATCH 0164/1752] Document mostly dead code in RPOPLPUSH implementation. --- src/t_list.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/t_list.c b/src/t_list.c index 6a16a632019..ca03916b953 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -699,6 +699,8 @@ void rpoplpushCommand(redisClient *c) { checkType(c,sobj,REDIS_LIST)) return; if (listTypeLength(sobj) == 0) { + /* This may only happen after loading very old RDB files. Recent + * versions of Redis delete keys of empty lists. */ addReply(c,shared.nullbulk); } else { robj *dobj = lookupKeyWrite(c->db,c->argv[2]); From 24982f2bbcbff24b0101dd14090c9c4f0bfe9420 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 17:56:17 +0200 Subject: [PATCH 0165/1752] New hash fuzzing test. --- tests/unit/type/hash.tcl | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index 47e10caab0d..950805d1bdd 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -395,4 +395,28 @@ start_server {tags {"hash"}} { r hset hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk b r hget hash kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk } {b} + + foreach size {10 512} { + test "Hash fuzzing - $size fields" { + for {set times 0} {$times < 10} {incr times} { + catch {unset hash} + array set hash {} + r del hash + + # Create + for {set j 0} {$j < $size} {incr j} { + set field [randomValue] + set value [randomValue] + r hset hash $field $value + set hash($field) $value + } + + # Verify + foreach {k v} [array get hash] { + assert_equal $v [r hget hash $k] + } + assert_equal [array size hash] [r hlen hash] + } + } + } } From ff5e31f74bbe41fd08f5e975ee1cde9bf7e44cce Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 18:00:12 +0200 Subject: [PATCH 0166/1752] Added an SMOVE test where src and dest key are the same. --- tests/unit/type/set.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index bdd1f9bfa11..45239aa9e35 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -317,6 +317,13 @@ start_server { assert_error "ERR*wrong kind*" {r smove myset2 x foo} } + test "SMOVE with identical source and destination" { + r del set + r sadd set a b c + r smove set set b + lsort [r smembers set] + } {a b c} + tags {slow} { test {intsets implementation stress testing} { for {set j 0} {$j < 20} {incr j} { From eb624e3416f8636f15b83349390d5c0dcbeb8b58 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 18:05:02 +0200 Subject: [PATCH 0167/1752] Test SINTER with non existing key. --- tests/unit/type/set.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index 45239aa9e35..e46784c88b8 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -216,6 +216,13 @@ start_server { assert_error "ERR*wrong kind*" {r sunion key1 noset} } + test "SINTER should handle non existing key as empty" { + r del set1 set2 set3 + r sadd set1 a b c + r sadd set2 b c d + r sinter set1 set2 set3 + } {} + test "SINTERSTORE against non existing keys should delete dstkey" { r set setres xxx assert_equal 0 [r sinterstore setres foo111 bar222] From 7a2065ef3340b4894e5031548f8dd3b04bd0236a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 18:10:48 +0200 Subject: [PATCH 0168/1752] Test SINTER against same integer elements, but different set encoding. --- tests/unit/type/set.tcl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index e46784c88b8..ec412b40902 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -223,6 +223,16 @@ start_server { r sinter set1 set2 set3 } {} + test "SINTER with same integer elements but different encoding" { + r del set1 set2 + r sadd set1 1 2 3 + r sadd set2 1 2 3 a + r srem set2 a + assert_encoding intset set1 + assert_encoding hashtable set2 + lsort [r sinter set1 set2] + } {1 2 3} + test "SINTERSTORE against non existing keys should delete dstkey" { r set setres xxx assert_equal 0 [r sinterstore setres foo111 bar222] From 5c45ae1f7b44c01b3cc7240f2529ebcd932a92ef Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 18:13:31 +0200 Subject: [PATCH 0169/1752] Test SDIFF with first set empty. --- tests/unit/type/set.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index ec412b40902..f4f2837351f 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -206,6 +206,13 @@ start_server { } } + test "SDIFF with first set empty" { + r del set1 set2 set3 + r sadd set2 1 2 3 4 + r sadd set3 a b c d + r sdiff set1 set2 set3 + } {} + test "SINTER against non-set should throw error" { r set key1 x assert_error "ERR*wrong kind*" {r sinter key1 noset} From e10768518cdfc5df532471711f1f97ed32ad4bc5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 20:33:02 +0200 Subject: [PATCH 0170/1752] redis-cli --bigkeys --- src/redis-cli.c | 93 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index bdaa3964217..4a936133a99 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -69,6 +69,7 @@ static struct config { int cluster_mode; int cluster_reissue_command; int slave_mode; + int bigkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; int output; /* output mode, see OUTPUT_* defines */ @@ -655,6 +656,8 @@ static int parseOptions(int argc, char **argv) { config.latency_mode = 1; } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; + } else if (!strcmp(argv[i],"--bigkeys")) { + config.bigkeys = 1; } else if (!strcmp(argv[i],"--eval") && !lastarg) { config.eval = argv[++i]; } else if (!strcmp(argv[i],"-c")) { @@ -711,6 +714,7 @@ static void usage() { " --raw Use raw formatting for replies (default when STDOUT is not a tty)\n" " --latency Enter a special mode continuously sampling latency.\n" " --slave Simulate a slave showing commands received from the master.\n" +" --bigkeys Sample Redis keys looking for big keys.\n" " --eval Send an EVAL command using the Lua script at .\n" " --help Output this help and exit\n" " --version Output version and exit\n" @@ -964,6 +968,87 @@ static void slaveMode(void) { while (cliReadReply(0) == REDIS_OK); } +#define TYPE_STRING 0 +#define TYPE_LIST 1 +#define TYPE_SET 2 +#define TYPE_HASH 3 +#define TYPE_ZSET 4 + +static void findBigKeys(void) { + unsigned long long biggest[5] = {0,0,0,0,0}; + unsigned long long samples = 0; + redisReply *reply1, *reply2, *reply3; + char *sizecmd, *typename[] = {"string","list","set","hash","zset"}; + int type; + + printf("\n# Press ctrl+c when you have had enough of it... :)\n"); + printf("# You can use -i 0.1 to sleep 0.1 sec every 100 sampled keys\n"); + printf("# in order to reduce server load (usually not needed).\n\n"); + while(1) { + /* Sample with RANDOMKEY */ + reply1 = redisCommand(context,"RANDOMKEY"); + if (reply1 == NULL) { + fprintf(stderr,"\nI/O error\n"); + exit(1); + } else if (reply1->type == REDIS_REPLY_ERROR) { + fprintf(stderr, "RANDOMKEY error: %s\n", + reply1->str); + exit(1); + } + /* Get the key type */ + reply2 = redisCommand(context,"TYPE %s",reply1->str); + assert(reply2 && reply2->type == REDIS_REPLY_STATUS); + samples++; + + /* Get the key "size" */ + if (!strcmp(reply2->str,"string")) { + sizecmd = "STRLEN"; + type = TYPE_STRING; + } else if (!strcmp(reply2->str,"list")) { + sizecmd = "LLEN"; + type = TYPE_LIST; + } else if (!strcmp(reply2->str,"set")) { + sizecmd = "SCARD"; + type = TYPE_SET; + } else if (!strcmp(reply2->str,"hash")) { + sizecmd = "HLEN"; + type = TYPE_HASH; + } else if (!strcmp(reply2->str,"zset")) { + sizecmd = "ZCARD"; + type = TYPE_ZSET; + } else if (!strcmp(reply2->str,"none")) { + freeReplyObject(reply1); + freeReplyObject(reply2); + freeReplyObject(reply3); + continue; + } else { + fprintf(stderr, "Unknown key type '%s' for key '%s'\n", + reply2->str, reply1->str); + exit(1); + } + + reply3 = redisCommand(context,"%s %s", sizecmd, reply1->str); + if (reply3 && reply3->type == REDIS_REPLY_INTEGER) { + if (biggest[type] < reply3->integer) { + printf("Biggest %s so far: %s, size: %llu\n", + typename[type], reply1->str, + (unsigned long long) reply3->integer); + biggest[type] = reply3->integer; + } + } + + if ((samples % 1000000) == 0) + printf("(%llu keys sampled)\n", samples); + + if ((samples % 100) == 0 && config.interval) + usleep(config.interval); + + freeReplyObject(reply1); + freeReplyObject(reply2); + if (reply3) freeReplyObject(reply3); + } +} + int main(int argc, char **argv) { int firstarg; @@ -979,6 +1064,8 @@ int main(int argc, char **argv) { config.pubsub_mode = 0; config.latency_mode = 0; config.cluster_mode = 0; + config.slave_mode = 0; + config.bigkeys = 0; config.stdinarg = 0; config.auth = NULL; config.eval = NULL; @@ -1005,6 +1092,12 @@ int main(int argc, char **argv) { slaveMode(); } + /* Find big keys */ + if (config.bigkeys) { + cliConnect(0); + findBigKeys(); + } + /* Start interactive mode when no command is provided */ if (argc == 0 && !config.eval) { /* Note that in repl mode we don't abort on connection error. From 0122cc4f42792066b2c4e8c0bcc500b5bfc00f42 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 20:53:37 +0200 Subject: [PATCH 0171/1752] redis-cli --bigkeys output modified to be simpler to read.. --- src/redis-cli.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 4a936133a99..3e0c58951a5 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1030,7 +1030,7 @@ static void findBigKeys(void) { reply3 = redisCommand(context,"%s %s", sizecmd, reply1->str); if (reply3 && reply3->type == REDIS_REPLY_INTEGER) { if (biggest[type] < reply3->integer) { - printf("Biggest %s so far: %s, size: %llu\n", + printf("[%6s] %s | biggest so far with size %llu\n", typename[type], reply1->str, (unsigned long long) reply3->integer); biggest[type] = reply3->integer; From 1d82bbd4329d53fa525a3d1e00005d53641e1d0d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 23:41:00 +0200 Subject: [PATCH 0172/1752] cr16.c removed from 2.6 branch, was not used. --- src/Makefile | 2 +- src/crc16.c | 87 ---------------------------------------------------- 2 files changed, 1 insertion(+), 88 deletions(-) delete mode 100644 src/crc16.c diff --git a/src/Makefile b/src/Makefile index f6432278713..a6e8e2f94f4 100644 --- a/src/Makefile +++ b/src/Makefile @@ -96,7 +96,7 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN endif REDIS_SERVER_NAME= redis-server -REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o crc16.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o +REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o REDIS_CLI_NAME= redis-cli REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o REDIS_BENCHMARK_NAME= redis-benchmark diff --git a/src/crc16.c b/src/crc16.c deleted file mode 100644 index 6ba7f2448b1..00000000000 --- a/src/crc16.c +++ /dev/null @@ -1,87 +0,0 @@ -#include "redis.h" - -/* - * Copyright 2001-2010 Georges Menie (www.menie.org) - * Copyright 2010 Salvatore Sanfilippo (adapted to Redis coding style) - * All rights reserved. - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of the University of California, Berkeley nor the - * names of its contributors may be used to endorse or promote products - * derived from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY - * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; - * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND - * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -/* CRC16 implementation acording to CCITT standards. - * - * Note by @antirez: this is actually the XMODEM CRC 16 algorithm, using the - * following parameters: - * - * Name : "XMODEM", also known as "ZMODEM", "CRC-16/ACORN" - * Width : 16 bit - * Poly : 1021 (That is actually x^16 + x^12 + x^5 + 1) - * Initialization : 0000 - * Reflect Input byte : False - * Reflect Output CRC : False - * Xor constant to output CRC : 0000 - * Output for "123456789" : 31C3 - */ - -static const uint16_t crc16tab[256]= { - 0x0000,0x1021,0x2042,0x3063,0x4084,0x50a5,0x60c6,0x70e7, - 0x8108,0x9129,0xa14a,0xb16b,0xc18c,0xd1ad,0xe1ce,0xf1ef, - 0x1231,0x0210,0x3273,0x2252,0x52b5,0x4294,0x72f7,0x62d6, - 0x9339,0x8318,0xb37b,0xa35a,0xd3bd,0xc39c,0xf3ff,0xe3de, - 0x2462,0x3443,0x0420,0x1401,0x64e6,0x74c7,0x44a4,0x5485, - 0xa56a,0xb54b,0x8528,0x9509,0xe5ee,0xf5cf,0xc5ac,0xd58d, - 0x3653,0x2672,0x1611,0x0630,0x76d7,0x66f6,0x5695,0x46b4, - 0xb75b,0xa77a,0x9719,0x8738,0xf7df,0xe7fe,0xd79d,0xc7bc, - 0x48c4,0x58e5,0x6886,0x78a7,0x0840,0x1861,0x2802,0x3823, - 0xc9cc,0xd9ed,0xe98e,0xf9af,0x8948,0x9969,0xa90a,0xb92b, - 0x5af5,0x4ad4,0x7ab7,0x6a96,0x1a71,0x0a50,0x3a33,0x2a12, - 0xdbfd,0xcbdc,0xfbbf,0xeb9e,0x9b79,0x8b58,0xbb3b,0xab1a, - 0x6ca6,0x7c87,0x4ce4,0x5cc5,0x2c22,0x3c03,0x0c60,0x1c41, - 0xedae,0xfd8f,0xcdec,0xddcd,0xad2a,0xbd0b,0x8d68,0x9d49, - 0x7e97,0x6eb6,0x5ed5,0x4ef4,0x3e13,0x2e32,0x1e51,0x0e70, - 0xff9f,0xefbe,0xdfdd,0xcffc,0xbf1b,0xaf3a,0x9f59,0x8f78, - 0x9188,0x81a9,0xb1ca,0xa1eb,0xd10c,0xc12d,0xf14e,0xe16f, - 0x1080,0x00a1,0x30c2,0x20e3,0x5004,0x4025,0x7046,0x6067, - 0x83b9,0x9398,0xa3fb,0xb3da,0xc33d,0xd31c,0xe37f,0xf35e, - 0x02b1,0x1290,0x22f3,0x32d2,0x4235,0x5214,0x6277,0x7256, - 0xb5ea,0xa5cb,0x95a8,0x8589,0xf56e,0xe54f,0xd52c,0xc50d, - 0x34e2,0x24c3,0x14a0,0x0481,0x7466,0x6447,0x5424,0x4405, - 0xa7db,0xb7fa,0x8799,0x97b8,0xe75f,0xf77e,0xc71d,0xd73c, - 0x26d3,0x36f2,0x0691,0x16b0,0x6657,0x7676,0x4615,0x5634, - 0xd94c,0xc96d,0xf90e,0xe92f,0x99c8,0x89e9,0xb98a,0xa9ab, - 0x5844,0x4865,0x7806,0x6827,0x18c0,0x08e1,0x3882,0x28a3, - 0xcb7d,0xdb5c,0xeb3f,0xfb1e,0x8bf9,0x9bd8,0xabbb,0xbb9a, - 0x4a75,0x5a54,0x6a37,0x7a16,0x0af1,0x1ad0,0x2ab3,0x3a92, - 0xfd2e,0xed0f,0xdd6c,0xcd4d,0xbdaa,0xad8b,0x9de8,0x8dc9, - 0x7c26,0x6c07,0x5c64,0x4c45,0x3ca2,0x2c83,0x1ce0,0x0cc1, - 0xef1f,0xff3e,0xcf5d,0xdf7c,0xaf9b,0xbfba,0x8fd9,0x9ff8, - 0x6e17,0x7e36,0x4e55,0x5e74,0x2e93,0x3eb2,0x0ed1,0x1ef0 -}; - -uint16_t crc16(const char *buf, int len) { - int counter; - uint16_t crc = 0; - for (counter = 0; counter < len; counter++) - crc = (crc<<8) ^ crc16tab[((crc>>8) ^ *buf++)&0x00FF]; - return crc; -} From c3312760fea71455a985f6c3197e09561386297c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 23:50:16 +0200 Subject: [PATCH 0173/1752] Tests for scripting PRNG. --- tests/unit/scripting.tcl | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 009c1347d38..ed58bca44a1 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -251,6 +251,26 @@ start_server {tags {"scripting"}} { lappend res [r eval $decr_if_gt 1 foo 2] set res } {4 3 2 2 2} + + test {Scripting engine resets PRNG at every script execution} { + set rand1 [r eval {return tostring(math.random())} 0] + set rand2 [r eval {return tostring(math.random())} 0] + assert_equal $rand1 $rand2 + } + + test {Scripting engine PRNG can be seeded correctly} { + set rand1 [r eval { + math.randomseed(ARGV[1]); return tostring(math.random()) + } 0 10] + set rand2 [r eval { + math.randomseed(ARGV[1]); return tostring(math.random()) + } 0 10] + set rand3 [r eval { + math.randomseed(ARGV[1]); return tostring(math.random()) + } 0 20] + assert_equal $rand1 $rand2 + assert {$rand2 ne $rand3} + } } start_server {tags {"scripting repl"}} { From d54943b76d97eae8954a89ca1a259f86ea6a8e32 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 18 Apr 2012 23:56:07 +0200 Subject: [PATCH 0174/1752] Currenly not used code in dict.c commented out. --- src/dict.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/src/dict.c b/src/dict.c index 6fd4584eede..2835de5668f 100644 --- a/src/dict.c +++ b/src/dict.c @@ -633,6 +633,21 @@ void dictEmpty(dict *d) { d->iterators = 0; } +void dictEnableResize(void) { + dict_can_resize = 1; +} + +void dictDisableResize(void) { + dict_can_resize = 0; +} + +#if 0 + +The following is code that we don't use for Redis currently, but that is part +of the library. + +/* ----------------------- Debugging ------------------------*/ + #define DICT_STATS_VECTLEN 50 static void _dictPrintStatsHt(dictht *ht) { unsigned long i, slots = 0, chainlen, maxchainlen = 0; @@ -686,20 +701,6 @@ void dictPrintStats(dict *d) { } } -void dictEnableResize(void) { - dict_can_resize = 1; -} - -void dictDisableResize(void) { - dict_can_resize = 0; -} - -#if 0 - -/* The following are just example hash table types implementations. - * Not useful for Redis so they are commented out. - */ - /* ----------------------- StringCopy Hash Table Type ------------------------*/ static unsigned int _dictStringCopyHTHashFunction(const void *key) From ca577d162ab6bf4c241c53f89ff5cda6131d345d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Apr 2012 23:35:15 +0200 Subject: [PATCH 0175/1752] SHUTDOWN NOSAVE now can stop a non returning script. Issue #466. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 46ae3ffa6a9..08995115163 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1557,7 +1557,7 @@ int processCommand(redisClient *c) { /* Lua script too slow? Only allow SHUTDOWN NOSAVE and SCRIPT KILL. */ if (server.lua_timedout && - !(c->cmd->proc != shutdownCommand && + !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && !(c->cmd->proc == scriptCommand && From abfd08f5ad6afe928bf80d1f1169900167d42ab9 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Apr 2012 23:49:33 +0200 Subject: [PATCH 0176/1752] New tests related to scripts max execution time. --- tests/unit/scripting.tcl | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index ed58bca44a1..a60c65b437d 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -273,6 +273,44 @@ start_server {tags {"scripting"}} { } } +# Start a new server since the last test in this stanza will kill the +# instance at all. +start_server {tags {"scripting"}} { + test {Timedout read-only scripts can be killed by SCRIPT KILL} { + set rd [redis_deferring_client] + r config set lua-time-limit 10 + $rd eval {while true do end} 0 + after 200 + catch {r ping} e + assert_match {BUSY*} $e + r script kill + assert_equal [r ping] "PONG" + } + + test {Timedout scripts that modified data can't be killed by SCRIPT KILL} { + set rd [redis_deferring_client] + r config set lua-time-limit 10 + $rd eval {redis.call('set','x','y'); while true do end} 0 + after 200 + catch {r ping} e + assert_match {BUSY*} $e + catch {r script kill} e + assert_match {ERR*} $e + catch {r ping} e + assert_match {BUSY*} $e + } + + test {SHUTDOWN NOSAVE can kill a timedout script anyway} { + # The server sould be still unresponding to normal commands. + catch {r ping} e + assert_match {BUSY*} $e + catch {r shutdown nosave} + # Make sure the server was killed + catch {set rd [redis_deferring_client]} e + assert_match {*connection refused*} $e + } +} + start_server {tags {"scripting repl"}} { start_server {} { test {Before the slave connects we issue an EVAL command} { From 30e89410d41cc62398c15ad7c60b4c3f9f03460d Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 21 Apr 2012 12:08:26 +0200 Subject: [PATCH 0177/1752] README now makes clear that our support for solaris derived systems is "best effort". --- README | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README b/README index bba2439c339..1c3f574678d 100644 --- a/README +++ b/README @@ -7,6 +7,13 @@ documentation at http://redis.io Building Redis -------------- +Redis can be compiled and used on Linux, OSX, OpenBSD, NetBSD, FreeBSD. +We support big endian and little endian architectures. + +It may compile on Solaris derived systems (for instance SmartOS) but our +support for this platform is "best effort" and Redis is not guaranteed to +work as well as in Linux, OSX, and *BSD there. + It is as simple as: % make From c11a01a030430f1bbb64e50a95dc172538acd1e0 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 20 Apr 2012 00:04:07 +0200 Subject: [PATCH 0178/1752] redis.conf AOF section comments improved. --- redis.conf | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/redis.conf b/redis.conf index 85220b80deb..bcad01e003f 100644 --- a/redis.conf +++ b/redis.conf @@ -298,21 +298,23 @@ slave-read-only yes ############################## APPEND ONLY MODE ############################### -# By default Redis asynchronously dumps the dataset on disk. If you can live -# with the idea that the latest records will be lost if something like a crash -# happens this is the preferred way to run Redis. If instead you care a lot -# about your data and don't want to that a single record can get lost you should -# enable the append only mode: when this mode is enabled Redis will append -# every write operation received in the file appendonly.aof. This file will -# be read on startup in order to rebuild the full dataset in memory. -# -# Note that you can have both the async dumps and the append only file if you -# like (you have to comment the "save" statements above to disable the dumps). -# Still if append only mode is enabled Redis will load the data from the -# log file at startup ignoring the dump.rdb file. -# -# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append -# log file in background when it gets too big. +# By default Redis asynchronously dumps the dataset on disk. This mode is +# good enough in many applications, but an issue with the Redis process or +# a power outage may result into a few minutes of writes lost (depending on +# the configured save points). +# +# The Append Only File is an alternative persistence mode that provides +# much better durability. For instance using the default data fsync policy +# (see later in the config file) Redis can lose just one second of writes in a +# dramatic event like a server power outage, or a single write if something +# wrong with the Redis process itself happens, but the operating system is +# still running correctly. +# +# AOF and RDB persistence can be enabled at the same time without problems. +# If the AOF is enabled on startup Redis will load the AOF, that is the file +# with the better durability guarantees. +# +# Please check http://redis.io/topics/persistence for more information. appendonly no @@ -327,7 +329,7 @@ appendonly no # # no: don't fsync, just let the OS flush the data when it wants. Faster. # always: fsync after every write to the append only log . Slow, Safest. -# everysec: fsync only if one second passed since the last fsync. Compromise. +# everysec: fsync only one time every second. Compromise. # # The default is "everysec" that's usually the right compromise between # speed and data safety. It's up to you to understand if you can relax this to @@ -337,6 +339,9 @@ appendonly no # or on the contrary, use "always" that's very slow but a bit safer than # everysec. # +# More details please check the following article: +# http://antirez.com/post/redis-persistence-demystified.html +# # If unsure, use "everysec". # appendfsync always From 590d55a2064bdb8363145e8a4826a3cf6c6d420e Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 21 Apr 2012 19:20:03 +0200 Subject: [PATCH 0179/1752] Limit memory used by big SLOWLOG entries. Two limits are added: 1) Up to SLOWLOG_ENTRY_MAX_ARGV arguments are logged. 2) Up to SLOWLOG_ENTRY_MAX_STRING bytes per argument are logged. 3) slowlog-max-len is set to 128 by default (was 1024). The number of remaining arguments / bytes is logged in the entry so that the user can understand better the nature of the logged command. --- redis.conf | 2 +- src/redis.h | 2 +- src/slowlog.c | 35 +++++++++++++++++++++++++++++------ src/slowlog.h | 3 +++ tests/unit/slowlog.tcl | 17 +++++++++++++++++ 5 files changed, 51 insertions(+), 8 deletions(-) diff --git a/redis.conf b/redis.conf index bcad01e003f..f5e15f69d22 100644 --- a/redis.conf +++ b/redis.conf @@ -428,7 +428,7 @@ slowlog-log-slower-than 10000 # There is no limit to this length. Just be aware that it will consume memory. # You can reclaim memory used by the slow log with SLOWLOG RESET. -slowlog-max-len 1024 +slowlog-max-len 128 ############################### ADVANCED CONFIG ############################### diff --git a/src/redis.h b/src/redis.h index 9702010e2de..2084dfd2a6c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -52,7 +52,7 @@ #define REDIS_AOF_REWRITE_MIN_SIZE (1024*1024) #define REDIS_AOF_REWRITE_ITEMS_PER_CMD 64 #define REDIS_SLOWLOG_LOG_SLOWER_THAN 10000 -#define REDIS_SLOWLOG_MAX_LEN 64 +#define REDIS_SLOWLOG_MAX_LEN 128 #define REDIS_MAX_CLIENTS 10000 #define REDIS_REPL_TIMEOUT 60 diff --git a/src/slowlog.c b/src/slowlog.c index cfd66dc634d..53c44a017dd 100644 --- a/src/slowlog.c +++ b/src/slowlog.c @@ -16,13 +16,36 @@ * this function. */ slowlogEntry *slowlogCreateEntry(robj **argv, int argc, long long duration) { slowlogEntry *se = zmalloc(sizeof(*se)); - int j; + int j, slargc = argc; + + if (slargc > SLOWLOG_ENTRY_MAX_ARGC) slargc = SLOWLOG_ENTRY_MAX_ARGC; + se->argc = slargc; + se->argv = zmalloc(sizeof(robj*)*slargc); + for (j = 0; j < slargc; j++) { + /* Logging too many arguments is a useless memory waste, so we stop + * at SLOWLOG_ENTRY_MAX_ARGC, but use the last argument to specify + * how many remaining arguments there were in the original command. */ + if (slargc != argc && j == slargc-1) { + se->argv[j] = createObject(REDIS_STRING, + sdscatprintf(sdsempty(),"... (%d more arguments)", + argc-slargc+1)); + } else { + /* Trim too long strings as well... */ + if (argv[j]->type == REDIS_STRING && + argv[j]->encoding == REDIS_ENCODING_RAW && + sdslen(argv[j]->ptr) > SLOWLOG_ENTRY_MAX_STRING) + { + sds s = sdsnewlen(argv[j]->ptr, SLOWLOG_ENTRY_MAX_STRING); - se->argc = argc; - se->argv = zmalloc(sizeof(robj*)*argc); - for (j = 0; j < argc; j++) { - se->argv[j] = argv[j]; - incrRefCount(argv[j]); + s = sdscatprintf(s,"... (%lu more bytes)", + (unsigned long) + sdslen(argv[j]->ptr) - SLOWLOG_ENTRY_MAX_STRING); + se->argv[j] = createObject(REDIS_STRING,s); + } else { + se->argv[j] = argv[j]; + incrRefCount(argv[j]); + } + } } se->time = time(NULL); se->duration = duration; diff --git a/src/slowlog.h b/src/slowlog.h index bad770db49f..bcc961cc9c5 100644 --- a/src/slowlog.h +++ b/src/slowlog.h @@ -1,3 +1,6 @@ +#define SLOWLOG_ENTRY_MAX_ARGC 32 +#define SLOWLOG_ENTRY_MAX_STRING 128 + /* This structure defines an entry inside the slow log list */ typedef struct slowlogEntry { robj **argv; diff --git a/tests/unit/slowlog.tcl b/tests/unit/slowlog.tcl index 55a71e98598..2216e925a70 100644 --- a/tests/unit/slowlog.tcl +++ b/tests/unit/slowlog.tcl @@ -38,4 +38,21 @@ start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} { assert_equal [expr {[lindex $e 2] > 100000}] 1 assert_equal [lindex $e 3] {debug sleep 0.2} } + + test {SLOWLOG - commands with too many arguments are trimmed} { + r config set slowlog-log-slower-than 0 + r slowlog reset + r sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 + set e [lindex [r slowlog get] 0] + lindex $e 3 + } {sadd set 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 {... (2 more arguments)}} + + test {SLOWLOG - too long arguments are trimmed} { + r config set slowlog-log-slower-than 0 + r slowlog reset + set arg [string repeat A 129] + r sadd set foo $arg + set e [lindex [r slowlog get] 0] + lindex $e 3 + } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} } From e337b260507d12f4a6312aa6c34c2f8a16a732b8 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 21 Apr 2012 21:49:21 +0200 Subject: [PATCH 0180/1752] Even inside #if 0 comments are comments. --- src/dict.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dict.c b/src/dict.c index 2835de5668f..69656734c60 100644 --- a/src/dict.c +++ b/src/dict.c @@ -643,8 +643,8 @@ void dictDisableResize(void) { #if 0 -The following is code that we don't use for Redis currently, but that is part -of the library. +/* The following is code that we don't use for Redis currently, but that is part +of the library. */ /* ----------------------- Debugging ------------------------*/ From 7d6bf7956eec198ef906ddb05b00a77d3cf6c427 Mon Sep 17 00:00:00 2001 From: Michael Schlenker Date: Tue, 17 Apr 2012 22:20:54 +0200 Subject: [PATCH 0181/1752] Replace unnecessary calls to echo and cat Tcl's exec can send data to stdout itself, no need to call cat/echo for that usually. --- tests/integration/aof.tcl | 6 +++--- tests/support/server.tcl | 2 +- tests/unit/aofrw.tcl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index 9c93b6a6ed6..ebf9cb56498 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -35,7 +35,7 @@ tags {"aof"} { set pattern "*Unexpected end of file reading the append only file*" set retry 10 while {$retry} { - set result [exec cat [dict get $srv stdout] | tail -n1] + set result [exec tail -n1 < [dict get $srv stdout]] if {[string match $pattern $result]} { break } @@ -59,7 +59,7 @@ tags {"aof"} { set pattern "*Bad file format reading the append only file*" set retry 10 while {$retry} { - set result [exec cat [dict get $srv stdout] | tail -n1] + set result [exec tail -n1 < [dict get $srv stdout]] if {[string match $pattern $result]} { break } @@ -81,7 +81,7 @@ tags {"aof"} { } test "Short read: Utility should be able to fix the AOF" { - set result [exec echo y | src/redis-check-aof --fix $aof_path] + set result [exec src/redis-check-aof --fix $aof_path << "y\n"] assert_match "*Successfully truncated AOF*" $result } diff --git a/tests/support/server.tcl b/tests/support/server.tcl index b1ab38fc108..35c1cb87076 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -252,7 +252,7 @@ proc start_server {options {code undefined}} { while 1 { # check that the server actually started and is ready for connections - if {[exec cat $stdout | grep "ready to accept" | wc -l] > 0} { + if {[exec grep "ready to accept" | wc -l < $stdout] > 0} { break } after 10 diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index e341d77b215..4716d08e60e 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -7,7 +7,7 @@ start_server {tags {"aofrw"}} { r bgrewriteaof r config set appendonly no r exec - set result [exec cat [srv 0 stdout] | tail -n1] + set result [exec tail -n1 < [srv 0 stdout] ] } {*Killing*AOF*child*} foreach d {string int} { From 537dafab8441bb691eb668cb3d4122caa9ce289f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Apr 2012 10:43:24 +0200 Subject: [PATCH 0182/1752] Remove loadfile() access from the scripting engine. --- src/scripting.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/scripting.c b/src/scripting.c index 4c7de33bf1d..a5f5683e15d 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -412,6 +412,13 @@ void luaLoadLibraries(lua_State *lua) { #endif } +/* Remove a functions that we don't want to expose to the Redis scripting + * environment. */ +void luaRemoveUnsupportedFunctions(lua_State *lua) { + lua_pushnil(lua); + lua_setglobal(lua,"loadfile"); +} + /* This function installs metamethods in the global table _G that prevent * the creation of globals accidentally. * @@ -455,7 +462,9 @@ void scriptingEnableGlobalsProtection(lua_State *lua) { * See scriptingReset() for more information. */ void scriptingInit(void) { lua_State *lua = lua_open(); + luaLoadLibraries(lua); + luaRemoveUnsupportedFunctions(lua); /* Initialize a dictionary we use to map SHAs to scripts. * This is useful for replication, as we need to replicate EVALSHA From 0a8a1e78dcc45f15833a3293ea909aecaa32ce12 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Apr 2012 10:57:43 +0200 Subject: [PATCH 0183/1752] New time limit for protocol desync test set to 30 seconds to reduce false positives. --- tests/unit/protocol.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl index 3110d3d74a1..1700e489249 100644 --- a/tests/unit/protocol.tcl +++ b/tests/unit/protocol.tcl @@ -68,7 +68,7 @@ start_server {tags {"protocol"}} { puts -nonewline $s $seq set payload [string repeat A 1024]"\n" set test_start [clock seconds] - set test_time_limit 5 + set test_time_limit 30 while 1 { if {[catch { puts -nonewline $s payload From 69b30cfcb6cdaee4a9e12eec2fb3a8a2aa093e7f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Apr 2012 17:27:39 +0200 Subject: [PATCH 0184/1752] Ziplist encoding now tested with negative integers as well. --- tests/unit/type/list-3.tcl | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/type/list-3.tcl b/tests/unit/type/list-3.tcl index 9410022fdc8..94f9a0b7976 100644 --- a/tests/unit/type/list-3.tcl +++ b/tests/unit/type/list-3.tcl @@ -29,6 +29,15 @@ start_server { set data [randomInt 4294967296] } { set data [randomInt 18446744073709551616] + } { + set data -[randomInt 65536] + if {$data eq {-0}} {set data 0} + } { + set data -[randomInt 4294967296] + if {$data eq {-0}} {set data 0} + } { + set data -[randomInt 18446744073709551616] + if {$data eq {-0}} {set data 0} } lappend l $data r rpush l $data From 38b60dea54009e909e3ce1be6751693516233e1b Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 11:07:15 +0200 Subject: [PATCH 0185/1752] Fix and refactoring of code used to get registers on crash. This fixes compilation on FreeBSD (and possibly other systems) by not using ucontext_t at all if HAVE_BACKTRACE is not defined. Also the ifdefs to get the registers are modified to explicitly test for the operating system in the first level, and the arch in the second level of nesting. --- src/debug.c | 52 ++++++++++++++++++++++++++++++++-------------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/src/debug.c b/src/debug.c index 37e6a1f72c1..8d51e959d80 100644 --- a/src/debug.c +++ b/src/debug.c @@ -398,30 +398,31 @@ void bugReportStart(void) { #ifdef HAVE_BACKTRACE static void *getMcontextEip(ucontext_t *uc) { -#if defined(__FreeBSD__) - return (void*) uc->uc_mcontext.mc_eip; -#elif defined(__dietlibc__) - return (void*) uc->uc_mcontext.eip; -#elif defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) - #if __x86_64__ +#if defined(__APPLE__) && !defined(MAC_OS_X_VERSION_10_6) + /* OSX < 10.6 */ + #if defined(__x86_64__) return (void*) uc->uc_mcontext->__ss.__rip; - #elif __i386__ + #elif defined(__i386__) return (void*) uc->uc_mcontext->__ss.__eip; - #else + #else return (void*) uc->uc_mcontext->__ss.__srr0; - #endif + #endif #elif defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6) - #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) + /* OSX >= 10.6 */ + #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) return (void*) uc->uc_mcontext->__ss.__rip; - #else + #else return (void*) uc->uc_mcontext->__ss.__eip; - #endif -#elif defined(__i386__) + #endif +#elif defined(__linux__) + /* Linux */ + #if defined(__i386__) return (void*) uc->uc_mcontext.gregs[14]; /* Linux 32 */ -#elif defined(__X86_64__) || defined(__x86_64__) + #elif defined(__X86_64__) || defined(__x86_64__) return (void*) uc->uc_mcontext.gregs[16]; /* Linux 64 */ -#elif defined(__ia64__) /* Linux IA64 */ + #elif defined(__ia64__) /* Linux IA64 */ return (void*) uc->uc_mcontext.sc_ip; + #endif #else return NULL; #endif @@ -439,8 +440,11 @@ void logStackContent(void **sp) { void logRegisters(ucontext_t *uc) { redisLog(REDIS_WARNING, "--- REGISTERS"); + +/* OSX */ #if defined(__APPLE__) && defined(MAC_OS_X_VERSION_10_6) - #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) + /* OSX AMD64 */ + #if defined(_STRUCT_X86_THREAD_STATE64) && !defined(__i386__) redisLog(REDIS_WARNING, "\n" "RAX:%016lx RBX:%016lx\nRCX:%016lx RDX:%016lx\n" @@ -471,7 +475,8 @@ void logRegisters(ucontext_t *uc) { uc->uc_mcontext->__ss.__gs ); logStackContent((void**)uc->uc_mcontext->__ss.__rsp); - #else + #else + /* OSX x86 */ redisLog(REDIS_WARNING, "\n" "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n" @@ -496,8 +501,11 @@ void logRegisters(ucontext_t *uc) { uc->uc_mcontext->__ss.__gs ); logStackContent((void**)uc->uc_mcontext->__ss.__esp); - #endif -#elif defined(__i386__) + #endif +/* Linux */ +#elif defined(__linux__) + /* Linux x86 */ + #if defined(__i386__) redisLog(REDIS_WARNING, "\n" "EAX:%08lx EBX:%08lx ECX:%08lx EDX:%08lx\n" @@ -522,7 +530,8 @@ void logRegisters(ucontext_t *uc) { uc->uc_mcontext.gregs[0] ); logStackContent((void**)uc->uc_mcontext.gregs[7]); -#elif defined(__X86_64__) || defined(__x86_64__) + #elif defined(__X86_64__) || defined(__x86_64__) + /* Linux AMD64 */ redisLog(REDIS_WARNING, "\n" "RAX:%016lx RBX:%016lx\nRCX:%016lx RDX:%016lx\n" @@ -551,6 +560,7 @@ void logRegisters(ucontext_t *uc) { uc->uc_mcontext.gregs[18] ); logStackContent((void**)uc->uc_mcontext.gregs[15]); + #endif #else redisLog(REDIS_WARNING, " Dumping of registers not supported for this OS/arch"); @@ -677,7 +687,9 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { #include void watchdogSignalHandler(int sig, siginfo_t *info, void *secret) { +#ifdef HAVE_BACKTRACE ucontext_t *uc = (ucontext_t*) secret; +#endif REDIS_NOTUSED(info); REDIS_NOTUSED(sig); sds st, log; From 9de5d4600a404f7bc809c57b8192eb983307f800 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 11:11:55 +0200 Subject: [PATCH 0186/1752] A few compiler warnings suppressed. --- src/redis-cli.c | 4 ++-- src/t_zset.c | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 3e0c58951a5..f7fe54fa893 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -890,7 +890,7 @@ static int evalMode(int argc, char **argv) { static void latencyMode(void) { redisReply *reply; - long long start, latency, min, max, tot, count = 0; + long long start, latency, min = 0, max = 0, tot = 0, count = 0; double avg; if (!context) exit(1); @@ -977,7 +977,7 @@ static void slaveMode(void) { static void findBigKeys(void) { unsigned long long biggest[5] = {0,0,0,0,0}; unsigned long long samples = 0; - redisReply *reply1, *reply2, *reply3; + redisReply *reply1, *reply2, *reply3 = NULL; char *sizecmd, *typename[] = {"string","list","set","hash","zset"}; int type; diff --git a/src/t_zset.c b/src/t_zset.c index d482d4c2f74..50ad8d433d2 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1259,7 +1259,9 @@ int zuiNext(zsetopsrc *op, zsetopval *val) { if (op->type == REDIS_SET) { iterset *it = &op->iter.set; if (op->encoding == REDIS_ENCODING_INTSET) { - if (!intsetGet(it->is.is,it->is.ii,(int64_t*)&val->ell)) + int64_t ell = val->ell; + + if (!intsetGet(it->is.is,it->is.ii,&ell)) return 0; val->score = 1.0; From ad91404a321917c25adbb0526f121e72bb4413b2 Mon Sep 17 00:00:00 2001 From: Grisha Trubetskoy Date: Fri, 20 Apr 2012 10:38:42 -0400 Subject: [PATCH 0187/1752] Add a 24bit integer to ziplists to save one byte for ints that can fit in 24 bits (thanks to antirez for catching and solving the two's compliment bug). Increment REDIS_RDB_VERSION to 6 --- src/rdb.h | 2 +- src/ziplist.c | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/rdb.h b/src/rdb.h index 2be5d9cd11b..0381c5b4c92 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -9,7 +9,7 @@ /* The current RDB version. When the format changes in a way that is no longer * backward compatible this number gets incremented. */ -#define REDIS_RDB_VERSION 5 +#define REDIS_RDB_VERSION 6 /* Defines related to the dump file format. To store 32 bits lengths for short * keys requires a lot of space, so we check the most significant 2 bits of diff --git a/src/ziplist.c b/src/ziplist.c index b214b2da6d9..a2c0edb3a1c 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -83,6 +83,10 @@ #define ZIP_INT_16B (0xc0 | 0<<4) #define ZIP_INT_32B (0xc0 | 1<<4) #define ZIP_INT_64B (0xc0 | 2<<4) +#define ZIP_INT_24B (0xc0 | 3<<4) + +#define INT24_MAX 0x7fffff +#define INT24_MIN (-INT24_MAX - 1) /* Macro to determine type */ #define ZIP_IS_STR(enc) (((enc) & ZIP_STR_MASK) < ZIP_STR_MASK) @@ -123,6 +127,7 @@ typedef struct zlentry { static unsigned int zipIntSize(unsigned char encoding) { switch(encoding) { case ZIP_INT_16B: return sizeof(int16_t); + case ZIP_INT_24B: return sizeof(int32_t)-sizeof(int8_t); case ZIP_INT_32B: return sizeof(int32_t); case ZIP_INT_64B: return sizeof(int64_t); } @@ -271,6 +276,8 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long * of our encoding types that can hold this value. */ if (value >= INT16_MIN && value <= INT16_MAX) { *encoding = ZIP_INT_16B; + } else if (value >= INT24_MIN && value <= INT24_MAX) { + *encoding = ZIP_INT_24B; } else if (value >= INT32_MIN && value <= INT32_MAX) { *encoding = ZIP_INT_32B; } else { @@ -291,6 +298,10 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi i16 = value; memcpy(p,&i16,sizeof(i16)); memrev16ifbe(p); + } else if (encoding == ZIP_INT_24B) { + i32 = value<<8; + memrev32ifbe(&i32); + memcpy(p,((unsigned char*)&i32)+1,sizeof(i32)-sizeof(int8_t)); } else if (encoding == ZIP_INT_32B) { i32 = value; memcpy(p,&i32,sizeof(i32)); @@ -317,6 +328,11 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { memcpy(&i32,p,sizeof(i32)); memrev32ifbe(&i32); ret = i32; + } else if (encoding == ZIP_INT_24B) { + i32 = 0; + memcpy(((unsigned char*)&i32)+1,p,sizeof(i32)-sizeof(int8_t)); + memrev32ifbe(&i32); + ret = i32>>8; } else if (encoding == ZIP_INT_64B) { memcpy(&i64,p,sizeof(i64)); memrev64ifbe(&i64); From dd51571cc3092d40c5f16c2fece874e288ea3395 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 12:51:59 +0200 Subject: [PATCH 0188/1752] ziplist.c: added comments about the new 24 bit encoding. --- src/ziplist.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/ziplist.c b/src/ziplist.c index a2c0edb3a1c..81eec18a378 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -58,6 +58,10 @@ * Integer encoded as int32_t (4 bytes). * |1110____| - 1 byte * Integer encoded as int64_t (8 bytes). + * |1111____| - 1 byte + * Integer encoded as 24 bit signed (3 bytes). + * + * All the integers are represented in little endian byte order. */ #include From 62bfa66284eceae5dd06c9b3a06f0fb903e55f9d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 12:53:30 +0200 Subject: [PATCH 0189/1752] rdbLoad() should check REDIS_RDB_VERSION instead of hardcoded number. --- src/rdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index 8ffd2c28f9d..e0491a72499 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1039,7 +1039,7 @@ int rdbLoad(char *filename) { return REDIS_ERR; } rdbver = atoi(buf+5); - if (rdbver < 1 || rdbver > 5) { + if (rdbver < 1 || rdbver > REDIS_RDB_VERSION) { fclose(fp); redisLog(REDIS_WARNING,"Can't handle RDB format version %d",rdbver); errno = EINVAL; From dcd4efe9ef68b3fa01f5e9e5f8bcb86cea2d15af Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 16:54:07 +0200 Subject: [PATCH 0190/1752] Added two new encodings to ziplist.c 1) One integer "immediate" encoding that can encode from 0 to 12 in the encoding byte itself. 2) One 8 bit signed integer encoding that can encode 8 bit signed small integers in a single byte. The idea is to exploit all the not used bits we have around in a backward compatible way. --- src/ziplist.c | 65 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 19 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 81eec18a378..420f13042ef 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -52,14 +52,21 @@ * String value with length less than or equal to 16383 bytes (14 bits). * |10______|qqqqqqqq|rrrrrrrr|ssssssss|tttttttt| - 5 bytes * String value with length greater than or equal to 16384 bytes. - * |1100____| - 1 byte + * |11000000| - 1 byte * Integer encoded as int16_t (2 bytes). - * |1101____| - 1 byte + * |11010000| - 1 byte * Integer encoded as int32_t (4 bytes). - * |1110____| - 1 byte + * |11100000| - 1 byte * Integer encoded as int64_t (8 bytes). - * |1111____| - 1 byte + * |11110000| - 1 byte * Integer encoded as 24 bit signed (3 bytes). + * |11111110| - 1 byte + * Integer encoded as 8 bit signed (1 byte). + * |1111xxxx| - (with xxxx between 0000 and 1101) immediate 4 bit integer. + * Unsigned integer from 0 to 12. The encoded value is actually from + * 1 to 13 because 0000 and 1111 can not be used, so 1 should be + * subtracted from the encoded 4 bit value to obtain the right value. + * |11111111| - End of ziplist. * * All the integers are represented in little endian byte order. */ @@ -79,8 +86,8 @@ #define ZIP_BIGLEN 254 /* Different encoding/length possibilities */ -#define ZIP_STR_MASK (0xc0) -#define ZIP_INT_MASK (0x30) +#define ZIP_STR_MASK 0xc0 +#define ZIP_INT_MASK 0x30 #define ZIP_STR_06B (0 << 6) #define ZIP_STR_14B (1 << 6) #define ZIP_STR_32B (2 << 6) @@ -88,6 +95,12 @@ #define ZIP_INT_32B (0xc0 | 1<<4) #define ZIP_INT_64B (0xc0 | 2<<4) #define ZIP_INT_24B (0xc0 | 3<<4) +#define ZIP_INT_8B 0xfe +/* 4 bit integer immediate encoding */ +#define ZIP_INT_IMM_MASK 0x0f +#define ZIP_INT_IMM_MIN 0xf1 /* 11110001 */ +#define ZIP_INT_IMM_MAX 0xfd /* 11111101 */ +#define ZIP_INT_IMM_VAL(v) (v & ZIP_INT_IMM_MASK) #define INT24_MAX 0x7fffff #define INT24_MIN (-INT24_MAX - 1) @@ -119,21 +132,22 @@ typedef struct zlentry { unsigned char *p; } zlentry; -#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \ - (encoding) = (ptr[0]) & (ZIP_STR_MASK | ZIP_INT_MASK); \ - if (((encoding) & ZIP_STR_MASK) < ZIP_STR_MASK) { \ - /* String encoding: 2 MSBs */ \ - (encoding) &= ZIP_STR_MASK; \ - } \ +/* Extract the encoding from the byte pointed by 'ptr' and set it into + * 'encoding'. */ +#define ZIP_ENTRY_ENCODING(ptr, encoding) do { \ + (encoding) = (ptr[0]); \ + if ((encoding) < ZIP_STR_MASK) (encoding) &= ZIP_STR_MASK; \ } while(0) /* Return bytes needed to store integer encoded by 'encoding' */ static unsigned int zipIntSize(unsigned char encoding) { switch(encoding) { - case ZIP_INT_16B: return sizeof(int16_t); - case ZIP_INT_24B: return sizeof(int32_t)-sizeof(int8_t); - case ZIP_INT_32B: return sizeof(int32_t); - case ZIP_INT_64B: return sizeof(int64_t); + case ZIP_INT_8B: return 1; + case ZIP_INT_16B: return 2; + case ZIP_INT_24B: return 3; + case ZIP_INT_32B: return 4; + case ZIP_INT_64B: return 8; + default: return 0; /* 4 bit immediate */ } assert(NULL); return 0; @@ -278,7 +292,11 @@ static int zipTryEncoding(unsigned char *entry, unsigned int entrylen, long long if (string2ll((char*)entry,entrylen,&value)) { /* Great, the string can be encoded. Check what's the smallest * of our encoding types that can hold this value. */ - if (value >= INT16_MIN && value <= INT16_MAX) { + if (value >= 0 && value <= 12) { + *encoding = ZIP_INT_IMM_MIN+value; + } else if (value >= INT8_MIN && value <= INT8_MAX) { + *encoding = ZIP_INT_8B; + } else if (value >= INT16_MIN && value <= INT16_MAX) { *encoding = ZIP_INT_16B; } else if (value >= INT24_MIN && value <= INT24_MAX) { *encoding = ZIP_INT_24B; @@ -298,7 +316,9 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi int16_t i16; int32_t i32; int64_t i64; - if (encoding == ZIP_INT_16B) { + if (encoding == ZIP_INT_8B) { + ((char*)p)[0] = (char)value; + } else if (encoding == ZIP_INT_16B) { i16 = value; memcpy(p,&i16,sizeof(i16)); memrev16ifbe(p); @@ -314,6 +334,8 @@ static void zipSaveInteger(unsigned char *p, int64_t value, unsigned char encodi i64 = value; memcpy(p,&i64,sizeof(i64)); memrev64ifbe(p); + } else if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX) { + /* Nothing to do, the value is stored in the encoding itself. */ } else { assert(NULL); } @@ -324,7 +346,10 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { int16_t i16; int32_t i32; int64_t i64, ret = 0; - if (encoding == ZIP_INT_16B) { + printf("%02x\n", encoding); + if (encoding == ZIP_INT_8B) { + ret = ((char*)p)[0]; + } else if (encoding == ZIP_INT_16B) { memcpy(&i16,p,sizeof(i16)); memrev16ifbe(&i16); ret = i16; @@ -341,6 +366,8 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { memcpy(&i64,p,sizeof(i64)); memrev64ifbe(&i64); ret = i64; + } else if (encoding >= ZIP_INT_IMM_MIN && encoding <= ZIP_INT_IMM_MAX) { + ret = (encoding & ZIP_INT_IMM_MASK)-1; } else { assert(NULL); } From 717145c1a3302f4a20746dce8c885ee590e0f0e2 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 17:15:21 +0200 Subject: [PATCH 0191/1752] Spurious debugging printf removed. --- src/ziplist.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ziplist.c b/src/ziplist.c index 420f13042ef..e3741f81eb1 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -346,7 +346,6 @@ static int64_t zipLoadInteger(unsigned char *p, unsigned char encoding) { int16_t i16; int32_t i32; int64_t i64, ret = 0; - printf("%02x\n", encoding); if (encoding == ZIP_INT_8B) { ret = ((char*)p)[0]; } else if (encoding == ZIP_INT_16B) { From 12a042f8635c5362332ca7a9b3eea66fdbfb6f08 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 19:05:27 +0200 Subject: [PATCH 0192/1752] redis-check-dump now is RDB version 6 ready. --- src/Makefile | 2 +- src/redis-check-dump.c | 38 +++++++++++++++++++++++++++++++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index a6e8e2f94f4..082b375798f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -102,7 +102,7 @@ REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o REDIS_BENCHMARK_NAME= redis-benchmark REDIS_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o REDIS_CHECK_DUMP_NAME= redis-check-dump -REDIS_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o +REDIS_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o crc64.o REDIS_CHECK_AOF_NAME= redis-check-aof REDIS_CHECK_AOF_OBJ= redis-check-aof.o diff --git a/src/redis-check-dump.c b/src/redis-check-dump.c index 42fe5181087..5eac925ae98 100644 --- a/src/redis-check-dump.c +++ b/src/redis-check-dump.c @@ -108,6 +108,9 @@ static double R_Zero, R_PosInf, R_NegInf, R_Nan; /* store string types for output */ static char types[256][16]; +/* Prototypes */ +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); + /* when number of bytes to read is negative, do a peek */ int readBytes(void *target, long num) { char peek = (num < 0) ? 1 : 0; @@ -137,10 +140,10 @@ int processHeader() { } dump_version = (int)strtol(buf + 5, NULL, 10); - if (dump_version < 1 || dump_version > 4) { + if (dump_version < 1 || dump_version > 6) { ERROR("Unknown RDB format version: %d\n", dump_version); } - return 1; + return dump_version; } int loadType(entry *e) { @@ -557,7 +560,16 @@ void printErrorStack(entry *e) { void process() { uint64_t num_errors = 0, num_valid_ops = 0, num_valid_bytes = 0; entry entry; - processHeader(); + int dump_version = processHeader(); + + /* Exclude the final checksum for RDB >= 5. Will be checked at the end. */ + if (dump_version >= 5) { + if (positions[0].size < 8) { + printf("RDB version >= 5 but no room for checksum.\n"); + exit(1); + } + positions[0].size -= 8;; + } level = 1; while(positions[0].offset < positions[0].size) { @@ -622,6 +634,26 @@ void process() { num_errors++; } + /* Verify checksum */ + if (dump_version >= 5) { + uint64_t crc = crc64(0,positions[0].data,positions[0].size); + uint64_t crc2; + unsigned char *p = (unsigned char*)positions[0].data+positions[0].size; + crc2 = ((uint64_t)p[0] << 0) | + ((uint64_t)p[1] << 8) | + ((uint64_t)p[2] << 16) | + ((uint64_t)p[3] << 24) | + ((uint64_t)p[4] << 32) | + ((uint64_t)p[5] << 40) | + ((uint64_t)p[6] << 48) | + ((uint64_t)p[7] << 56); + if (crc != crc2) { + SHIFT_ERROR(positions[0].offset, "RDB CRC64 does not match."); + } else { + printf("CRC64 checksum is OK\n"); + } + } + /* print summary on errors */ if (num_errors) { printf("\n"); From c16bf717de0b97f78c2fd6316b9460493b80898e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 24 Apr 2012 19:14:03 +0200 Subject: [PATCH 0193/1752] Explicitly use bash for install_server.sh. Fixes issue #397 --- utils/install_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/install_server.sh b/utils/install_server.sh index 93b5b411b80..70f0adfe36d 100755 --- a/utils/install_server.sh +++ b/utils/install_server.sh @@ -1,4 +1,4 @@ -#! /bin/sh +#!/bin/bash # Copyright 2011 Dvir Volk . All rights reserved. # From 8111a803cbbcb9ec6f494baef0fdd5e0664fb33e Mon Sep 17 00:00:00 2001 From: David Tran Date: Wed, 25 Apr 2012 12:21:56 -0700 Subject: [PATCH 0194/1752] Spelling: s/synchrnonization/synchronization --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index ef148f1c28e..f03c424faa7 100644 --- a/src/replication.c +++ b/src/replication.c @@ -323,7 +323,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { } server.repl_transfer_lastio = server.unixtime; if (write(server.repl_transfer_fd,buf,nread) != nread) { - redisLog(REDIS_WARNING,"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchrnonization: %s", strerror(errno)); + redisLog(REDIS_WARNING,"Write error or short write writing to the DB dump file needed for MASTER <-> SLAVE synchronization: %s", strerror(errno)); goto error; } server.repl_transfer_left -= nread; From d7bad544dc28d024a070d2163f473e420d31fd65 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Apr 2012 11:16:52 +0200 Subject: [PATCH 0195/1752] Redis test: scripting EVALSHA replication test more reliable. A new primitive wait_for_condition was introduced in the scripting engine that makes waiting for events simpler, so that it is simpler to write tests that are more resistant to timing issues. --- tests/support/test.tcl | 17 +++++++++++++++++ tests/unit/scripting.tcl | 10 ++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/support/test.tcl b/tests/support/test.tcl index f66e54b87cf..91ab8117be1 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -3,6 +3,10 @@ set ::num_passed 0 set ::num_failed 0 set ::tests_failed {} +proc fail {msg} { + error "assertion:$msg" +} + proc assert {condition} { if {![uplevel 1 [list expr $condition]]} { error "assertion:Expected condition '$condition' to be true ([uplevel 1 [list subst -nocommands $condition]])" @@ -44,6 +48,19 @@ proc assert_type {type key} { assert_equal $type [r type $key] } +# Wait for the specified condition to be true, with the specified number of +# max retries and delay between retries. Otherwise the 'elsescript' is +# executed. +proc wait_for_condition {maxtries delay e _else_ elsescript} { + while {[incr maxtries -1] >= 0} { + if {[uplevel 1 expr $e]} break + after $delay + } + if {$maxtries == -1} { + uplevel 1 $elsescript + } +} + # Test if TERM looks like to support colors proc color_term {} { expr {[info exists ::env(TERM)] && [string match *xterm* $::env(TERM)]} diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index a60c65b437d..daf0c0f26e7 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -327,10 +327,12 @@ start_server {tags {"scripting repl"}} { r evalsha ae3477e27be955de7e1bc9adfdca626b478d3cb2 0 } {2} - if {$::valgrind} {after 2000} else {after 100} - test {If EVALSHA was replicated as EVAL the slave should be ok} { - r -1 get x - } {2} + wait_for_condition 50 100 { + [r -1 get x] eq {2} + } else { + fail "Expected 2 in x, but value is '[r -1 get x]'" + } + } } } From b1ee7da75a4191bf1785f598c158b191104fdc38 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Apr 2012 11:25:13 +0200 Subject: [PATCH 0196/1752] Redis test: More reliable BRPOPLPUSH replication test. Now it uses the new wait_for_condition testing primitive. Also wait_for_condition implementation was fixed in this commit to properly escape the expr command and its argument. --- tests/integration/replication.tcl | 7 +++++-- tests/support/test.tcl | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 7c1edb550a2..3689181f840 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -10,8 +10,11 @@ start_server {tags {"repl"}} { set rd [redis_deferring_client] $rd brpoplpush a b 5 r lpush a foo - after 1000 - assert_equal [r debug digest] [r -1 debug digest] + wait_for_condition 50 100 { + [r debug digest] eq [r -1 debug digest] + } else { + fail "Master and slave have different digest: [r debug digest] VS [r -1 debug digest]" + } } test {BRPOPLPUSH replication, list exists} { diff --git a/tests/support/test.tcl b/tests/support/test.tcl index 91ab8117be1..480c674e05b 100644 --- a/tests/support/test.tcl +++ b/tests/support/test.tcl @@ -53,7 +53,7 @@ proc assert_type {type key} { # executed. proc wait_for_condition {maxtries delay e _else_ elsescript} { while {[incr maxtries -1] >= 0} { - if {[uplevel 1 expr $e]} break + if {[uplevel 1 [list expr $e]]} break after $delay } if {$maxtries == -1} { From a28ab2a92b9b586b06ca75b2deaabae4a2b05fb4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Apr 2012 16:21:19 +0200 Subject: [PATCH 0197/1752] Don't use an alternative stack for SIGSEGV & co. This commit reverts most of c575766202773c858be0870c20cd495b722927c3, in order to use back main stack for signal handling. The main reason is that otherwise it is completely pointless that we do a lot of efforts to print the stack trace on crash, and the content of the stack and registers as well. Using an alternate stack broken this feature completely. --- src/redis.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/redis.c b/src/redis.c index 08995115163..e2f9024b30a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -62,9 +62,6 @@ double R_Zero, R_PosInf, R_NegInf, R_Nan; /*================================= Globals ================================= */ -/* Alternate stack for SIGSEGV/etc handlers */ -char altstack[SIGSTKSZ]; - /* Global vars */ struct redisServer server; /* server global state */ struct redisCommand *commandTable; @@ -2300,13 +2297,6 @@ static void sigtermHandler(int sig) { void setupSignalHandlers(void) { struct sigaction act; - stack_t stack; - - stack.ss_sp = altstack; - stack.ss_flags = 0; - stack.ss_size = SIGSTKSZ; - - sigaltstack(&stack, NULL); /* When the SA_SIGINFO flag is set in sa_flags then sa_sigaction is used. * Otherwise, sa_handler is used. */ @@ -2316,10 +2306,8 @@ void setupSignalHandlers(void) { sigaction(SIGTERM, &act, NULL); #ifdef HAVE_BACKTRACE - /* Use alternate stack so we don't clobber stack in case of segv, or when we run out of stack .. - * also resethand & nodefer so we can get interrupted (and killed) if we cause SEGV during SEGV handler */ sigemptyset(&act.sa_mask); - act.sa_flags = SA_NODEFER | SA_ONSTACK | SA_RESETHAND | SA_SIGINFO; + act.sa_flags = SA_NODEFER | SA_RESETHAND | SA_SIGINFO; act.sa_sigaction = sigsegvHandler; sigaction(SIGSEGV, &act, NULL); sigaction(SIGBUS, &act, NULL); From 9db4cea23576d9735c906829d4c4d2feb7720bf0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 26 Apr 2012 16:04:53 +0200 Subject: [PATCH 0198/1752] Produce the stack trace in an async safe way. --- src/debug.c | 52 +++++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/debug.c b/src/debug.c index 8d51e959d80..42a73883d9d 100644 --- a/src/debug.c +++ b/src/debug.c @@ -7,6 +7,7 @@ #ifdef HAVE_BACKTRACE #include #include +#include #endif /* HAVE_BACKTRACE */ /* ================================= Debugging ============================== */ @@ -567,27 +568,30 @@ void logRegisters(ucontext_t *uc) { #endif } -/* Logs the stack trace using the backtrace() call. */ -sds getStackTrace(ucontext_t *uc) { +/* Logs the stack trace using the backtrace() call. This function is designed + * to be called from signal handlers safely. */ +void logStackTrace(ucontext_t *uc) { void *trace[100]; - int i, trace_size = 0; - char **messages = NULL; - sds st = sdsempty(); + int trace_size = 0, fd; + + /* Open the log file in append mode. */ + fd = server.logfile ? + open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644) : + STDOUT_FILENO; + if (fd == -1) return; /* Generate the stack trace */ trace_size = backtrace(trace, 100); /* overwrite sigaction with caller's address */ - if (getMcontextEip(uc) != NULL) { + if (getMcontextEip(uc) != NULL) trace[1] = getMcontextEip(uc); - } - messages = backtrace_symbols(trace, trace_size); - for (i=1; i Date: Thu, 26 Apr 2012 16:53:11 +0200 Subject: [PATCH 0199/1752] Re-introduce -g -rdynamic -ggdb when linking, fixing strack traces. A previous commit removed -g -rdynamic -ggdb as LDFLAGS, not allowing Redis to produce a stack trace wth symbol names on crash. This commit fixes the issue. --- src/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 082b375798f..dce0324b776 100644 --- a/src/Makefile +++ b/src/Makefile @@ -47,12 +47,12 @@ endif ifeq ($(uname_S),SunOS) FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 - FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) + FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) -g -ggdb FINAL_LIBS= -ldl -lnsl -lsocket -lm -lpthread DEBUG= -g -ggdb else FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) - FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) + FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) -g -rdynamic -ggdb FINAL_LIBS= -lm -pthread DEBUG= -g -rdynamic -ggdb endif From 77a75fdef5c56033e328252f47bf3a26a3dab1b4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 11:41:25 +0200 Subject: [PATCH 0200/1752] Set LUA_MASKCOUNT hook more selectively. Fixes issue #480. An user reported a crash with Redis scripting (see issue #480 on github), inspection of the kindly provided strack trace showed that server.lua_caller was probably set to NULL. The stack trace also slowed that the call to the hook was originating from a point where we just used to set/get a few global variables in the Lua state. What was happening is that we did not set the timeout hook selectively only when the user script was called. Now we set it more selectively, specifically only in the context of the lua_pcall() call, and make sure to remove the hook when the call returns. Otherwise the hook can get called in random contexts every time we do something with the Lua state. --- src/scripting.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index a5f5683e15d..8c89c923c0c 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -716,6 +716,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { lua_State *lua = server.lua; char funcname[43]; long long numkeys; + int delhook = 0; /* We want the same PRNG sequence at every call so that our PRNG is * not affected by external state. */ @@ -786,19 +787,19 @@ void evalGenericCommand(redisClient *c, int evalsha) { * is running for too much time. * We set the hook only if the time limit is enabled as the hook will * make the Lua script execution slower. */ + server.lua_caller = c; + server.lua_time_start = ustime()/1000; + server.lua_kill = 0; if (server.lua_time_limit > 0 && server.masterhost == NULL) { lua_sethook(lua,luaMaskCountHook,LUA_MASKCOUNT,100000); - } else { - lua_sethook(lua,luaMaskCountHook,0,0); + delhook = 1; } /* At this point whatever this script was never seen before or if it was * already defined, we can call it. We have zero arguments and expect * a single return value. */ - server.lua_caller = c; - server.lua_time_start = ustime()/1000; - server.lua_kill = 0; if (lua_pcall(lua,0,1,0)) { + if (delhook) lua_sethook(lua,luaMaskCountHook,0,0); /* Disable hook */ if (server.lua_timedout) { server.lua_timedout = 0; /* Restore the readable handler that was unregistered when the @@ -814,6 +815,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { lua_gc(lua,LUA_GCCOLLECT,0); return; } + if (delhook) lua_sethook(lua,luaMaskCountHook,0,0); /* Disable hook */ server.lua_timedout = 0; server.lua_caller = NULL; selectDb(c,server.lua_client->db->id); /* set DB ID from Lua client */ From b1aa7183d772dcee73f2e4fb1650c9595fe8096f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 15:56:16 +0200 Subject: [PATCH 0201/1752] Update makefile dependencies. --- src/Makefile.dep | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Makefile.dep b/src/Makefile.dep index a098154ae88..5331189f1bc 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -13,9 +13,6 @@ bio.o: bio.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ config.o: config.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h -crc16.o: crc16.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ - ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ - ziplist.h intset.h version.h util.h rdb.h rio.h crc64.o: crc64.c db.o: db.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ From 2c0aae760fe9d55789a5da937a09a91b03777c11 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 15:57:17 +0200 Subject: [PATCH 0202/1752] redis-cli commands description in help.h updated. --- src/help.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/help.h b/src/help.h index cde95c1f841..406fb224dcc 100644 --- a/src/help.h +++ b/src/help.h @@ -325,7 +325,7 @@ struct commandHelp { 1, "1.0.0" }, { "MIGRATE", - "host port key destination db timeout", + "host port key destination-db timeout", "Atomically transfer a key from a Redis instance to another one.", 0, "2.6.0" }, @@ -370,7 +370,7 @@ struct commandHelp { 0, "2.6.0" }, { "PEXPIREAT", - "key milliseconds timestamp", + "key milliseconds-timestamp", "Set the expiration for a key as a UNIX timestamp specified in milliseconds", 0, "2.6.0" }, @@ -425,7 +425,7 @@ struct commandHelp { 0, "1.0.0" }, { "RESTORE", - "key ttl serialized value", + "key ttl serialized-value", "Create a key using the provided serialized value, previously obtained using DUMP.", 0, "2.6.0" }, From 748f206e3d7f66f687f42cfa3341928d5cb548f1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 16:07:42 +0200 Subject: [PATCH 0203/1752] Release notes updated with the new 2.6 features. --- 00-RELEASENOTES | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 64acd87d1df..67abf9e65d5 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -46,6 +46,8 @@ UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to in slaves. * Milliseconds resolution expires, also added new commands with milliseconds precision (PEXPIRE, PTTL, ...). +* Better memory usage for "small" lists, ziplists and hashes when fields or + values contain small integers. * Clients max output buffer soft and hard limits. You can specifiy different limits for different classes of clients (normal,pubsub,slave). * AOF is now able to rewrite aggregate data types using variadic commands, @@ -54,6 +56,12 @@ UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to redis-server binary, with the same name and number of arguments. * Hash table seed randomization for protection against collisions attacks. * Performances improved when writing large objects to Redis. +* Integrated memory test, see redis-server --test-memory. +* INCRBYFLOAT and HINCRBYFLOAT commands. +* New DUMP, RESTORE, MIGRATE commands (back ported from Redis Cluster to 2.6). +* CRC64 checksump in RDB files. +* Better MONITOR output and behavior (now commands are logged before execution). +* "Software Watchdog" feature to debug latency issues. * Significant parts of the core refactored or rewritten. New internal APIs and core changes allowed to develop Redis Cluster on top of the new code, however for 2.6 all the cluster code was removed, and will be released with @@ -70,7 +78,8 @@ UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to * New statistics about how many time a command was called, and how much execution time it used (INFO commandstats). * More predictable SORT behavior in edge cases. -* INCRBYFLOAT and HINCRBYFLOAT commands. +* Better support for big endian and *BSD systems. +* Build system improved. -------------------------------------------------------------------------------- From 603adb2b2991129fc600c2d8dad549e339873a5b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 16:28:31 +0200 Subject: [PATCH 0204/1752] memtest.c fixed to actually use v1 and v2 in memtest_fill_value(). --- src/memtest.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/memtest.c b/src/memtest.c index 09b4d831b1e..88c7213af7d 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -132,13 +132,13 @@ void memtest_fill_value(unsigned long *l, size_t bytes, unsigned long v1, v = (off & 1) ? v2 : v1; for (w = 0; w < iwords; w++) { #ifdef MEMTEST_32BIT - *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | - (((unsigned long) (rand()&0xffff)) << 16); + *l1 = *l2 = ((unsigned long) v) | + (((unsigned long) v) << 16); #else - *l1 = *l2 = ((unsigned long) (rand()&0xffff)) | - (((unsigned long) (rand()&0xffff)) << 16) | - (((unsigned long) (rand()&0xffff)) << 32) | - (((unsigned long) (rand()&0xffff)) << 48); + *l1 = *l2 = ((unsigned long) v) | + (((unsigned long) v) << 16) | + (((unsigned long) v) << 32) | + (((unsigned long) v) << 48); #endif l1 += step; l2 += step; From 7c5d96d98e6cc2a0e7ea5ea30e3ca4772a84b3cb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 16:40:07 +0200 Subject: [PATCH 0205/1752] Redis 2.5.7 (2.6 RC1) --- 00-RELEASENOTES | 15 ++++++++++----- src/version.h | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 67abf9e65d5..d4364bc757e 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -32,12 +32,17 @@ Also the following redis.conf and CONFIG GET / SET parameters changed name: CHANGELOG --------- -What's new in Redis 2.6.0 -========================= +What's new in Redis 2.5.7 (aka 2.6 Release Candidate 1) +======================================================= + +UPGRADE URGENCY: upgrade not recommended because this is an RC release. + +* This is the first release candidate for Redis 2.6. We are not aware of + bugs, but part of this code is young and was never tested in production + environments, so handle with care. -UPGRADE URGENCY: We suggest new users to start with 2.6.0, and old users to - upgrade after some testing of the application with the new - Redis version. +An overview of new features and changes in Redis 2.6.x +====================================================== * Server side Lua scripting, see http://redis.io/commands/eval * Virtual Memory removed (was deprecated in 2.4) diff --git a/src/version.h b/src/version.h index d25a2e1d8a5..4588d54ed85 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.6" +#define REDIS_VERSION "2.5.7" From 8b97442c2021469bd5fc83780cb2610ba7332f86 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 17:01:44 +0200 Subject: [PATCH 0206/1752] Fixed release notes typo --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index d4364bc757e..3faa0accf47 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -26,7 +26,7 @@ Also the following redis.conf and CONFIG GET / SET parameters changed name: * hash-max-zipmap-entries, now replaced by hash-max-ziplist-entries * hash-max-zipmap-value, now replaced by hash-max-ziplist-value - * glueoutputbuf was no completely removed as it does not make sense + * glueoutputbuf option was now completely removed (was reprecated) --------- CHANGELOG From 18759c927a322eab424726fb38d0b453fffe7c1b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 17:04:03 +0200 Subject: [PATCH 0207/1752] yet another typo fixed in release notes. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 3faa0accf47..b10f54523dd 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -26,7 +26,7 @@ Also the following redis.conf and CONFIG GET / SET parameters changed name: * hash-max-zipmap-entries, now replaced by hash-max-ziplist-entries * hash-max-zipmap-value, now replaced by hash-max-ziplist-value - * glueoutputbuf option was now completely removed (was reprecated) + * glueoutputbuf option was now completely removed (was deprecated) --------- CHANGELOG From b330de57ff79f2beff36c81a07f47ce1b60113e6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 27 Apr 2012 23:06:02 +0200 Subject: [PATCH 0208/1752] Added "read-only slaves" in new features section of 2.6 release notes. --- 00-RELEASENOTES | 1 + 1 file changed, 1 insertion(+) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index b10f54523dd..2c7fad2bbe6 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -53,6 +53,7 @@ An overview of new features and changes in Redis 2.6.x precision (PEXPIRE, PTTL, ...). * Better memory usage for "small" lists, ziplists and hashes when fields or values contain small integers. +* Read only slaves. * Clients max output buffer soft and hard limits. You can specifiy different limits for different classes of clients (normal,pubsub,slave). * AOF is now able to rewrite aggregate data types using variadic commands, From dd418873dbaa71c2496ab7a52828f8881cf8a0ed Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 28 Apr 2012 15:39:00 +0200 Subject: [PATCH 0209/1752] A more lightweight implementation of issue 141 regression test. --- tests/integration/replication.tcl | 39 ++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 3689181f840..71a22f69173 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -82,12 +82,11 @@ start_server {tags {"repl"}} { set master_host [srv 0 host] set master_port [srv 0 port] set slaves {} - set load_handle0 [start_write_load $master_host $master_port 20] - set load_handle1 [start_write_load $master_host $master_port 20] + set load_handle0 [start_write_load $master_host $master_port 3] + set load_handle1 [start_write_load $master_host $master_port 5] set load_handle2 [start_write_load $master_host $master_port 20] - set load_handle3 [start_write_load $master_host $master_port 20] - set load_handle4 [start_write_load $master_host $master_port 20] - after 2000 + set load_handle3 [start_write_load $master_host $master_port 8] + set load_handle4 [start_write_load $master_host $master_port 4] start_server {} { lappend slaves [srv 0 client] start_server {} { @@ -95,6 +94,7 @@ start_server {tags {"repl"}} { start_server {} { lappend slaves [srv 0 client] test "Connect multiple slaves at the same time (issue #141)" { + # Send SALVEOF commands to slaves [lindex $slaves 0] slaveof $master_host $master_port [lindex $slaves 1] slaveof $master_host $master_port [lindex $slaves 2] slaveof $master_host $master_port @@ -113,16 +113,33 @@ start_server {tags {"repl"}} { if {$retry == 0} { error "assertion:Slaves not correctly synchronized" } + + # Stop the write load stop_write_load $load_handle0 stop_write_load $load_handle1 stop_write_load $load_handle2 stop_write_load $load_handle3 stop_write_load $load_handle4 - set retry 10 - while {$retry && ([$master debug digest] ne [[lindex $slaves 0] debug digest])} { - after 1000 - incr retry -1 + + # Wait that slaves exit the "loading" state + wait_for_condition 500 100 { + ![string match {*loading:1*} [[lindex $slaves 0] info]] && + ![string match {*loading:1*} [[lindex $slaves 1] info]] && + ![string match {*loading:1*} [[lindex $slaves 2] info]] + } else { + fail "Slaves still loading data after too much time" } + + # Make sure that slaves and master have same number of keys + wait_for_condition 500 100 { + [$master dbsize] == [[lindex $slaves 0] dbsize] && + [$master dbsize] == [[lindex $slaves 1] dbsize] && + [$master dbsize] == [[lindex $slaves 2] dbsize] + } else { + fail "Different number of keys between masted and slave after too long time." + } + + # Check digests set digest [$master debug digest] set digest0 [[lindex $slaves 0] debug digest] set digest1 [[lindex $slaves 1] debug digest] @@ -131,10 +148,6 @@ start_server {tags {"repl"}} { assert {$digest eq $digest0} assert {$digest eq $digest1} assert {$digest eq $digest2} - #puts [$master dbsize] - #puts [[lindex $slaves 0] dbsize] - #puts [[lindex $slaves 1] dbsize] - #puts [[lindex $slaves 2] dbsize] } } } From 8f984bef2964cab3ddf6b4ea3065fcf7f6bdc36a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Apr 2012 10:55:03 +0200 Subject: [PATCH 0210/1752] Properly wait the slave to sync with master in BRPOPLPUSH test. --- tests/integration/replication.tcl | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 71a22f69173..18e639d41f6 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -2,9 +2,13 @@ start_server {tags {"repl"}} { start_server {} { test {First server should have role slave after SLAVEOF} { r -1 slaveof [srv 0 host] [srv 0 port] - after 1000 - s -1 role - } {slave} + wait_for_condition 50 100 { + [s -1 role] eq {slave} && + [string match {*master_link_status:up*} [r -1 info replication]] + } else { + fail "Can't turn the instance into a slave" + } + } test {BRPOPLPUSH replication, when blocking against empty list} { set rd [redis_deferring_client] From 8520066d7b4782adaf9caa56036071ddb3de86fc Mon Sep 17 00:00:00 2001 From: Harmen Date: Mon, 30 Apr 2012 09:57:12 -0600 Subject: [PATCH 0211/1752] Show problem with 'keys' command with specific command sequence. --- tests/unit/expire.tcl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 82876ddd790..56a59f7680d 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -141,4 +141,15 @@ start_server {tags {"expire"}} { set size2 [r dbsize] list $size1 $size2 } {3 0} + + test {5 keys in, 5 keys out} { + r flushdb + r set a c + r expire a 5 + r set t c + r set e c + r set s c + r set foo b + lsort [r keys *] + } {a e foo s t} } From 9311d2b527b0bb3e72e61b143718e6def51491e6 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 30 Apr 2012 10:16:20 -0700 Subject: [PATCH 0212/1752] Use safe dictionary iterator from KEYS Every matched key in a KEYS call is checked for expiration. When the key is set to expire, the call to `getExpire` will assert that the key also exists in the main dictionary. This in turn causes a rehashing step to be executed. Rehashing a dictionary when there is an iterator active may result in the iterator emitting duplicate entries, or not emitting some entries at all. By using a safe iterator, the rehash step is omitted. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 3c75f6bad61..e65106a583c 100644 --- a/src/db.c +++ b/src/db.c @@ -255,7 +255,7 @@ void keysCommand(redisClient *c) { unsigned long numkeys = 0; void *replylen = addDeferredMultiBulkLength(c); - di = dictGetIterator(c->db->dict); + di = dictGetSafeIterator(c->db->dict); allkeys = (pattern[0] == '*' && pattern[1] == '\0'); while((de = dictNext(di)) != NULL) { sds key = dictGetKey(de); From 1858da2faae3b6a8becf4f7eef3f712d6e4b986b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 May 2012 11:40:46 +0200 Subject: [PATCH 0213/1752] Test "Turning off AOF kills the background writing child if any" is now more reliable. --- tests/unit/aofrw.tcl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index 4716d08e60e..8b09d1995a2 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -7,8 +7,12 @@ start_server {tags {"aofrw"}} { r bgrewriteaof r config set appendonly no r exec - set result [exec tail -n1 < [srv 0 stdout] ] - } {*Killing*AOF*child*} + wait_for_condition 50 100 { + [string match {*Killing*AOF*child*} [exec tail -n5 < [srv 0 stdout]]] + } else { + fail "Can't find 'Killing AOF child' into recent logs" + } + } foreach d {string int} { foreach e {ziplist linkedlist} { From 0f07781538a4cb0249c70045ec4c83011defd923 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 May 2012 12:16:53 +0200 Subject: [PATCH 0214/1752] Redis 2.5.8 (2.6.0 RC2). --- 00-RELEASENOTES | 9 +++++++++ src/version.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 2c7fad2bbe6..006a163ee80 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -32,6 +32,15 @@ Also the following redis.conf and CONFIG GET / SET parameters changed name: CHANGELOG --------- +What's new in Redis 2.5.8 (aka 2.6 Release Candidate 2) +======================================================= + +UPGRADE URGENCY: high for all the users of the KEYS command, otherwise low. + +* [BUGFIX] Fix for KEYS command: if the DB contains keys with expires the KEYS + command may return the wrong output, having duplicated or missing + keys. See issue #487 and #488 on github for details. + What's new in Redis 2.5.7 (aka 2.6 Release Candidate 1) ======================================================= diff --git a/src/version.h b/src/version.h index 4588d54ed85..64fced48aae 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.7" +#define REDIS_VERSION "2.5.8" From 96d9c6cb8be9fc3b4a9fab0cb281418a3deb0533 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 May 2012 16:35:53 +0200 Subject: [PATCH 0215/1752] Add a note abotu read-only slaves in 2.6 release notes. --- 00-RELEASENOTES | 3 +++ 1 file changed, 3 insertions(+) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 006a163ee80..f4505d11d48 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -21,6 +21,9 @@ but it is better to keep them in mind: * INFO output is a bit different now, and contains empty lines and comments starting with '#'. All the major clients should be already fixed to work with the new INFO format. +* Slaves are only read-only by default (but you can change this easily + setting the "slave-read-only" configuration option to "no" editing your + redis.conf or using CONFIG SET. Also the following redis.conf and CONFIG GET / SET parameters changed name: From 0b08d64882e453b04f23c184c047dce1de65845b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 May 2012 17:14:45 +0200 Subject: [PATCH 0216/1752] Use specific error if master is down and slave-serve-stale-data is set to no. We used to reply -ERR ... message ..., now the reply is instead -MASTERDOWN ... message ... so that it can be distinguished easily by the other error conditions. --- src/redis.c | 5 +++-- src/redis.h | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index e2f9024b30a..0a12ccd2e2e 100644 --- a/src/redis.c +++ b/src/redis.c @@ -970,6 +970,8 @@ void createSharedObjects(void) { "-LOADING Redis is loading the dataset in memory\r\n")); shared.slowscripterr = createObject(REDIS_STRING,sdsnew( "-BUSY Redis is busy running a script. You can only call SCRIPT KILL or SHUTDOWN NOSAVE.\r\n")); + shared.masterdownerr = createObject(REDIS_STRING,sdsnew( + "-MASTERDOWN Link with MASTER is down and slave-serve-stale-data is set to 'no'.\r\n")); shared.bgsaveerr = createObject(REDIS_STRING,sdsnew( "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Commands that may modify the data set are disabled. Please check Redis logs for details about the error.\r\n")); shared.roslaveerr = createObject(REDIS_STRING,sdsnew( @@ -1541,8 +1543,7 @@ int processCommand(redisClient *c) { server.repl_serve_stale_data == 0 && c->cmd->proc != infoCommand && c->cmd->proc != slaveofCommand) { - addReplyError(c, - "link with MASTER is down and slave-serve-stale-data is set to no"); + addReply(c, shared.masterdownerr); return REDIS_OK; } diff --git a/src/redis.h b/src/redis.h index 2084dfd2a6c..281c0878789 100644 --- a/src/redis.h +++ b/src/redis.h @@ -366,7 +366,8 @@ struct sharedObjectsStruct { *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, - *roslaveerr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, + *masterdownerr, *roslaveerr, + *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, *select[REDIS_SHARED_SELECT_CMDS], *integers[REDIS_SHARED_INTEGERS], From 9b43b1ef4d646922fc904fda61ff5ab67a2581db Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 May 2012 21:45:01 +0200 Subject: [PATCH 0217/1752] Remove useless trailing space in SYNC command sent to master. --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index f03c424faa7..5c5bc9abfa2 100644 --- a/src/replication.c +++ b/src/replication.c @@ -423,7 +423,7 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { } /* Issue the SYNC command */ - if (syncWrite(fd,"SYNC \r\n",7,server.repl_syncio_timeout*1000) == -1) { + if (syncWrite(fd,"SYNC\r\n",6,server.repl_syncio_timeout*1000) == -1) { redisLog(REDIS_WARNING,"I/O error writing to MASTER: %s", strerror(errno)); goto error; From 0cf10e8e867274397f2b7201e51b00996263a143 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 2 May 2012 22:41:50 +0200 Subject: [PATCH 0218/1752] syncio.c read / write functions reworked for correctness and performance. The new implementation start reading / writing before blocking with aeWait(), likely the descriptor can accept writes or has buffered data inside and we can go faster, otherwise we get an error and wait. This change has effects on speed but also on correctness: on socket errors when we perform non blocking connect(2) write is performed ASAP and the error is returned ASAP before waiting. So the practical effect is that now a Redis slave is more available if it can not connect to the master, previously the slave continued to block on syncWrite() trying to send SYNC, and serving commands very slowly. --- src/syncio.c | 42 ++++++++++++++++++++++++++++-------------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/syncio.c b/src/syncio.c index b0c2969e2cc..0c202c9e242 100644 --- a/src/syncio.c +++ b/src/syncio.c @@ -42,10 +42,10 @@ #define REDIS_SYNCIO_RESOLUTION 10 /* Resolution in milliseconds */ -/* Write the specified payload to 'fd'. If writing the whole payload will be done - * within 'timeout' milliseconds the operation succeeds and 'size' is returned. - * Otherwise the operation fails, -1 is returned, and an unspecified partial write - * could be performed against the file descriptor. */ +/* Write the specified payload to 'fd'. If writing the whole payload will be + * done within 'timeout' milliseconds the operation succeeds and 'size' is + * returned. Otherwise the operation fails, -1 is returned, and an unspecified + * partial write could be performed against the file descriptor. */ ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout) { ssize_t nwritten, ret = size; long long start = mstime(); @@ -56,13 +56,19 @@ ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout) { remaining : REDIS_SYNCIO_RESOLUTION; long long elapsed; - if (aeWait(fd,AE_WRITABLE,wait) & AE_WRITABLE) { - nwritten = write(fd,ptr,size); - if (nwritten == -1) return -1; + /* Optimistically try to write before checking if the file descriptor + * is actually writable. At worst we get EAGAIN. */ + nwritten = write(fd,ptr,size); + if (nwritten == -1) { + if (errno != EAGAIN) return -1; + } else { ptr += nwritten; size -= nwritten; - if (size == 0) return ret; } + if (size == 0) return ret; + + /* Wait */ + aeWait(fd,AE_WRITABLE,wait); elapsed = mstime() - start; if (elapsed >= timeout) { errno = ETIMEDOUT; @@ -72,8 +78,8 @@ ssize_t syncWrite(int fd, char *ptr, ssize_t size, long long timeout) { } } -/* Read the specified amount of bytes from 'fd'. If all the bytes are read within - * 'timeout' milliseconds the operation succeed and 'size' is returned. +/* Read the specified amount of bytes from 'fd'. If all the bytes are read + * within 'timeout' milliseconds the operation succeed and 'size' is returned. * Otherwise the operation fails, -1 is returned, and an unspecified amount of * data could be read from the file descriptor. */ ssize_t syncRead(int fd, char *ptr, ssize_t size, long long timeout) { @@ -81,19 +87,27 @@ ssize_t syncRead(int fd, char *ptr, ssize_t size, long long timeout) { long long start = mstime(); long long remaining = timeout; + if (size == 0) return 0; while(1) { long long wait = (remaining > REDIS_SYNCIO_RESOLUTION) ? remaining : REDIS_SYNCIO_RESOLUTION; long long elapsed; - if (aeWait(fd,AE_READABLE,wait) & AE_READABLE) { - nread = read(fd,ptr,size); - if (nread <= 0) return -1; + /* Optimistically try to read before checking if the file descriptor + * is actually readable. At worst we get EAGAIN. */ + nread = read(fd,ptr,size); + if (nread == 0) return -1; /* short read. */ + if (nread == -1) { + if (errno != EAGAIN) return -1; + } else { ptr += nread; size -= nread; totread += nread; - if (size == 0) return totread; } + if (size == 0) return totread; + + /* Wait */ + aeWait(fd,AE_READABLE,wait); elapsed = mstime() - start; if (elapsed >= timeout) { errno = ETIMEDOUT; From 7f10703b3318e5490d70cd89af5f0953e6041860 Mon Sep 17 00:00:00 2001 From: quiver Date: Fri, 4 May 2012 20:07:00 +0900 Subject: [PATCH 0219/1752] fix several bugs of init.d scripts - PIDFILE environ variable was not properly retrieved - chkconfig command failed --- utils/redis_init_script.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/redis_init_script.tpl b/utils/redis_init_script.tpl index e0c5b2f88fe..e2af1fe7757 100755 --- a/utils/redis_init_script.tpl +++ b/utils/redis_init_script.tpl @@ -1,7 +1,7 @@ case "$1" in start) - if [ -f $$PIDFILE ] + if [ -f $PIDFILE ] then echo "$PIDFILE exists, process is already running or crashed" else From 1b1cf8e7bf8d3088caa25159a964b1692ae791c9 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 4 May 2012 11:54:06 -0700 Subject: [PATCH 0220/1752] Fix `install` target on OSX (see #495) --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4f08126df79..e614ede891f 100644 --- a/Makefile +++ b/Makefile @@ -5,3 +5,7 @@ default: all .DEFAULT: cd src && $(MAKE) $@ +install: + cd src && $(MAKE) $@ + +.PHONY: install From 0ef889274f104dd1e69021deec1d73ae2a1090a2 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 4 May 2012 17:26:24 -0700 Subject: [PATCH 0221/1752] Compare integers in ziplist regardless of encoding Because of the introduction of new integer encoding types for ziplists in the 2.6 tree, the same integer value may have a different encoding in different versions of the ziplist implementation. This means that the encoding can NOT be used as a fast path in comparing integers. --- src/ziplist.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index e3741f81eb1..31e61633e5b 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -773,12 +773,11 @@ unsigned int ziplistCompare(unsigned char *p, unsigned char *sstr, unsigned int return 0; } } else { - /* Try to compare encoded values */ + /* Try to compare encoded values. Don't compare encoding because + * different implementations may encoded integers differently. */ if (zipTryEncoding(sstr,slen,&sval,&sencoding)) { - if (entry.encoding == sencoding) { - zval = zipLoadInteger(p+entry.headersize,entry.encoding); - return zval == sval; - } + zval = zipLoadInteger(p+entry.headersize,entry.encoding); + return zval == sval; } } return 0; From 184b8e78c62030f4489f8be8607a738fa0122bbe Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 6 May 2012 10:11:54 +0200 Subject: [PATCH 0222/1752] Redis 2.5.9 (2.6 RC3). --- 00-RELEASENOTES | 9 +++++++++ src/version.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index f4505d11d48..f379a2441a4 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -35,6 +35,15 @@ Also the following redis.conf and CONFIG GET / SET parameters changed name: CHANGELOG --------- +What's new in Redis 2.5.9 (aka 2.6 Release Candidate 3) +======================================================= + +UPGRADE URGENCY: critical, upgrade ASAP. + +* [BUGFIX] Fix for issue #500 (https://github.com/antirez/redis/pull/500). + Redis 2.6-RC1 and RC2 may corrupt ziplist-encoded sorted sets + produced by Redis 2.4.x. + What's new in Redis 2.5.8 (aka 2.6 Release Candidate 2) ======================================================= diff --git a/src/version.h b/src/version.h index 64fced48aae..c4e2caf3546 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.8" +#define REDIS_VERSION "2.5.9" From af2455be29d71a3acb755275fbe7988e16f04ce2 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 6 May 2012 10:15:40 +0200 Subject: [PATCH 0223/1752] More complete release notes for 2.5.9 --- 00-RELEASENOTES | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index f379a2441a4..6227632a96a 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -43,6 +43,13 @@ UPGRADE URGENCY: critical, upgrade ASAP. * [BUGFIX] Fix for issue #500 (https://github.com/antirez/redis/pull/500). Redis 2.6-RC1 and RC2 may corrupt ziplist-encoded sorted sets produced by Redis 2.4.x. +* [BUGFIX] Fixed several bugs in init.d script. +* [BUGFIX] syncio.c functions modified for speed and correctness. On osx + (and possibly other BSD-based systems) the slave would block on + replication to send the SYNC command when the master was not + available. This is fixed now, but was not affecting Linux installs. +* Now when slave-serve-stake-data is set to yes and the master is down, instead + of reporting a generic error Redis replies with -MASTERDOWN. What's new in Redis 2.5.8 (aka 2.6 Release Candidate 2) ======================================================= From 8afa5b707228bce240189b3f8d69e70cfd09a318 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 6 May 2012 10:22:07 +0200 Subject: [PATCH 0224/1752] Fixed typo in RC3 release notes. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 6227632a96a..b232fdb200f 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -48,7 +48,7 @@ UPGRADE URGENCY: critical, upgrade ASAP. (and possibly other BSD-based systems) the slave would block on replication to send the SYNC command when the master was not available. This is fixed now, but was not affecting Linux installs. -* Now when slave-serve-stake-data is set to yes and the master is down, instead +* Now when slave-serve-stale-data is set to yes and the master is down, instead of reporting a generic error Redis replies with -MASTERDOWN. What's new in Redis 2.5.8 (aka 2.6 Release Candidate 2) From f580a3e3a0824a647038628ca26d6c5bd7d7b730 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 9 May 2012 10:34:52 +0200 Subject: [PATCH 0225/1752] Allow PREFIX to be overrided in Makefile. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index dce0324b776..c60199aaa0f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -79,7 +79,7 @@ endif REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) -PREFIX= /usr/local +PREFIX=? /usr/local INSTALL_BIN= $(PREFIX)/bin INSTALL= cp -pf From 91d18504c2c0fd5f5af2d6b9c89abb5d3d1b627d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 9 May 2012 20:45:00 +0200 Subject: [PATCH 0226/1752] Fix PREFIX typo in Makefile. --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index c60199aaa0f..b9e183363a0 100644 --- a/src/Makefile +++ b/src/Makefile @@ -79,7 +79,7 @@ endif REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) -PREFIX=? /usr/local +PREFIX?=/usr/local INSTALL_BIN= $(PREFIX)/bin INSTALL= cp -pf From dd4e8203b2e9d79af5ed96562488415ca8a74815 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 6 May 2012 10:05:31 +0200 Subject: [PATCH 0227/1752] redis-cli --pipe for mass import. --- src/Makefile | 2 +- src/redis-cli.c | 144 +++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 3 deletions(-) diff --git a/src/Makefile b/src/Makefile index b9e183363a0..e8543d95732 100644 --- a/src/Makefile +++ b/src/Makefile @@ -98,7 +98,7 @@ endif REDIS_SERVER_NAME= redis-server REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o REDIS_CLI_NAME= redis-cli -REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o +REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o REDIS_BENCHMARK_NAME= redis-benchmark REDIS_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o REDIS_CHECK_DUMP_NAME= redis-check-dump diff --git a/src/redis-cli.c b/src/redis-cli.c index f7fe54fa893..1603c2d008b 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -46,6 +46,8 @@ #include "zmalloc.h" #include "linenoise.h" #include "help.h" +#include "anet.h" +#include "ae.h" #define REDIS_NOTUSED(V) ((void) V) @@ -69,6 +71,7 @@ static struct config { int cluster_mode; int cluster_reissue_command; int slave_mode; + int pipe_mode; int bigkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; @@ -656,6 +659,8 @@ static int parseOptions(int argc, char **argv) { config.latency_mode = 1; } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; + } else if (!strcmp(argv[i],"--pipe")) { + config.pipe_mode = 1; } else if (!strcmp(argv[i],"--bigkeys")) { config.bigkeys = 1; } else if (!strcmp(argv[i],"--eval") && !lastarg) { @@ -714,6 +719,7 @@ static void usage() { " --raw Use raw formatting for replies (default when STDOUT is not a tty)\n" " --latency Enter a special mode continuously sampling latency.\n" " --slave Simulate a slave showing commands received from the master.\n" +" --pipe Transfer raw Redis protocol from stdin to server.\n" " --bigkeys Sample Redis keys looking for big keys.\n" " --eval Send an EVAL command using the Lua script at .\n" " --help Output this help and exit\n" @@ -968,6 +974,133 @@ static void slaveMode(void) { while (cliReadReply(0) == REDIS_OK); } +static void pipeMode(void) { + int fd = context->fd; + long long errors = 0, replies = 0, obuf_len = 0, obuf_pos = 0; + char ibuf[1024*16], obuf[1024*16]; /* Input and output buffers */ + char aneterr[ANET_ERR_LEN]; + redisReader *reader = redisReaderCreate(); + redisReply *reply; + int eof = 0; /* True once we consumed all the standard input. */ + int done = 0; + char magic[20]; /* Special reply we recognize. */ + + srand(time(NULL)); + + /* Use non blocking I/O. */ + if (anetNonBlock(aneterr,fd) == ANET_ERR) { + fprintf(stderr, "Can't set the socket in non blocking mode: %s\n", + aneterr); + exit(1); + } + + /* Transfer raw protocol and read replies from the server at the same + * time. */ + while(!done) { + int mask = AE_READABLE; + + if (!eof || obuf_len != 0) mask |= AE_WRITABLE; + mask = aeWait(fd,mask,1000); + + /* Handle the readable state: we can read replies from the server. */ + if (mask & AE_READABLE) { + ssize_t nread; + + /* Read from socket and feed the hiredis reader. */ + do { + nread = read(fd,ibuf,sizeof(ibuf)); + if (nread == -1 && errno != EAGAIN) { + fprintf(stderr, "Error reading from the server: %s\n", + strerror(errno)); + exit(1); + } + if (nread > 0) redisReaderFeed(reader,ibuf,nread); + } while(nread > 0); + + /* Consume replies. */ + do { + if (redisReaderGetReply(reader,(void**)&reply) == REDIS_ERR) { + fprintf(stderr, "Error reading replies from server\n"); + exit(1); + } + if (reply) { + if (reply->type == REDIS_REPLY_ERROR) { + fprintf(stderr,"%s\n", reply->str); + errors++; + } else if (eof && reply->type == REDIS_REPLY_STRING && + reply->len == 20) { + /* Check if this is the reply to our final ECHO + * command. If so everything was received + * from the server. */ + if (memcmp(reply->str,magic,20) == 0) { + printf("Last reply received from server.\n"); + done = 1; + replies--; + } + } + replies++; + freeReplyObject(reply); + } + } while(reply); + } + + /* Handle the writable state: we can send protocol to the server. */ + if (mask & AE_WRITABLE) { + while(1) { + /* Transfer current buffer to server. */ + if (obuf_len != 0) { + ssize_t nwritten = write(fd,obuf+obuf_pos,obuf_len); + + if (nwritten == -1) { + fprintf(stderr, "Error writing to the server: %s\n", + strerror(errno)); + exit(1); + } + obuf_len -= nwritten; + obuf_pos += nwritten; + if (obuf_len != 0) break; /* Can't accept more data. */ + } + /* If buffer is empty, load from stdin. */ + if (obuf_len == 0 && !eof) { + ssize_t nread = read(STDIN_FILENO,obuf,sizeof(obuf)); + + if (nread == 0) { + char echo[] = + "*2\r\n$4\r\nECHO\r\n$20\r\n01234567890123456789\r\n"; + int j; + + eof = 1; + /* Everything transfered, so we queue a special + * ECHO command that we can match in the replies + * to make sure everything was read from the server. */ + for (j = 0; j < 20; j++) + magic[j] = rand() & 0xff; + memcpy(echo+19,magic,20); + memcpy(obuf,echo,sizeof(echo)-1); + obuf_len = sizeof(echo)-1; + obuf_pos = 0; + printf("All data transferred. Waiting for the last reply...\n"); + } else if (nread == -1) { + fprintf(stderr, "Error reading from stdin: %s\n", + strerror(errno)); + exit(1); + } else { + obuf_len = nread; + obuf_pos = 0; + } + } + if (obuf_len == 0 && eof) break; + } + } + } + redisReaderFree(reader); + printf("errors: %lld, replies: %lld\n", errors, replies); + if (errors) + exit(1); + else + exit(0); +} + #define TYPE_STRING 0 #define TYPE_LIST 1 #define TYPE_SET 2 @@ -1065,6 +1198,7 @@ int main(int argc, char **argv) { config.latency_mode = 0; config.cluster_mode = 0; config.slave_mode = 0; + config.pipe_mode = 0; config.bigkeys = 0; config.stdinarg = 0; config.auth = NULL; @@ -1080,18 +1214,24 @@ int main(int argc, char **argv) { argc -= firstarg; argv += firstarg; - /* Start in latency mode if appropriate */ + /* Latency mode */ if (config.latency_mode) { cliConnect(0); latencyMode(); } - /* Start in slave mode if appropriate */ + /* Slave mode */ if (config.slave_mode) { cliConnect(0); slaveMode(); } + /* Pipe mode */ + if (config.pipe_mode) { + cliConnect(0); + pipeMode(); + } + /* Find big keys */ if (config.bigkeys) { cliConnect(0); From 346825c7ed8e5e57704ff2603a51435249b7c59e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 May 2012 10:45:12 +0200 Subject: [PATCH 0228/1752] redis-cli pipe mode: handle EAGAIN while writing to socket. --- src/redis-cli.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 1603c2d008b..1b3c0c74086 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1052,9 +1052,13 @@ static void pipeMode(void) { ssize_t nwritten = write(fd,obuf+obuf_pos,obuf_len); if (nwritten == -1) { - fprintf(stderr, "Error writing to the server: %s\n", - strerror(errno)); - exit(1); + if (errno != EAGAIN) { + fprintf(stderr, "Error writing to the server: %s\n", + strerror(errno)); + exit(1); + } else { + nwritten = 0; + } } obuf_len -= nwritten; obuf_pos += nwritten; From 25496f47005f87dfb29ca4b797927b3cba555225 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 May 2012 16:08:57 +0200 Subject: [PATCH 0229/1752] redis-cli pipe mode: handle EINTR properly as well so that SIGSTOP/SIGCONT are handled correctly. --- src/redis-cli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 1b3c0c74086..ca2f06233dc 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1009,7 +1009,7 @@ static void pipeMode(void) { /* Read from socket and feed the hiredis reader. */ do { nread = read(fd,ibuf,sizeof(ibuf)); - if (nread == -1 && errno != EAGAIN) { + if (nread == -1 && errno != EAGAIN && errno != EINTR) { fprintf(stderr, "Error reading from the server: %s\n", strerror(errno)); exit(1); @@ -1052,7 +1052,7 @@ static void pipeMode(void) { ssize_t nwritten = write(fd,obuf+obuf_pos,obuf_len); if (nwritten == -1) { - if (errno != EAGAIN) { + if (errno != EAGAIN && errno != EINTR) { fprintf(stderr, "Error writing to the server: %s\n", strerror(errno)); exit(1); From 064223107e3ce2a8cae6e35245bb1037ff5b984c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 May 2012 17:26:09 +0200 Subject: [PATCH 0230/1752] If the computer running the Redis test is slow, we revert to --clients 1 to avoid false positives. --- tests/test_helper.tcl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 598a3929165..7874256da64 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -404,6 +404,18 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +# With the parallel test running multiple Redis instances at the same time +# we need a fast enough computer, otherwise a lot of tests may generate +# false positives. +# If the computer is too slow we revert the sequetial test without any +# parallelism, that is, clients == 1. +proc is_a_slow_computer {} { + set start [clock milliseconds] + for {set j 0} {$j < 1000000} {incr j} {} + set elapsed [expr [clock milliseconds]-$start] + expr {$elapsed > 200} +} + if {$::client} { if {[catch { test_client_main $::test_server_port } err]} { set estr "Executing test client: $err.\n$::errorInfo" @@ -413,6 +425,11 @@ if {$::client} { exit 1 } } else { + if {[is_a_slow_computer]} { + puts "** SLOW COMPUTER ** Using a single client to avoid false positives." + set ::numclients 1 + } + if {[catch { test_server_main } err]} { if {[string length $err] > 0} { # only display error when not generated by the test suite From 3a401464e9c42fafeaa9405db60d10b3256aca51 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 May 2012 19:17:31 +0200 Subject: [PATCH 0231/1752] More incremental active expired keys collection process. If a large amonut of keys are all expiring about at the same time, the "active" expired keys collection cycle used to block as far as the percentage of already expired keys was >= 25% of the total population of keys with an expire set. This could block the server even for many seconds in order to reclaim memory ASAP. The new algorithm uses at max a small amount of milliseconds per cycle, even if this means reclaiming the memory less promptly it also means a more responsive server. --- src/redis.c | 9 ++++++++- src/redis.h | 1 + 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 0a12ccd2e2e..4cc378e24a4 100644 --- a/src/redis.c +++ b/src/redis.c @@ -609,9 +609,10 @@ void updateDictResizePolicy(void) { * keys that can be removed from the keyspace. */ void activeExpireCycle(void) { int j; + long long start = mstime(); for (j = 0; j < server.dbnum; j++) { - int expired; + int expired, iteration = 0; redisDb *db = server.db+j; /* Continue to expire if at the end of the cycle more than 25% @@ -640,6 +641,12 @@ void activeExpireCycle(void) { server.stat_expiredkeys++; } } + /* We can't block forever here even if there are many keys to + * expire. So after a given amount of milliseconds return to the + * caller waiting for the other active expire cycle. */ + iteration++; + if ((iteration & 0xff) == 0 && /* & 0xff is the same as % 255 */ + (mstime()-start) > REDIS_EXPIRELOOKUPS_TIME_LIMIT) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } } diff --git a/src/redis.h b/src/redis.h index 281c0878789..b842b5f7905 100644 --- a/src/redis.h +++ b/src/redis.h @@ -43,6 +43,7 @@ #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_EXPIRELOOKUPS_PER_CRON 10 /* lookup 10 expires per loop */ +#define REDIS_EXPIRELOOKUPS_TIME_LIMIT 25 /* Time limit in milliseconds */ #define REDIS_MAX_WRITE_PER_EVENT (1024*64) #define REDIS_SHARED_SELECT_CMDS 10 #define REDIS_SHARED_INTEGERS 10000 From f078d5628d5931903111e384d55bacf205986fe5 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 May 2012 22:33:28 +0200 Subject: [PATCH 0232/1752] Comment improved so that the code goal is more clear. Thx to @agladysh. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 4cc378e24a4..ecf02e0a378 100644 --- a/src/redis.c +++ b/src/redis.c @@ -645,7 +645,7 @@ void activeExpireCycle(void) { * expire. So after a given amount of milliseconds return to the * caller waiting for the other active expire cycle. */ iteration++; - if ((iteration & 0xff) == 0 && /* & 0xff is the same as % 255 */ + if ((iteration & 0xff) == 0 && /* Check once every 255 iterations */ (mstime()-start) > REDIS_EXPIRELOOKUPS_TIME_LIMIT) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } From f7f2b2610e8181c13657c35e4f3cd06c94ba3ff9 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 13 May 2012 16:40:29 +0200 Subject: [PATCH 0233/1752] Redis timer interrupt frequency configurable as REDIS_HZ. Redis uses a function called serverCron() that is very similar to the timer interrupt of an operating system. This function is used to handle a number of asynchronous things, like active expired keys collection, clients timeouts, update of statistics, things related to the cluster and replication, triggering of BGSAVE and AOF rewrite process, and so forth. In the past the timer was called 1 time per second. At some point it was raised to 10 times per second, but it still was fixed and could not be changed even at compile time, because different functions called from serverCron() assumed a given fixed frequency. This commmit makes the frequency configurable, so that it is simpler to pick a good tradeoff between overhead of this function (that is usually very small) and the responsiveness of Redis during a few critical circumstances where a lot of work is done inside the timer. An example of such a critical condition is mass-expire of a lot of keys in the same second. Up to a given percentage of CPU time is used to perform expired keys collection per expire cylce. Now changing the REDIS_HZ macro it is possible to do less work but more times per second in order to block the server for less time. If this patch will work well in our tests it will enter Redis 2.6-final. --- src/redis.c | 71 +++++++++++++++++++++++++++++++++++++++-------------- src/redis.h | 3 ++- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/redis.c b/src/redis.c index ecf02e0a378..7505de489f3 100644 --- a/src/redis.c +++ b/src/redis.c @@ -609,7 +609,14 @@ void updateDictResizePolicy(void) { * keys that can be removed from the keyspace. */ void activeExpireCycle(void) { int j; - long long start = mstime(); + long long start = mstime(), timelimit; + + /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time + * per iteration. Since this function gets called with a frequency of + * REDIS_HZ times per second, the following is the max amount of + * milliseconds we can spend here: */ + timelimit = (1000/REDIS_HZ/100)*REDIS_EXPIRELOOKUPS_TIME_PERC; + if (timelimit <= 0) timelimit = 1; for (j = 0; j < server.dbnum; j++) { int expired, iteration = 0; @@ -646,7 +653,7 @@ void activeExpireCycle(void) { * caller waiting for the other active expire cycle. */ iteration++; if ((iteration & 0xff) == 0 && /* Check once every 255 iterations */ - (mstime()-start) > REDIS_EXPIRELOOKUPS_TIME_LIMIT) return; + (mstime()-start) > timelimit) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } } @@ -732,13 +739,13 @@ int clientsCronResizeQueryBuffer(redisClient *c) { } void clientsCron(void) { - /* Make sure to process at least 1/100 of clients per call. - * Since this function is called 10 times per second we are sure that + /* Make sure to process at least 1/(REDIS_HZ*10) of clients per call. + * Since this function is called REDIS_HZ times per second we are sure that * in the worst case we process all the clients in 10 seconds. * In normal conditions (a reasonable number of clients) we process * all the clients in a shorter time. */ int numclients = listLength(server.clients); - int iterations = numclients/100; + int iterations = numclients/(REDIS_HZ*10); if (iterations < 50) iterations = (numclients < 50) ? numclients : 50; @@ -760,6 +767,30 @@ void clientsCron(void) { } } +/* This is our timer interrupt, called REDIS_HZ times per second. + * Here is where we do a number of things that need to be done asynchronously. + * For instance: + * + * - Active expired keys collection (it is also performed in a lazy way on + * lookup). + * - Software watchdong. + * - Update some statistic. + * - Incremental rehashing of the DBs hash tables. + * - Triggering BGSAVE / AOF rewrite, and handling of terminated children. + * - Clients timeout of differnet kinds. + * - Replication reconnection. + * - Many more... + * + * Everything directly called here will be called REDIS_HZ times per second, + * so in order to throttle execution of things we want to do less frequently + * a macro is used: run_with_period(milliseconds) { .... } + */ + +/* Using the following macro you can run code inside serverCron() with the + * specified period, specified in milliseconds. + * The actual resolution depends on REDIS_HZ. */ +#define run_with_period(_ms_) if (!(loops % ((_ms_)/(1000/REDIS_HZ)))) + int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { int j, loops = server.cronloops; REDIS_NOTUSED(eventLoop); @@ -776,7 +807,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * To access a global var is faster than calling time(NULL) */ server.unixtime = time(NULL); - trackOperationsPerSecond(); + run_with_period(100) trackOperationsPerSecond(); /* We have just 22 bits per object for LRU information. * So we use an (eventually wrapping) LRU clock with 10 seconds resolution. @@ -804,15 +835,17 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } /* Show some info about non-empty databases */ - for (j = 0; j < server.dbnum; j++) { - long long size, used, vkeys; - - size = dictSlots(server.db[j].dict); - used = dictSize(server.db[j].dict); - vkeys = dictSize(server.db[j].expires); - if (!(loops % 50) && (used || vkeys)) { - redisLog(REDIS_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size); - /* dictPrintStats(server.dict); */ + run_with_period(5000) { + for (j = 0; j < server.dbnum; j++) { + long long size, used, vkeys; + + size = dictSlots(server.db[j].dict); + used = dictSize(server.db[j].dict); + vkeys = dictSize(server.db[j].expires); + if (used || vkeys) { + redisLog(REDIS_VERBOSE,"DB %d: %lld keys (%lld volatile) in %lld slots HT.",j,used,vkeys,size); + /* dictPrintStats(server.dict); */ + } } } @@ -823,12 +856,12 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * a lot of memory movements in the parent will cause a lot of pages * copied. */ if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { - if (!(loops % 10)) tryResizeHashTables(); + run_with_period(1000) tryResizeHashTables(); if (server.activerehashing) incrementallyRehash(); } /* Show information about connected clients */ - if (!(loops % 50)) { + run_with_period(5000) { redisLog(REDIS_VERBOSE,"%d clients connected (%d slaves), %zu bytes in use", listLength(server.clients)-listLength(server.slaves), listLength(server.slaves), @@ -910,10 +943,10 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* Replication cron function -- used to reconnect to master and * to detect transfer failures. */ - if (!(loops % 10)) replicationCron(); + run_with_period(1000) replicationCron(); server.cronloops++; - return 100; + return 1000/REDIS_HZ; } /* This function gets called every time Redis is entering the diff --git a/src/redis.h b/src/redis.h index b842b5f7905..d476c72478f 100644 --- a/src/redis.h +++ b/src/redis.h @@ -38,12 +38,13 @@ #define REDIS_ERR -1 /* Static server configuration */ +#define REDIS_HZ 10 /* Time interrupt calls/sec. */ #define REDIS_SERVERPORT 6379 /* TCP port */ #define REDIS_MAXIDLETIME 0 /* default client timeout: infinite */ #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_EXPIRELOOKUPS_PER_CRON 10 /* lookup 10 expires per loop */ -#define REDIS_EXPIRELOOKUPS_TIME_LIMIT 25 /* Time limit in milliseconds */ +#define REDIS_EXPIRELOOKUPS_TIME_PERC 25 /* CPU max % for keys collection */ #define REDIS_MAX_WRITE_PER_EVENT (1024*64) #define REDIS_SHARED_SELECT_CMDS 10 #define REDIS_SHARED_INTEGERS 10000 From a8a981a834f1e8704ed2250ec14646e01760756b Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 13 May 2012 21:52:35 +0200 Subject: [PATCH 0234/1752] Impovements for: Redis timer, hashes rehashing, keys collection. A previous commit introduced REDIS_HZ define that changes the frequency of calls to the serverCron() Redis function. This commit improves different related things: 1) Software watchdog: now the minimal period can be set according to REDIS_HZ. The minimal period is two times the timer period, that is: (1000/REDIS_HZ)*2 milliseconds 2) The incremental rehashing is now performed in the expires dictionary as well. 3) The activeExpireCycle() function was improved in different ways: - Now it checks if it already used too much time using microseconds instead of milliseconds for better precision. - The time limit is now calculated correctly, in the previous version the division was performed before of the multiplication resulting in a timelimit of 0 if HZ was big enough. - Databases with less than 1% of buckets fill in the hash table are skipped, because getting random keys is too expensive in this condition. 4) tryResizeHashTables() is now called at every timer call, we need to match the number of calls we do to the expired keys colleciton cycle. 5) REDIS_HZ was raised to 100. --- src/debug.c | 8 +++++++- src/redis.c | 25 +++++++++++++++++++------ src/redis.h | 2 +- 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/debug.c b/src/debug.c index 42a73883d9d..4687fb6c072 100644 --- a/src/debug.c +++ b/src/debug.c @@ -722,6 +722,8 @@ void watchdogScheduleSignal(int period) { /* Enable the software watchdong with the specified period in milliseconds. */ void enableWatchdog(int period) { + int min_period; + if (server.watchdog_period == 0) { struct sigaction act; @@ -732,7 +734,11 @@ void enableWatchdog(int period) { act.sa_sigaction = watchdogSignalHandler; sigaction(SIGALRM, &act, NULL); } - if (period < 200) period = 200; /* We don't accept periods < 200 ms. */ + /* If the configured period is smaller than twice the timer period, it is + * too short for the software watchdog to work reliably. Fix it now + * if needed. */ + min_period = (1000/REDIS_HZ)*2; + if (period < min_period) period = min_period; watchdogScheduleSignal(period); /* Adjust the current timer. */ server.watchdog_period = period; } diff --git a/src/redis.c b/src/redis.c index 7505de489f3..48d032b6c41 100644 --- a/src/redis.c +++ b/src/redis.c @@ -581,10 +581,16 @@ void incrementallyRehash(void) { int j; for (j = 0; j < server.dbnum; j++) { + /* Keys dictionary */ if (dictIsRehashing(server.db[j].dict)) { dictRehashMilliseconds(server.db[j].dict,1); break; /* already used our millisecond for this loop... */ } + /* Expires */ + if (dictIsRehashing(server.db[j].expires)) { + dictRehashMilliseconds(server.db[j].expires,1); + break; /* already used our millisecond for this loop... */ + } } } @@ -609,13 +615,13 @@ void updateDictResizePolicy(void) { * keys that can be removed from the keyspace. */ void activeExpireCycle(void) { int j; - long long start = mstime(), timelimit; + long long start = ustime(), timelimit; /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time * per iteration. Since this function gets called with a frequency of * REDIS_HZ times per second, the following is the max amount of - * milliseconds we can spend here: */ - timelimit = (1000/REDIS_HZ/100)*REDIS_EXPIRELOOKUPS_TIME_PERC; + * microseconds we can spend in this function. */ + timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/REDIS_HZ/100; if (timelimit <= 0) timelimit = 1; for (j = 0; j < server.dbnum; j++) { @@ -625,9 +631,16 @@ void activeExpireCycle(void) { /* Continue to expire if at the end of the cycle more than 25% * of the keys were expired. */ do { - long num = dictSize(db->expires); + unsigned long num = dictSize(db->expires); + unsigned long slots = dictSlots(db->expires); long long now = mstime(); + /* When there are less than 1% filled slots getting random + * keys is expensive, so stop here waiting for better times... + * The dictionary will be resized asap. */ + if (num && slots > DICT_HT_INITIAL_SIZE && + (num*100/slots < 1)) break; + expired = 0; if (num > REDIS_EXPIRELOOKUPS_PER_CRON) num = REDIS_EXPIRELOOKUPS_PER_CRON; @@ -653,7 +666,7 @@ void activeExpireCycle(void) { * caller waiting for the other active expire cycle. */ iteration++; if ((iteration & 0xff) == 0 && /* Check once every 255 iterations */ - (mstime()-start) > timelimit) return; + (ustime()-start) > timelimit) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } } @@ -856,7 +869,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * a lot of memory movements in the parent will cause a lot of pages * copied. */ if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { - run_with_period(1000) tryResizeHashTables(); + tryResizeHashTables(); if (server.activerehashing) incrementallyRehash(); } diff --git a/src/redis.h b/src/redis.h index d476c72478f..89524160661 100644 --- a/src/redis.h +++ b/src/redis.h @@ -38,7 +38,7 @@ #define REDIS_ERR -1 /* Static server configuration */ -#define REDIS_HZ 10 /* Time interrupt calls/sec. */ +#define REDIS_HZ 100 /* Time interrupt calls/sec. */ #define REDIS_SERVERPORT 6379 /* TCP port */ #define REDIS_MAXIDLETIME 0 /* default client timeout: infinite */ #define REDIS_DEFAULT_DBNUM 16 From eee680541b4ce3abbb86c39f47932c648fe478f5 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 14 May 2012 16:04:41 +0200 Subject: [PATCH 0235/1752] activeExpireCycle(): better precision in max time used. activeExpireCycle() can consume no more than a few milliseconds per iteration. This commit improves the precision of the check for the time elapsed in two ways: 1) We check every 16 iterations instead of the main loop instead of 256. 2) We reset iterations at the start of the function and not every time we switch to the next database, so the check is correctly performed every 16 iterations. --- src/redis.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 48d032b6c41..e12d27cc339 100644 --- a/src/redis.c +++ b/src/redis.c @@ -614,7 +614,7 @@ void updateDictResizePolicy(void) { * it will get more aggressive to avoid that too much memory is used by * keys that can be removed from the keyspace. */ void activeExpireCycle(void) { - int j; + int j, iteration = 0; long long start = ustime(), timelimit; /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time @@ -625,7 +625,7 @@ void activeExpireCycle(void) { if (timelimit <= 0) timelimit = 1; for (j = 0; j < server.dbnum; j++) { - int expired, iteration = 0; + int expired; redisDb *db = server.db+j; /* Continue to expire if at the end of the cycle more than 25% @@ -641,6 +641,8 @@ void activeExpireCycle(void) { if (num && slots > DICT_HT_INITIAL_SIZE && (num*100/slots < 1)) break; + /* The main collection cycle. Sample random keys among keys + * with an expire set, checking for expired ones. */ expired = 0; if (num > REDIS_EXPIRELOOKUPS_PER_CRON) num = REDIS_EXPIRELOOKUPS_PER_CRON; @@ -665,7 +667,7 @@ void activeExpireCycle(void) { * expire. So after a given amount of milliseconds return to the * caller waiting for the other active expire cycle. */ iteration++; - if ((iteration & 0xff) == 0 && /* Check once every 255 iterations */ + if ((iteration & 0xf) == 0 && /* check once every 16 cycles. */ (ustime()-start) > timelimit) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } From e67d014d9a4a5a7c9dbab3f2ace52f25661e7a64 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 14 May 2012 17:35:51 +0200 Subject: [PATCH 0236/1752] Added time.h include in redis-cli. redis-cli.c uses the time() function to seed the PRNG, but time.h was not included. This was not noticed since sys/time.h is included and was enough in most systems (but not correct). With Ubuntu 12.04 GCC generates a warning that made us aware of the issue. --- src/redis-cli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index ca2f06233dc..f485587957d 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include From 2aa1efb8a5bba003e94604514fbccea8115ca90d Mon Sep 17 00:00:00 2001 From: Dave Pacheco Date: Mon, 26 Mar 2012 17:58:19 -0700 Subject: [PATCH 0237/1752] first cut at event port support --- src/ae.c | 14 ++- src/ae_evport.c | 254 ++++++++++++++++++++++++++++++++++++++++++++++++ src/config.h | 7 ++ 3 files changed, 270 insertions(+), 5 deletions(-) create mode 100644 src/ae_evport.c diff --git a/src/ae.c b/src/ae.c index 668277a7808..c58c4b74bcc 100644 --- a/src/ae.c +++ b/src/ae.c @@ -44,13 +44,17 @@ /* Include the best multiplexing layer supported by this system. * The following should be ordered by performances, descending. */ -#ifdef HAVE_EPOLL -#include "ae_epoll.c" +#ifdef HAVE_EVPORT +#include "ae_evport.c" #else - #ifdef HAVE_KQUEUE - #include "ae_kqueue.c" + #ifdef HAVE_EPOLL + #include "ae_epoll.c" #else - #include "ae_select.c" + #ifdef HAVE_KQUEUE + #include "ae_kqueue.c" + #else + #include "ae_select.c" + #endif #endif #endif diff --git a/src/ae_evport.c b/src/ae_evport.c new file mode 100644 index 00000000000..4a3334203aa --- /dev/null +++ b/src/ae_evport.c @@ -0,0 +1,254 @@ +/* ae.c module for illumos event ports. + * Copyright (c) 2012, Joyent, Inc. All rights reserved. + * Released under the BSD license. See the COPYING file for more info. */ + +#include +#include +#include +#include + +#include +#include + +#include + +int evport_debug = 0; + +/* + * This file implements the ae API using event ports, present on Solaris-based + * systems since Solaris 10. Using the event port interface, we associate file + * descriptors with the port. Each association also includes the set of poll(2) + * events that the consumer is interested in (e.g., POLLIN and POLLOUT). + * + * There's one tricky piece to this implementation: when we return events via + * aeApiPoll, the corresponding file descriptor becomes dissociated from the + * port. This is necessary because poll events are level-triggered, so if the + * fd didn't become dissociated, it would immediately fire another event since + * the underlying state hasn't changed yet. We must reassociate the file + * descriptor, but only after we know that our caller has actually read from it. + * The ae API does not tell us exactly when that happens, but we do know that + * it must happen by the time aeApiPoll is called again. Our solution is to + * keep track of the last fd returned by aeApiPoll and reassociate it next time + * aeApiPoll is invoked. + * + * To summarize, in this module, each fd association is EITHER (a) represented + * only via the in-kernel assocation OR (b) represented by pending_fd and + * pending_mask. (b) is only true for the last fd we returned from aeApiPoll, + * and only until we enter aeApiPoll again (at which point we restore the + * in-kernel association). + * + * We currently only return one fd event per call to aeApiPoll. This could be + * extended to return more than one by extending the corresponding pending + * fields and using port_getn(). + */ +typedef struct aeApiState { + int portfd; /* event port */ + int pending_fd; /* pending fd */ + int pending_mask; /* pending fd's mask */ +} aeApiState; + +static int aeApiCreate(aeEventLoop *eventLoop) { + aeApiState *state = zmalloc(sizeof(aeApiState)); + if (!state) return -1; + + state->portfd = port_create(); + if (state->portfd == -1) { + zfree(state); + return -1; + } + + state->pending_fd = -1; + state->pending_mask = AE_NONE; + + eventLoop->apidata = state; + return 0; +} + +static void aeApiFree(aeEventLoop *eventLoop) { + aeApiState *state = eventLoop->apidata; + + close(state->portfd); + zfree(state); +} + +/* + * Helper function to invoke port_associate for the given fd and mask. + */ +static int aeApiAssociate(const char *where, int portfd, int fd, int mask) +{ + int events = 0; + int rv, err; + + if (mask & AE_READABLE) + events |= POLLIN; + if (mask & AE_WRITABLE) + events |= POLLOUT; + + if (evport_debug) + fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events); + + rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, (void *)mask); + err = errno; + + if (evport_debug) + fprintf(stderr, "%d (%s)\n", rv, rv == 0 ? "no error" : strerror(err)); + + if (rv == -1) { + fprintf(stderr, "%s: port_associate: %s\n", where, strerror(err)); + + if (err == EAGAIN) + fprintf(stderr, "aeApiAssociate: event port limit exceeded."); + } + + return rv; +} + +static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask; + + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask); + + /* + * Since port_associate's "events" argument replaces any existing events, we + * must be sure to include whatever events are already associated when + * we call port_associate() again. + */ + fullmask = mask | eventLoop->events[fd].mask; + + if (fd == state->pending_fd) { + /* + * This fd was recently returned from aeApiPoll. It should be safe to + * assume that the consumer has processed that poll event, but we play + * it safer by simply updating pending_mask. The fd will be + * reassociated as usual when aeApiPoll is called again. + */ + if (evport_debug) + fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); + state->pending_mask |= fullmask; + return 0; + } + + return (aeApiAssociate("aeApiAddEvent", state->portfd, fd, fullmask)); +} + +static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { + aeApiState *state = eventLoop->apidata; + int fullmask; + + if (evport_debug) + fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask); + + if (fd == state->pending_fd) { + if (evport_debug) + fprintf(stderr, "deleting event from pending fd %d\n", fd); + + /* + * This fd was just returned from aeApiPoll, so it's not currently + * associated with the port. All we need to do is update + * pending_mask appropriately. + */ + state->pending_mask &= ~mask; + + if (state->pending_mask == AE_NONE) + state->pending_fd = -1; + + return; + } + + /* + * The fd is currently associated with the port. Like with the add case + * above, we must look at the full mask for the file descriptor before + * updating that association. We don't have a good way of knowing what the + * events are without looking into the eventLoop state directly. We rely on + * the fact that our caller has already updated the mask in the eventLoop. + */ + + fullmask = eventLoop->events[fd].mask; + if (fullmask == AE_NONE) { + /* + * We're removing *all* events, so use port_dissociate to remove the + * association completely. Failure here indicates a bug. + */ + if (evport_debug) + fprintf(stderr, "aeApiDelEvent: port_dissociate(%d)\n", fd); + + if (port_dissociate(state->portfd, PORT_SOURCE_FD, fd) != 0) { + perror("aeApiDelEvent: port_dissociate"); + abort(); /* will not return */ + } + } else if (aeApiAssociate("aeApiDelEvent", state->portfd, fd, + fullmask) != 0) { + /* + * ENOMEM is a potentially transient condition, but the kernel won't + * generally return it unless things are really bad. EAGAIN indicates + * we've reached an resource limit, for which it doesn't make sense to + * retry (counterintuitively). All other errors indicate a bug. In any + * of these cases, the best we can do is to abort. + */ + abort(); /* will not return */ + } +} + +static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { + aeApiState *state = eventLoop->apidata; + struct timespec timeout, *tsp; + int mask; + port_event_t event; + + /* + * If we've returned an fd event before, we must reassociated that fd with + * the port now, before calling port_get(). See the block comment at the + * top of this file for an explanation of why. + */ + if (state->pending_mask != AE_NONE) { + if (aeApiAssociate("aeApiPoll", state->portfd, state->pending_fd, + state->pending_mask) != 0) { + /* See aeApiDelEvent for why this case is fatal. */ + abort(); + } + + state->pending_mask = AE_NONE; + state->pending_fd = -1; + } + + if (tvp != NULL) { + timeout.tv_sec = tvp->tv_sec; + timeout.tv_nsec = tvp->tv_usec * 1000; + tsp = &timeout; + } else { + tsp = NULL; + } + + if (port_get(state->portfd, &event, tsp) == -1) { + if (errno == ETIME || errno == EINTR) + return 0; + + /* Any other error indicates a bug. */ + perror("aeApiPoll: port_get"); + abort(); + } + + mask = 0; + if (event.portev_events & POLLIN) + mask |= AE_READABLE; + if (event.portev_events & POLLOUT) + mask |= AE_WRITABLE; + + eventLoop->fired[0].fd = event.portev_object; + eventLoop->fired[0].mask = mask; + + if (evport_debug) + fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", event.portev_object, + mask); + + state->pending_fd = event.portev_object; + state->pending_mask = (uintptr_t)event.portev_user; + + return 1; +} + +static char *aeApiName(void) { + return "evport"; +} diff --git a/src/config.h b/src/config.h index 136fd40c4c8..28ef37d6ed0 100644 --- a/src/config.h +++ b/src/config.h @@ -38,6 +38,13 @@ #define HAVE_KQUEUE 1 #endif +#ifdef __sun +#include +#ifdef _DTRACE_VERSION +#define HAVE_EVPORT 1 +#endif +#endif + /* Define aof_fsync to fdatasync() in Linux and fsync() for all the rest */ #ifdef __linux__ #define aof_fsync fdatasync From 72f30bcd309b4a182b51106aa874aa83734f0558 Mon Sep 17 00:00:00 2001 From: Dave Pacheco Date: Mon, 26 Mar 2012 20:41:58 -0700 Subject: [PATCH 0238/1752] use port_getn instead of port_get --- src/ae_evport.c | 135 ++++++++++++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 50 deletions(-) diff --git a/src/ae_evport.c b/src/ae_evport.c index 4a3334203aa..c4fc9a40b0e 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -12,7 +12,7 @@ #include -int evport_debug = 0; +static int evport_debug = 0; /* * This file implements the ae API using event ports, present on Solaris-based @@ -21,33 +21,33 @@ int evport_debug = 0; * events that the consumer is interested in (e.g., POLLIN and POLLOUT). * * There's one tricky piece to this implementation: when we return events via - * aeApiPoll, the corresponding file descriptor becomes dissociated from the + * aeApiPoll, the corresponding file descriptors become dissociated from the * port. This is necessary because poll events are level-triggered, so if the * fd didn't become dissociated, it would immediately fire another event since * the underlying state hasn't changed yet. We must reassociate the file * descriptor, but only after we know that our caller has actually read from it. * The ae API does not tell us exactly when that happens, but we do know that * it must happen by the time aeApiPoll is called again. Our solution is to - * keep track of the last fd returned by aeApiPoll and reassociate it next time - * aeApiPoll is invoked. + * keep track of the last fds returned by aeApiPoll and reassociate them next + * time aeApiPoll is invoked. * * To summarize, in this module, each fd association is EITHER (a) represented - * only via the in-kernel assocation OR (b) represented by pending_fd and - * pending_mask. (b) is only true for the last fd we returned from aeApiPoll, + * only via the in-kernel assocation OR (b) represented by pending_fds and + * pending_masks. (b) is only true for the last fds we returned from aeApiPoll, * and only until we enter aeApiPoll again (at which point we restore the * in-kernel association). - * - * We currently only return one fd event per call to aeApiPoll. This could be - * extended to return more than one by extending the corresponding pending - * fields and using port_getn(). */ +#define MAX_EVENT_BATCHSZ 512 + typedef struct aeApiState { - int portfd; /* event port */ - int pending_fd; /* pending fd */ - int pending_mask; /* pending fd's mask */ + int portfd; /* event port */ + int npending; /* # of pending fds */ + int pending_fds[MAX_EVENT_BATCHSZ]; /* pending fds */ + int pending_masks[MAX_EVENT_BATCHSZ]; /* pending fds' masks */ } aeApiState; static int aeApiCreate(aeEventLoop *eventLoop) { + int i; aeApiState *state = zmalloc(sizeof(aeApiState)); if (!state) return -1; @@ -57,8 +57,12 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return -1; } - state->pending_fd = -1; - state->pending_mask = AE_NONE; + state->npending = 0; + + for (i = 0; i < MAX_EVENT_BATCHSZ; i++) { + state->pending_fds[i] = -1; + state->pending_masks[i] = AE_NONE; + } eventLoop->apidata = state; return 0; @@ -71,11 +75,21 @@ static void aeApiFree(aeEventLoop *eventLoop) { zfree(state); } +static int aeApiLookupPending(aeApiState *state, int fd) { + int i; + + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == fd) + return (i); + } + + return (-1); +} + /* * Helper function to invoke port_associate for the given fd and mask. */ -static int aeApiAssociate(const char *where, int portfd, int fd, int mask) -{ +static int aeApiAssociate(const char *where, int portfd, int fd, int mask) { int events = 0; int rv, err; @@ -87,7 +101,8 @@ static int aeApiAssociate(const char *where, int portfd, int fd, int mask) if (evport_debug) fprintf(stderr, "%s: port_associate(%d, 0x%x) = ", where, fd, events); - rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, (void *)mask); + rv = port_associate(portfd, PORT_SOURCE_FD, fd, events, + (void *)(uintptr_t)mask); err = errno; if (evport_debug) @@ -105,7 +120,7 @@ static int aeApiAssociate(const char *where, int portfd, int fd, int mask) static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { aeApiState *state = eventLoop->apidata; - int fullmask; + int fullmask, pfd; if (evport_debug) fprintf(stderr, "aeApiAddEvent: fd %d mask 0x%x\n", fd, mask); @@ -116,8 +131,9 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { * we call port_associate() again. */ fullmask = mask | eventLoop->events[fd].mask; + pfd = aeApiLookupPending(state, fd); - if (fd == state->pending_fd) { + if (pfd != -1) { /* * This fd was recently returned from aeApiPoll. It should be safe to * assume that the consumer has processed that poll event, but we play @@ -126,7 +142,7 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { */ if (evport_debug) fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); - state->pending_mask |= fullmask; + state->pending_masks[pfd] |= fullmask; return 0; } @@ -135,12 +151,14 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { aeApiState *state = eventLoop->apidata; - int fullmask; + int fullmask, pfd; if (evport_debug) fprintf(stderr, "del fd %d mask 0x%x\n", fd, mask); - if (fd == state->pending_fd) { + pfd = aeApiLookupPending(state, fd); + + if (pfd != -1) { if (evport_debug) fprintf(stderr, "deleting event from pending fd %d\n", fd); @@ -149,10 +167,10 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { * associated with the port. All we need to do is update * pending_mask appropriately. */ - state->pending_mask &= ~mask; + state->pending_masks[pfd] &= ~mask; - if (state->pending_mask == AE_NONE) - state->pending_fd = -1; + if (state->pending_masks[pfd] == AE_NONE) + state->pending_fds[pfd] = -1; return; } @@ -194,25 +212,32 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { aeApiState *state = eventLoop->apidata; struct timespec timeout, *tsp; - int mask; - port_event_t event; + int mask, i; + uint_t nevents; + port_event_t event[MAX_EVENT_BATCHSZ]; /* - * If we've returned an fd event before, we must reassociated that fd with - * the port now, before calling port_get(). See the block comment at the - * top of this file for an explanation of why. + * If we've returned fd events before, we must reassociate them with the + * port now, before calling port_get(). See the block comment at the top of + * this file for an explanation of why. */ - if (state->pending_mask != AE_NONE) { - if (aeApiAssociate("aeApiPoll", state->portfd, state->pending_fd, - state->pending_mask) != 0) { + for (i = 0; i < state->npending; i++) { + if (state->pending_fds[i] == -1) + /* This fd has since been deleted. */ + continue; + + if (aeApiAssociate("aeApiPoll", state->portfd, + state->pending_fds[i], state->pending_masks[i]) != 0) { /* See aeApiDelEvent for why this case is fatal. */ abort(); } - state->pending_mask = AE_NONE; - state->pending_fd = -1; + state->pending_masks[i] = AE_NONE; + state->pending_fds[i] = -1; } + state->npending = 0; + if (tvp != NULL) { timeout.tv_sec = tvp->tv_sec; timeout.tv_nsec = tvp->tv_usec * 1000; @@ -221,7 +246,13 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { tsp = NULL; } - if (port_get(state->portfd, &event, tsp) == -1) { + /* + * port_getn can return with errno == ETIME having returned some events (!). + * So if we get ETIME, we check nevents, too. + */ + nevents = 1; + if (port_getn(state->portfd, event, MAX_EVENT_BATCHSZ, &nevents, + tsp) == -1 && (errno != ETIME || nevents == 0)) { if (errno == ETIME || errno == EINTR) return 0; @@ -230,23 +261,27 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { abort(); } - mask = 0; - if (event.portev_events & POLLIN) - mask |= AE_READABLE; - if (event.portev_events & POLLOUT) - mask |= AE_WRITABLE; + state->npending = nevents; - eventLoop->fired[0].fd = event.portev_object; - eventLoop->fired[0].mask = mask; + for (i = 0; i < nevents; i++) { + mask = 0; + if (event[i].portev_events & POLLIN) + mask |= AE_READABLE; + if (event[i].portev_events & POLLOUT) + mask |= AE_WRITABLE; - if (evport_debug) - fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", event.portev_object, - mask); + eventLoop->fired[i].fd = event[i].portev_object; + eventLoop->fired[i].mask = mask; - state->pending_fd = event.portev_object; - state->pending_mask = (uintptr_t)event.portev_user; + if (evport_debug) + fprintf(stderr, "aeApiPoll: fd %d mask 0x%x\n", + (int)event[i].portev_object, mask); + + state->pending_fds[i] = event[i].portev_object; + state->pending_masks[i] = (uintptr_t)event[i].portev_user; + } - return 1; + return nevents; } static char *aeApiName(void) { From d318803f9439e9e4a129727b69c8d5103ae994fd Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 14 May 2012 11:06:34 -0700 Subject: [PATCH 0239/1752] Whitespace --- src/ae_evport.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ae_evport.c b/src/ae_evport.c index c4fc9a40b0e..5b6602d0276 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -65,7 +65,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { } eventLoop->apidata = state; - return 0; + return 0; } static void aeApiFree(aeEventLoop *eventLoop) { @@ -247,7 +247,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { } /* - * port_getn can return with errno == ETIME having returned some events (!). + * port_getn can return with errno == ETIME having returned some events (!). * So if we get ETIME, we check nevents, too. */ nevents = 1; From 4934f93dfb30c93a1636e3227584e791cd062bfb Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 15 May 2012 15:27:12 +0200 Subject: [PATCH 0240/1752] Jemalloc updated to 3.0.0. Full changelog here: http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git;a=blob_plain;f=ChangeLog;hb=master Notable improvements from the point of view of Redis: 1) Bugfixing. 2) Support for Valgrind. 3) Support for OSX Lion, FreeBSD. --- deps/jemalloc.orig/.gitignore | 23 + deps/jemalloc.orig/COPYING | 51 + deps/jemalloc.orig/ChangeLog | 250 + deps/jemalloc.orig/INSTALL | 257 + deps/jemalloc.orig/Makefile.in | 259 + deps/jemalloc.orig/README | 16 + deps/jemalloc.orig/autogen.sh | 17 + deps/jemalloc.orig/bin/pprof | 4893 +++++++++++++++++ deps/jemalloc.orig/config.guess | 1456 +++++ deps/jemalloc.orig/config.stamp.in | 0 deps/jemalloc.orig/config.sub | 1549 ++++++ deps/jemalloc.orig/configure.ac | 938 ++++ deps/jemalloc.orig/doc/html.xsl.in | 4 + deps/jemalloc.orig/doc/jemalloc.xml.in | 2280 ++++++++ deps/jemalloc.orig/doc/manpages.xsl.in | 4 + deps/jemalloc.orig/doc/stylesheet.xsl | 7 + .../include/jemalloc/internal/arena.h | 743 +++ .../include/jemalloc/internal/atomic.h | 169 + .../include/jemalloc/internal/base.h | 24 + .../include/jemalloc/internal/bitmap.h | 184 + .../include/jemalloc/internal/chunk.h | 65 + .../include/jemalloc/internal/chunk_dss.h | 30 + .../include/jemalloc/internal/chunk_mmap.h | 23 + .../include/jemalloc/internal/chunk_swap.h | 0 .../include/jemalloc/internal/ckh.h | 95 + .../include/jemalloc/internal/ctl.h | 118 + .../include/jemalloc/internal/extent.h | 49 + .../include/jemalloc/internal/hash.h | 70 + .../include/jemalloc/internal/huge.h | 41 + .../jemalloc/internal/jemalloc_internal.h.in | 788 +++ .../include/jemalloc/internal/mb.h | 108 + .../include/jemalloc/internal/mutex.h | 86 + .../jemalloc/internal/private_namespace.h | 195 + .../include/jemalloc/internal/prn.h | 0 .../include/jemalloc/internal/prof.h | 547 ++ .../include/jemalloc/internal/ql.h | 83 + .../include/jemalloc/internal/qr.h | 67 + .../include/jemalloc/internal/rb.h | 973 ++++ .../include/jemalloc/internal/rtree.h | 161 + .../include/jemalloc/internal/stats.h | 207 + .../include/jemalloc/internal/tcache.h | 431 ++ .../include/jemalloc/internal/zone.h | 0 .../include/jemalloc/jemalloc.h.in | 66 + .../include/jemalloc/jemalloc_defs.h.in | 167 + deps/jemalloc.orig/install-sh | 250 + deps/jemalloc.orig/src/arena.c | 2704 +++++++++ deps/jemalloc.orig/src/atomic.c | 2 + deps/jemalloc.orig/src/base.c | 106 + deps/jemalloc.orig/src/bitmap.c | 90 + deps/jemalloc.orig/src/chunk.c | 173 + deps/jemalloc.orig/src/chunk_dss.c | 284 + deps/jemalloc.orig/src/chunk_mmap.c | 239 + .../src/chunk_swap.c | 0 deps/jemalloc.orig/src/ckh.c | 619 +++ deps/jemalloc.orig/src/ctl.c | 1670 ++++++ deps/jemalloc.orig/src/extent.c | 41 + deps/jemalloc.orig/src/hash.c | 2 + deps/jemalloc.orig/src/huge.c | 386 ++ deps/jemalloc.orig/src/jemalloc.c | 1881 +++++++ deps/jemalloc.orig/src/mb.c | 2 + deps/jemalloc.orig/src/mutex.c | 90 + deps/jemalloc.orig/src/prof.c | 1244 +++++ deps/jemalloc.orig/src/rtree.c | 46 + deps/jemalloc.orig/src/stats.c | 790 +++ deps/jemalloc.orig/src/tcache.c | 480 ++ deps/jemalloc.orig/src/zone.c | 354 ++ deps/jemalloc.orig/test/allocated.c | 142 + deps/jemalloc.orig/test/allocated.exp | 2 + deps/jemalloc.orig/test/allocm.c | 133 + deps/jemalloc.orig/test/allocm.exp | 25 + deps/jemalloc.orig/test/bitmap.c | 157 + deps/jemalloc.orig/test/bitmap.exp | 2 + deps/jemalloc.orig/test/mremap.c | 67 + deps/jemalloc.orig/test/mremap.exp | 2 + deps/jemalloc.orig/test/posix_memalign.c | 121 + deps/jemalloc.orig/test/posix_memalign.exp | 25 + deps/jemalloc.orig/test/rallocm.c | 127 + deps/jemalloc.orig/test/rallocm.exp | 2 + deps/jemalloc.orig/test/thread_arena.c | 92 + deps/jemalloc.orig/test/thread_arena.exp | 2 + deps/jemalloc/.gitignore | 2 + deps/jemalloc/COPYING | 32 +- deps/jemalloc/ChangeLog | 89 + deps/jemalloc/INSTALL | 93 +- deps/jemalloc/Makefile.in | 312 +- deps/jemalloc/README | 12 +- deps/jemalloc/VERSION | 2 +- deps/jemalloc/bin/jemalloc.sh.in | 9 + deps/jemalloc/bin/pprof | 841 ++- deps/jemalloc/config.guess | 1018 ++-- deps/jemalloc/config.sub | 432 +- deps/jemalloc/configure | 2544 ++++++--- deps/jemalloc/configure.ac | 720 ++- deps/jemalloc/doc/jemalloc.3 | 482 +- deps/jemalloc/doc/jemalloc.html | 599 +- deps/jemalloc/doc/jemalloc.xml.in | 720 +-- .../include/jemalloc/internal/arena.h | 744 ++- .../include/jemalloc/internal/atomic.h | 177 +- .../jemalloc/include/jemalloc/internal/base.h | 6 +- .../include/jemalloc/internal/chunk.h | 10 +- .../include/jemalloc/internal/chunk_dss.h | 14 +- .../include/jemalloc/internal/chunk_mmap.h | 7 +- deps/jemalloc/include/jemalloc/internal/ckh.h | 7 +- deps/jemalloc/include/jemalloc/internal/ctl.h | 58 +- .../include/jemalloc/internal/extent.h | 6 - .../jemalloc/include/jemalloc/internal/hash.h | 18 +- .../jemalloc/include/jemalloc/internal/huge.h | 7 +- .../jemalloc/internal/jemalloc_internal.h.in | 781 +-- deps/jemalloc/include/jemalloc/internal/mb.h | 9 +- .../include/jemalloc/internal/mutex.h | 73 +- .../jemalloc/internal/private_namespace.h | 184 +- .../jemalloc/include/jemalloc/internal/prng.h | 60 + .../jemalloc/include/jemalloc/internal/prof.h | 177 +- .../include/jemalloc/internal/quarantine.h | 24 + deps/jemalloc/include/jemalloc/internal/rb.h | 92 +- .../include/jemalloc/internal/size_classes.sh | 122 + .../include/jemalloc/internal/stats.h | 34 - .../include/jemalloc/internal/tcache.h | 405 +- deps/jemalloc/include/jemalloc/internal/tsd.h | 397 ++ .../jemalloc/include/jemalloc/internal/util.h | 160 + deps/jemalloc/include/jemalloc/jemalloc.h.in | 136 +- .../include/jemalloc/jemalloc_defs.h.in | 158 +- deps/jemalloc/include/msvc_compat/inttypes.h | 313 ++ deps/jemalloc/include/msvc_compat/stdbool.h | 16 + deps/jemalloc/include/msvc_compat/stdint.h | 247 + deps/jemalloc/include/msvc_compat/strings.h | 23 + deps/jemalloc/src/arena.c | 2115 +++---- deps/jemalloc/src/base.c | 36 +- deps/jemalloc/src/chunk.c | 315 +- deps/jemalloc/src/chunk_dss.c | 260 +- deps/jemalloc/src/chunk_mmap.c | 277 +- deps/jemalloc/src/ckh.c | 46 +- deps/jemalloc/src/ctl.c | 1168 ++-- deps/jemalloc/src/extent.c | 2 - deps/jemalloc/src/huge.c | 200 +- deps/jemalloc/src/jemalloc.c | 1701 +++--- deps/jemalloc/src/mutex.c | 95 +- deps/jemalloc/src/prof.c | 657 +-- deps/jemalloc/src/quarantine.c | 210 + deps/jemalloc/src/stats.c | 485 +- deps/jemalloc/src/tcache.c | 394 +- deps/jemalloc/src/tsd.c | 107 + deps/jemalloc/src/util.c | 646 +++ deps/jemalloc/src/zone.c | 276 +- deps/jemalloc/test/aligned_alloc.c | 119 + deps/jemalloc/test/aligned_alloc.exp | 25 + deps/jemalloc/test/allocated.c | 88 +- deps/jemalloc/test/allocm.c | 157 +- deps/jemalloc/test/bitmap.c | 32 +- deps/jemalloc/test/jemalloc_test.h.in | 47 + deps/jemalloc/test/mremap.c | 31 +- deps/jemalloc/test/posix_memalign.c | 58 +- deps/jemalloc/test/rallocm.c | 84 +- deps/jemalloc/test/thread_arena.c | 56 +- deps/jemalloc/test/thread_tcache_enabled.c | 91 + deps/jemalloc/test/thread_tcache_enabled.exp | 2 + src/zmalloc.h | 3 +- 157 files changed, 42911 insertions(+), 9090 deletions(-) create mode 100644 deps/jemalloc.orig/.gitignore create mode 100644 deps/jemalloc.orig/COPYING create mode 100644 deps/jemalloc.orig/ChangeLog create mode 100644 deps/jemalloc.orig/INSTALL create mode 100644 deps/jemalloc.orig/Makefile.in create mode 100644 deps/jemalloc.orig/README create mode 100755 deps/jemalloc.orig/autogen.sh create mode 100755 deps/jemalloc.orig/bin/pprof create mode 100755 deps/jemalloc.orig/config.guess create mode 100644 deps/jemalloc.orig/config.stamp.in create mode 100755 deps/jemalloc.orig/config.sub create mode 100644 deps/jemalloc.orig/configure.ac create mode 100644 deps/jemalloc.orig/doc/html.xsl.in create mode 100644 deps/jemalloc.orig/doc/jemalloc.xml.in create mode 100644 deps/jemalloc.orig/doc/manpages.xsl.in create mode 100644 deps/jemalloc.orig/doc/stylesheet.xsl create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/arena.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/atomic.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/base.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/bitmap.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h rename deps/{jemalloc => jemalloc.orig}/include/jemalloc/internal/chunk_swap.h (100%) create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/ckh.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/ctl.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/extent.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/hash.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/huge.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/mb.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/mutex.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h rename deps/{jemalloc => jemalloc.orig}/include/jemalloc/internal/prn.h (100%) create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/prof.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/ql.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/qr.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/rb.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/rtree.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/stats.h create mode 100644 deps/jemalloc.orig/include/jemalloc/internal/tcache.h rename deps/{jemalloc => jemalloc.orig}/include/jemalloc/internal/zone.h (100%) create mode 100644 deps/jemalloc.orig/include/jemalloc/jemalloc.h.in create mode 100644 deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in create mode 100755 deps/jemalloc.orig/install-sh create mode 100644 deps/jemalloc.orig/src/arena.c create mode 100644 deps/jemalloc.orig/src/atomic.c create mode 100644 deps/jemalloc.orig/src/base.c create mode 100644 deps/jemalloc.orig/src/bitmap.c create mode 100644 deps/jemalloc.orig/src/chunk.c create mode 100644 deps/jemalloc.orig/src/chunk_dss.c create mode 100644 deps/jemalloc.orig/src/chunk_mmap.c rename deps/{jemalloc => jemalloc.orig}/src/chunk_swap.c (100%) create mode 100644 deps/jemalloc.orig/src/ckh.c create mode 100644 deps/jemalloc.orig/src/ctl.c create mode 100644 deps/jemalloc.orig/src/extent.c create mode 100644 deps/jemalloc.orig/src/hash.c create mode 100644 deps/jemalloc.orig/src/huge.c create mode 100644 deps/jemalloc.orig/src/jemalloc.c create mode 100644 deps/jemalloc.orig/src/mb.c create mode 100644 deps/jemalloc.orig/src/mutex.c create mode 100644 deps/jemalloc.orig/src/prof.c create mode 100644 deps/jemalloc.orig/src/rtree.c create mode 100644 deps/jemalloc.orig/src/stats.c create mode 100644 deps/jemalloc.orig/src/tcache.c create mode 100644 deps/jemalloc.orig/src/zone.c create mode 100644 deps/jemalloc.orig/test/allocated.c create mode 100644 deps/jemalloc.orig/test/allocated.exp create mode 100644 deps/jemalloc.orig/test/allocm.c create mode 100644 deps/jemalloc.orig/test/allocm.exp create mode 100644 deps/jemalloc.orig/test/bitmap.c create mode 100644 deps/jemalloc.orig/test/bitmap.exp create mode 100644 deps/jemalloc.orig/test/mremap.c create mode 100644 deps/jemalloc.orig/test/mremap.exp create mode 100644 deps/jemalloc.orig/test/posix_memalign.c create mode 100644 deps/jemalloc.orig/test/posix_memalign.exp create mode 100644 deps/jemalloc.orig/test/rallocm.c create mode 100644 deps/jemalloc.orig/test/rallocm.exp create mode 100644 deps/jemalloc.orig/test/thread_arena.c create mode 100644 deps/jemalloc.orig/test/thread_arena.exp create mode 100644 deps/jemalloc/bin/jemalloc.sh.in create mode 100644 deps/jemalloc/include/jemalloc/internal/prng.h create mode 100644 deps/jemalloc/include/jemalloc/internal/quarantine.h create mode 100755 deps/jemalloc/include/jemalloc/internal/size_classes.sh create mode 100644 deps/jemalloc/include/jemalloc/internal/tsd.h create mode 100644 deps/jemalloc/include/jemalloc/internal/util.h create mode 100644 deps/jemalloc/include/msvc_compat/inttypes.h create mode 100644 deps/jemalloc/include/msvc_compat/stdbool.h create mode 100644 deps/jemalloc/include/msvc_compat/stdint.h create mode 100644 deps/jemalloc/include/msvc_compat/strings.h create mode 100644 deps/jemalloc/src/quarantine.c create mode 100644 deps/jemalloc/src/tsd.c create mode 100644 deps/jemalloc/src/util.c create mode 100644 deps/jemalloc/test/aligned_alloc.c create mode 100644 deps/jemalloc/test/aligned_alloc.exp create mode 100644 deps/jemalloc/test/thread_tcache_enabled.c create mode 100644 deps/jemalloc/test/thread_tcache_enabled.exp diff --git a/deps/jemalloc.orig/.gitignore b/deps/jemalloc.orig/.gitignore new file mode 100644 index 00000000000..32b4c424bde --- /dev/null +++ b/deps/jemalloc.orig/.gitignore @@ -0,0 +1,23 @@ +/autom4te.cache/ +/config.stamp +/config.log +/config.status +/configure +/doc/html.xsl +/doc/manpages.xsl +/doc/jemalloc.xml +/doc/jemalloc.html +/doc/jemalloc.3 +/lib/ +/Makefile +/include/jemalloc/internal/jemalloc_internal\.h +/include/jemalloc/jemalloc\.h +/include/jemalloc/jemalloc_defs\.h +/test/jemalloc_test\.h +/src/*.[od] +/test/*.[od] +/test/*.out +/test/[a-z]* +!test/*.c +!test/*.exp +/VERSION diff --git a/deps/jemalloc.orig/COPYING b/deps/jemalloc.orig/COPYING new file mode 100644 index 00000000000..10ade120049 --- /dev/null +++ b/deps/jemalloc.orig/COPYING @@ -0,0 +1,51 @@ +Unless otherwise specified, files in the jemalloc source distribution are +subject to the following licenses: +-------------------------------------------------------------------------------- +Copyright (C) 2002-2010 Jason Evans . +All rights reserved. +Copyright (C) 2007-2010 Mozilla Foundation. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: +1. Redistributions of source code must retain the above copyright notice(s), + this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice(s), + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- +Copyright (C) 2009-2010 Facebook, Inc. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. +* Neither the name of Facebook, Inc. nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-------------------------------------------------------------------------------- diff --git a/deps/jemalloc.orig/ChangeLog b/deps/jemalloc.orig/ChangeLog new file mode 100644 index 00000000000..326ee7a97e1 --- /dev/null +++ b/deps/jemalloc.orig/ChangeLog @@ -0,0 +1,250 @@ +Following are change highlights associated with official releases. Important +bug fixes are all mentioned, but internal enhancements are omitted here for +brevity (even though they are more fun to write about). Much more detail can be +found in the git revision history: + + http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git + git://canonware.com/jemalloc.git + +* 2.2.5 (November 14, 2011) + + Bug fixes: + - Fix huge_ralloc() race when using mremap(2). This is a serious bug that + could cause memory corruption and/or crashes. + - Fix huge_ralloc() to maintain chunk statistics. + - Fix malloc_stats_print(..., "a") output. + +* 2.2.4 (November 5, 2011) + + Bug fixes: + - Initialize arenas_tsd before using it. This bug existed for 2.2.[0-3], as + well as for --disable-tls builds in earlier releases. + - Do not assume a 4 KiB page size in test/rallocm.c. + +* 2.2.3 (August 31, 2011) + + This version fixes numerous bugs related to heap profiling. + + Bug fixes: + - Fix a prof-related race condition. This bug could cause memory corruption, + but only occurred in non-default configurations (prof_accum:false). + - Fix off-by-one backtracing issues (make sure that prof_alloc_prep() is + excluded from backtraces). + - Fix a prof-related bug in realloc() (only triggered by OOM errors). + - Fix prof-related bugs in allocm() and rallocm(). + - Fix prof_tdata_cleanup() for --disable-tls builds. + - Fix a relative include path, to fix objdir builds. + +* 2.2.2 (July 30, 2011) + + Bug fixes: + - Fix a build error for --disable-tcache. + - Fix assertions in arena_purge() (for real this time). + - Add the --with-private-namespace option. This is a workaround for symbol + conflicts that can inadvertently arise when using static libraries. + +* 2.2.1 (March 30, 2011) + + Bug fixes: + - Implement atomic operations for x86/x64. This fixes compilation failures + for versions of gcc that are still in wide use. + - Fix an assertion in arena_purge(). + +* 2.2.0 (March 22, 2011) + + This version incorporates several improvements to algorithms and data + structures that tend to reduce fragmentation and increase speed. + + New features: + - Add the "stats.cactive" mallctl. + - Update pprof (from google-perftools 1.7). + - Improve backtracing-related configuration logic, and add the + --disable-prof-libgcc option. + + Bug fixes: + - Change default symbol visibility from "internal", to "hidden", which + decreases the overhead of library-internal function calls. + - Fix symbol visibility so that it is also set on OS X. + - Fix a build dependency regression caused by the introduction of the .pic.o + suffix for PIC object files. + - Add missing checks for mutex initialization failures. + - Don't use libgcc-based backtracing except on x64, where it is known to work. + - Fix deadlocks on OS X that were due to memory allocation in + pthread_mutex_lock(). + - Heap profiling-specific fixes: + + Fix memory corruption due to integer overflow in small region index + computation, when using a small enough sample interval that profiling + context pointers are stored in small run headers. + + Fix a bootstrap ordering bug that only occurred with TLS disabled. + + Fix a rallocm() rsize bug. + + Fix error detection bugs for aligned memory allocation. + +* 2.1.3 (March 14, 2011) + + Bug fixes: + - Fix a cpp logic regression (due to the "thread.{de,}allocatedp" mallctl fix + for OS X in 2.1.2). + - Fix a "thread.arena" mallctl bug. + - Fix a thread cache stats merging bug. + +* 2.1.2 (March 2, 2011) + + Bug fixes: + - Fix "thread.{de,}allocatedp" mallctl for OS X. + - Add missing jemalloc.a to build system. + +* 2.1.1 (January 31, 2011) + + Bug fixes: + - Fix aligned huge reallocation (affected allocm()). + - Fix the ALLOCM_LG_ALIGN macro definition. + - Fix a heap dumping deadlock. + - Fix a "thread.arena" mallctl bug. + +* 2.1.0 (December 3, 2010) + + This version incorporates some optimizations that can't quite be considered + bug fixes. + + New features: + - Use Linux's mremap(2) for huge object reallocation when possible. + - Avoid locking in mallctl*() when possible. + - Add the "thread.[de]allocatedp" mallctl's. + - Convert the manual page source from roff to DocBook, and generate both roff + and HTML manuals. + + Bug fixes: + - Fix a crash due to incorrect bootstrap ordering. This only impacted + --enable-debug --enable-dss configurations. + - Fix a minor statistics bug for mallctl("swap.avail", ...). + +* 2.0.1 (October 29, 2010) + + Bug fixes: + - Fix a race condition in heap profiling that could cause undefined behavior + if "opt.prof_accum" were disabled. + - Add missing mutex unlocks for some OOM error paths in the heap profiling + code. + - Fix a compilation error for non-C99 builds. + +* 2.0.0 (October 24, 2010) + + This version focuses on the experimental *allocm() API, and on improved + run-time configuration/introspection. Nonetheless, numerous performance + improvements are also included. + + New features: + - Implement the experimental {,r,s,d}allocm() API, which provides a superset + of the functionality available via malloc(), calloc(), posix_memalign(), + realloc(), malloc_usable_size(), and free(). These functions can be used to + allocate/reallocate aligned zeroed memory, ask for optional extra memory + during reallocation, prevent object movement during reallocation, etc. + - Replace JEMALLOC_OPTIONS/JEMALLOC_PROF_PREFIX with MALLOC_CONF, which is + more human-readable, and more flexible. For example: + JEMALLOC_OPTIONS=AJP + is now: + MALLOC_CONF=abort:true,fill:true,stats_print:true + - Port to Apple OS X. Sponsored by Mozilla. + - Make it possible for the application to control thread-->arena mappings via + the "thread.arena" mallctl. + - Add compile-time support for all TLS-related functionality via pthreads TSD. + This is mainly of interest for OS X, which does not support TLS, but has a + TSD implementation with similar performance. + - Override memalign() and valloc() if they are provided by the system. + - Add the "arenas.purge" mallctl, which can be used to synchronously purge all + dirty unused pages. + - Make cumulative heap profiling data optional, so that it is possible to + limit the amount of memory consumed by heap profiling data structures. + - Add per thread allocation counters that can be accessed via the + "thread.allocated" and "thread.deallocated" mallctls. + + Incompatible changes: + - Remove JEMALLOC_OPTIONS and malloc_options (see MALLOC_CONF above). + - Increase default backtrace depth from 4 to 128 for heap profiling. + - Disable interval-based profile dumps by default. + + Bug fixes: + - Remove bad assertions in fork handler functions. These assertions could + cause aborts for some combinations of configure settings. + - Fix strerror_r() usage to deal with non-standard semantics in GNU libc. + - Fix leak context reporting. This bug tended to cause the number of contexts + to be underreported (though the reported number of objects and bytes were + correct). + - Fix a realloc() bug for large in-place growing reallocation. This bug could + cause memory corruption, but it was hard to trigger. + - Fix an allocation bug for small allocations that could be triggered if + multiple threads raced to create a new run of backing pages. + - Enhance the heap profiler to trigger samples based on usable size, rather + than request size. + - Fix a heap profiling bug due to sometimes losing track of requested object + size for sampled objects. + +* 1.0.3 (August 12, 2010) + + Bug fixes: + - Fix the libunwind-based implementation of stack backtracing (used for heap + profiling). This bug could cause zero-length backtraces to be reported. + - Add a missing mutex unlock in library initialization code. If multiple + threads raced to initialize malloc, some of them could end up permanently + blocked. + +* 1.0.2 (May 11, 2010) + + Bug fixes: + - Fix junk filling of large objects, which could cause memory corruption. + - Add MAP_NORESERVE support for chunk mapping, because otherwise virtual + memory limits could cause swap file configuration to fail. Contributed by + Jordan DeLong. + +* 1.0.1 (April 14, 2010) + + Bug fixes: + - Fix compilation when --enable-fill is specified. + - Fix threads-related profiling bugs that affected accuracy and caused memory + to be leaked during thread exit. + - Fix dirty page purging race conditions that could cause crashes. + - Fix crash in tcache flushing code during thread destruction. + +* 1.0.0 (April 11, 2010) + + This release focuses on speed and run-time introspection. Numerous + algorithmic improvements make this release substantially faster than its + predecessors. + + New features: + - Implement autoconf-based configuration system. + - Add mallctl*(), for the purposes of introspection and run-time + configuration. + - Make it possible for the application to manually flush a thread's cache, via + the "tcache.flush" mallctl. + - Base maximum dirty page count on proportion of active memory. + - Compute various addtional run-time statistics, including per size class + statistics for large objects. + - Expose malloc_stats_print(), which can be called repeatedly by the + application. + - Simplify the malloc_message() signature to only take one string argument, + and incorporate an opaque data pointer argument for use by the application + in combination with malloc_stats_print(). + - Add support for allocation backed by one or more swap files, and allow the + application to disable over-commit if swap files are in use. + - Implement allocation profiling and leak checking. + + Removed features: + - Remove the dynamic arena rebalancing code, since thread-specific caching + reduces its utility. + + Bug fixes: + - Modify chunk allocation to work when address space layout randomization + (ASLR) is in use. + - Fix thread cleanup bugs related to TLS destruction. + - Handle 0-size allocation requests in posix_memalign(). + - Fix a chunk leak. The leaked chunks were never touched, so this impacted + virtual memory usage, but not physical memory usage. + +* linux_2008082[78]a (August 27/28, 2008) + + These snapshot releases are the simple result of incorporating Linux-specific + support into the FreeBSD malloc sources. + +-------------------------------------------------------------------------------- +vim:filetype=text:textwidth=80 diff --git a/deps/jemalloc.orig/INSTALL b/deps/jemalloc.orig/INSTALL new file mode 100644 index 00000000000..2a1e469cb8b --- /dev/null +++ b/deps/jemalloc.orig/INSTALL @@ -0,0 +1,257 @@ +Building and installing jemalloc can be as simple as typing the following while +in the root directory of the source tree: + + ./configure + make + make install + +=== Advanced configuration ===================================================== + +The 'configure' script supports numerous options that allow control of which +functionality is enabled, where jemalloc is installed, etc. Optionally, pass +any of the following arguments (not a definitive list) to 'configure': + +--help + Print a definitive list of options. + +--prefix= + Set the base directory in which to install. For example: + + ./configure --prefix=/usr/local + + will cause files to be installed into /usr/local/include, /usr/local/lib, + and /usr/local/man. + +--with-rpath= + Embed one or more library paths, so that libjemalloc can find the libraries + it is linked to. This works only on ELF-based systems. + +--with-jemalloc-prefix= + Prefix all public APIs with . For example, if is + "prefix_", API changes like the following occur: + + malloc() --> prefix_malloc() + malloc_conf --> prefix_malloc_conf + /etc/malloc.conf --> /etc/prefix_malloc.conf + MALLOC_CONF --> PREFIX_MALLOC_CONF + + This makes it possible to use jemalloc at the same time as the system + allocator, or even to use multiple copies of jemalloc simultaneously. + + By default, the prefix is "", except on OS X, where it is "je_". On OS X, + jemalloc overlays the default malloc zone, but makes no attempt to actually + replace the "malloc", "calloc", etc. symbols. + +--with-private-namespace= + Prefix all library-private APIs with . For shared libraries, + symbol visibility mechanisms prevent these symbols from being exported, but + for static libraries, naming collisions are a real possibility. By + default, the prefix is "" (empty string). + +--with-install-suffix= + Append to the base name of all installed files, such that multiple + versions of jemalloc can coexist in the same installation directory. For + example, libjemalloc.so.0 becomes libjemalloc.so.0. + +--enable-cc-silence + Enable code that silences non-useful compiler warnings. This is helpful + when trying to tell serious warnings from those due to compiler + limitations, but it potentially incurs a performance penalty. + +--enable-debug + Enable assertions and validation code. This incurs a substantial + performance hit, but is very useful during application development. + +--enable-stats + Enable statistics gathering functionality. See the "opt.stats_print" + option documentation for usage details. + +--enable-prof + Enable heap profiling and leak detection functionality. See the "opt.prof" + option documentation for usage details. When enabled, there are several + approaches to backtracing, and the configure script chooses the first one + in the following list that appears to function correctly: + + + libunwind (requires --enable-prof-libunwind) + + libgcc (unless --disable-prof-libgcc) + + gcc intrinsics (unless --disable-prof-gcc) + +--enable-prof-libunwind + Use the libunwind library (http://www.nongnu.org/libunwind/) for stack + backtracing. + +--disable-prof-libgcc + Disable the use of libgcc's backtracing functionality. + +--disable-prof-gcc + Disable the use of gcc intrinsics for backtracing. + +--with-static-libunwind= + Statically link against the specified libunwind.a rather than dynamically + linking with -lunwind. + +--disable-tiny + Disable tiny (sub-quantum-sized) object support. Technically it is not + legal for a malloc implementation to allocate objects with less than + quantum alignment (8 or 16 bytes, depending on architecture), but in + practice it never causes any problems if, for example, 4-byte allocations + are 4-byte-aligned. + +--disable-tcache + Disable thread-specific caches for small objects. Objects are cached and + released in bulk, thus reducing the total number of mutex operations. See + the "opt.tcache" option for usage details. + +--enable-swap + Enable mmap()ed swap file support. When this feature is built in, it is + possible to specify one or more files that act as backing store. This + effectively allows for per application swap files. + +--enable-dss + Enable support for page allocation/deallocation via sbrk(2), in addition to + mmap(2). + +--enable-fill + Enable support for junk/zero filling of memory. See the "opt.junk"/ + "opt.zero" option documentation for usage details. + +--enable-xmalloc + Enable support for optional immediate termination due to out-of-memory + errors, as is commonly implemented by "xmalloc" wrapper function for malloc. + See the "opt.xmalloc" option documentation for usage details. + +--enable-sysv + Enable support for System V semantics, wherein malloc(0) returns NULL + rather than a minimal allocation. See the "opt.sysv" option documentation + for usage details. + +--enable-dynamic-page-shift + Under most conditions, the system page size never changes (usually 4KiB or + 8KiB, depending on architecture and configuration), and unless this option + is enabled, jemalloc assumes that page size can safely be determined during + configuration and hard-coded. Enabling dynamic page size determination has + a measurable impact on performance, since the compiler is forced to load + the page size from memory rather than embedding immediate values. + +--disable-lazy-lock + Disable code that wraps pthread_create() to detect when an application + switches from single-threaded to multi-threaded mode, so that it can avoid + mutex locking/unlocking operations while in single-threaded mode. In + practice, this feature usually has little impact on performance unless + thread-specific caching is disabled. + +--disable-tls + Disable thread-local storage (TLS), which allows for fast access to + thread-local variables via the __thread keyword. If TLS is available, + jemalloc uses it for several purposes. + +--with-xslroot= + Specify where to find DocBook XSL stylesheets when building the + documentation. + +The following environment variables (not a definitive list) impact configure's +behavior: + +CFLAGS="?" + Pass these flags to the compiler. You probably shouldn't define this unless + you know what you are doing. (Use EXTRA_CFLAGS instead.) + +EXTRA_CFLAGS="?" + Append these flags to CFLAGS. This makes it possible to add flags such as + -Werror, while allowing the configure script to determine what other flags + are appropriate for the specified configuration. + + The configure script specifically checks whether an optimization flag (-O*) + is specified in EXTRA_CFLAGS, and refrains from specifying an optimization + level if it finds that one has already been specified. + +CPPFLAGS="?" + Pass these flags to the C preprocessor. Note that CFLAGS is not passed to + 'cpp' when 'configure' is looking for include files, so you must use + CPPFLAGS instead if you need to help 'configure' find header files. + +LD_LIBRARY_PATH="?" + 'ld' uses this colon-separated list to find libraries. + +LDFLAGS="?" + Pass these flags when linking. + +PATH="?" + 'configure' uses this to find programs. + +=== Advanced compilation ======================================================= + +To install only parts of jemalloc, use the following targets: + + install_bin + install_include + install_lib + install_doc + +To clean up build results to varying degrees, use the following make targets: + + clean + distclean + relclean + +=== Advanced installation ====================================================== + +Optionally, define make variables when invoking make, including (not +exclusively): + +INCLUDEDIR="?" + Use this as the installation prefix for header files. + +LIBDIR="?" + Use this as the installation prefix for libraries. + +MANDIR="?" + Use this as the installation prefix for man pages. + +DESTDIR="?" + Prepend DESTDIR to INCLUDEDIR, LIBDIR, DATADIR, and MANDIR. This is useful + when installing to a different path than was specified via --prefix. + +CC="?" + Use this to invoke the C compiler. + +CFLAGS="?" + Pass these flags to the compiler. + +CPPFLAGS="?" + Pass these flags to the C preprocessor. + +LDFLAGS="?" + Pass these flags when linking. + +PATH="?" + Use this to search for programs used during configuration and building. + +=== Development ================================================================ + +If you intend to make non-trivial changes to jemalloc, use the 'autogen.sh' +script rather than 'configure'. This re-generates 'configure', enables +configuration dependency rules, and enables re-generation of automatically +generated source files. + +The build system supports using an object directory separate from the source +tree. For example, you can create an 'obj' directory, and from within that +directory, issue configuration and build commands: + + autoconf + mkdir obj + cd obj + ../configure --enable-autogen + make + +=== Documentation ============================================================== + +The manual page is generated in both html and roff formats. Any web browser +can be used to view the html manual. The roff manual page can be formatted +prior to installation via any of the following commands: + + nroff -man -t doc/jemalloc.3 + + groff -man -t -Tps doc/jemalloc.3 | ps2pdf - doc/jemalloc.3.pdf + + (cd doc; groff -man -man-ext -t -Thtml jemalloc.3 > jemalloc.3.html) diff --git a/deps/jemalloc.orig/Makefile.in b/deps/jemalloc.orig/Makefile.in new file mode 100644 index 00000000000..de7492f951a --- /dev/null +++ b/deps/jemalloc.orig/Makefile.in @@ -0,0 +1,259 @@ +# Clear out all vpaths, then set just one (default vpath) for the main build +# directory. +vpath +vpath % . + +# Clear the default suffixes, so that built-in rules are not used. +.SUFFIXES : + +SHELL := /bin/sh + +CC := @CC@ + +# Configuration parameters. +DESTDIR = +BINDIR := $(DESTDIR)@BINDIR@ +INCLUDEDIR := $(DESTDIR)@INCLUDEDIR@ +LIBDIR := $(DESTDIR)@LIBDIR@ +DATADIR := $(DESTDIR)@DATADIR@ +MANDIR := $(DESTDIR)@MANDIR@ + +# Build parameters. +CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include +CFLAGS := @CFLAGS@ +ifeq (macho, @abi@) +CFLAGS += -dynamic +endif +LDFLAGS := @LDFLAGS@ +LIBS := @LIBS@ +RPATH_EXTRA := @RPATH_EXTRA@ +ifeq (macho, @abi@) +SO := dylib +WL_SONAME := dylib_install_name +else +SO := so +WL_SONAME := soname +endif +REV := 1 +ifeq (macho, @abi@) +TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH=@objroot@lib +else +TEST_LIBRARY_PATH := +endif + +# Lists of files. +BINS := @srcroot@bin/pprof +CHDRS := @objroot@include/jemalloc/jemalloc@install_suffix@.h \ + @objroot@include/jemalloc/jemalloc_defs@install_suffix@.h +CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \ + @srcroot@src/base.c @srcroot@src/bitmap.c @srcroot@src/chunk.c \ + @srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \ + @srcroot@src/chunk_swap.c @srcroot@src/ckh.c @srcroot@src/ctl.c \ + @srcroot@src/extent.c @srcroot@src/hash.c @srcroot@src/huge.c \ + @srcroot@src/mb.c @srcroot@src/mutex.c @srcroot@src/prof.c \ + @srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c +ifeq (macho, @abi@) +CSRCS += @srcroot@src/zone.c +endif +STATIC_LIBS := @objroot@lib/libjemalloc@install_suffix@.a +DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \ + @objroot@lib/libjemalloc@install_suffix@.$(SO) \ + @objroot@lib/libjemalloc@install_suffix@_pic.a +MAN3 := @objroot@doc/jemalloc@install_suffix@.3 +DOCS_XML := @objroot@doc/jemalloc@install_suffix@.xml +DOCS_HTML := $(DOCS_XML:@objroot@%.xml=@srcroot@%.html) +DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3) +DOCS := $(DOCS_HTML) $(DOCS_MAN3) +CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \ + @srcroot@test/bitmap.c @srcroot@test/mremap.c \ + @srcroot@test/posix_memalign.c @srcroot@test/rallocm.c \ + @srcroot@test/thread_arena.c + +.PHONY: all dist doc_html doc_man doc +.PHONY: install_bin install_include install_lib +.PHONY: install_html install_man install_doc install +.PHONY: tests check clean distclean relclean + +.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o) + +# Default target. +all: $(DSOS) $(STATIC_LIBS) + +dist: doc + +@srcroot@doc/%.html : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/html.xsl + @XSLTPROC@ -o $@ @objroot@doc/html.xsl $< + +@srcroot@doc/%.3 : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/manpages.xsl + @XSLTPROC@ -o $@ @objroot@doc/manpages.xsl $< + +doc_html: $(DOCS_HTML) +doc_man: $(DOCS_MAN3) +doc: $(DOCS) + +# +# Include generated dependency files. +# +-include $(CSRCS:@srcroot@%.c=@objroot@%.d) +-include $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) +-include $(CTESTS:@srcroot@%.c=@objroot@%.d) + +@objroot@src/%.o: @srcroot@src/%.c + @mkdir -p $(@D) + $(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $< + @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)" + +@objroot@src/%.pic.o: @srcroot@src/%.c + @mkdir -p $(@D) + $(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $< + @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $(basename $@))))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.pic.o \2/g\" > $(@:%.o=%.d)" + +%.$(SO) : %.$(SO).$(REV) + @mkdir -p $(@D) + ln -sf $( $(@:%.o=%.d)" + +# Automatic dependency generation misses #include "*.c". +@objroot@test/bitmap.o : @objroot@src/bitmap.o + +@objroot@test/%: @objroot@test/%.o \ + @objroot@lib/libjemalloc@install_suffix@.$(SO) + @mkdir -p $(@D) +ifneq (@RPATH@, ) + $(CC) -o $@ $< @RPATH@@objroot@lib -L@objroot@lib -ljemalloc@install_suffix@ -lpthread +else + $(CC) -o $@ $< -L@objroot@lib -ljemalloc@install_suffix@ -lpthread +endif + +install_bin: + install -d $(BINDIR) + @for b in $(BINS); do \ + echo "install -m 755 $$b $(BINDIR)"; \ + install -m 755 $$b $(BINDIR); \ +done + +install_include: + install -d $(INCLUDEDIR)/jemalloc + @for h in $(CHDRS); do \ + echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \ + install -m 644 $$h $(INCLUDEDIR)/jemalloc; \ +done + +install_lib: $(DSOS) $(STATIC_LIBS) + install -d $(LIBDIR) + install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR) + ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO) + install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR) + install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR) + +install_html: + install -d $(DATADIR)/doc/jemalloc@install_suffix@ + @for d in $(DOCS_HTML); do \ + echo "install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@"; \ + install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@; \ +done + +install_man: + install -d $(MANDIR)/man3 + @for d in $(DOCS_MAN3); do \ + echo "install -m 644 $$d $(MANDIR)/man3"; \ + install -m 644 $$d $(MANDIR)/man3; \ +done + +install_doc: install_html install_man + +install: install_bin install_include install_lib install_doc + +tests: $(CTESTS:@srcroot@%.c=@objroot@%) + +check: tests + @mkdir -p @objroot@test + @$(SHELL) -c 'total=0; \ + failures=0; \ + echo "========================================="; \ + for t in $(CTESTS:@srcroot@%.c=@objroot@%); do \ + total=`expr $$total + 1`; \ + /bin/echo -n "$${t} ... "; \ + $(TEST_LIBRARY_PATH) $${t} @abs_srcroot@ @abs_objroot@ \ + > @objroot@$${t}.out 2>&1; \ + if test -e "@srcroot@$${t}.exp"; then \ + diff -u @srcroot@$${t}.exp \ + @objroot@$${t}.out >/dev/null 2>&1; \ + fail=$$?; \ + if test "$${fail}" -eq "1" ; then \ + failures=`expr $${failures} + 1`; \ + echo "*** FAIL ***"; \ + else \ + echo "pass"; \ + fi; \ + else \ + echo "*** FAIL *** (.exp file is missing)"; \ + failures=`expr $${failures} + 1`; \ + fi; \ + done; \ + echo "========================================="; \ + echo "Failures: $${failures}/$${total}"' + +clean: + rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o) + rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o) + rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d) + rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) + rm -f $(CTESTS:@srcroot@%.c=@objroot@%) + rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o) + rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d) + rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out) + rm -f $(DSOS) $(STATIC_LIBS) + +distclean: clean + rm -rf @objroot@autom4te.cache + rm -f @objroot@config.log + rm -f @objroot@config.status + rm -f @objroot@config.stamp + rm -f @cfghdrs_out@ + rm -f @cfgoutputs_out@ + +relclean: distclean + rm -f @objroot@configure + rm -f @srcroot@VERSION + rm -f $(DOCS_HTML) + rm -f $(DOCS_MAN3) + +#=============================================================================== +# Re-configuration rules. + +ifeq (@enable_autogen@, 1) +@srcroot@configure : @srcroot@configure.ac + cd ./@srcroot@ && @AUTOCONF@ + +@objroot@config.status : @srcroot@configure + ./@objroot@config.status --recheck + +@srcroot@config.stamp.in : @srcroot@configure.ac + echo stamp > @srcroot@config.stamp.in + +@objroot@config.stamp : @cfgoutputs_in@ @cfghdrs_in@ @srcroot@configure + ./@objroot@config.status + @touch $@ + +# There must be some action in order for make to re-read Makefile when it is +# out of date. +@cfgoutputs_out@ @cfghdrs_out@ : @objroot@config.stamp + @true +endif diff --git a/deps/jemalloc.orig/README b/deps/jemalloc.orig/README new file mode 100644 index 00000000000..4d7b552bf3d --- /dev/null +++ b/deps/jemalloc.orig/README @@ -0,0 +1,16 @@ +jemalloc is a general-purpose scalable concurrent malloc(3) implementation. +This distribution is a stand-alone "portable" implementation that currently +targets Linux and Apple OS X. jemalloc is included as the default allocator in +the FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox +web browser on Microsoft Windows-related platforms. Depending on your needs, +one of the other divergent versions may suit your needs better than this +distribution. + +The COPYING file contains copyright and licensing information. + +The INSTALL file contains information on how to configure, build, and install +jemalloc. + +The ChangeLog file contains a brief summary of changes for each release. + +URL: http://www.canonware.com/jemalloc/ diff --git a/deps/jemalloc.orig/autogen.sh b/deps/jemalloc.orig/autogen.sh new file mode 100755 index 00000000000..75f32da6873 --- /dev/null +++ b/deps/jemalloc.orig/autogen.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +for i in autoconf; do + echo "$i" + $i + if [ $? -ne 0 ]; then + echo "Error $? in $i" + exit 1 + fi +done + +echo "./configure --enable-autogen $@" +./configure --enable-autogen $@ +if [ $? -ne 0 ]; then + echo "Error $? in ./configure" + exit 1 +fi diff --git a/deps/jemalloc.orig/bin/pprof b/deps/jemalloc.orig/bin/pprof new file mode 100755 index 00000000000..280ddcc8329 --- /dev/null +++ b/deps/jemalloc.orig/bin/pprof @@ -0,0 +1,4893 @@ +#! /usr/bin/env perl + +# Copyright (c) 1998-2007, Google Inc. +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# --- +# Program for printing the profile generated by common/profiler.cc, +# or by the heap profiler (common/debugallocation.cc) +# +# The profile contains a sequence of entries of the form: +# +# This program parses the profile, and generates user-readable +# output. +# +# Examples: +# +# % tools/pprof "program" "profile" +# Enters "interactive" mode +# +# % tools/pprof --text "program" "profile" +# Generates one line per procedure +# +# % tools/pprof --gv "program" "profile" +# Generates annotated call-graph and displays via "gv" +# +# % tools/pprof --gv --focus=Mutex "program" "profile" +# Restrict to code paths that involve an entry that matches "Mutex" +# +# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" +# Restrict to code paths that involve an entry that matches "Mutex" +# and does not match "string" +# +# % tools/pprof --list=IBF_CheckDocid "program" "profile" +# Generates disassembly listing of all routines with at least one +# sample that match the --list= pattern. The listing is +# annotated with the flat and cumulative sample counts at each line. +# +# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" +# Generates disassembly listing of all routines with at least one +# sample that match the --disasm= pattern. The listing is +# annotated with the flat and cumulative sample counts at each PC value. +# +# TODO: Use color to indicate files? + +use strict; +use warnings; +use Getopt::Long; + +my $PPROF_VERSION = "1.7"; + +# These are the object tools we use which can come from a +# user-specified location using --tools, from the PPROF_TOOLS +# environment variable, or from the environment. +my %obj_tool_map = ( + "objdump" => "objdump", + "nm" => "nm", + "addr2line" => "addr2line", + "c++filt" => "c++filt", + ## ConfigureObjTools may add architecture-specific entries: + #"nm_pdb" => "nm-pdb", # for reading windows (PDB-format) executables + #"addr2line_pdb" => "addr2line-pdb", # ditto + #"otool" => "otool", # equivalent of objdump on OS X +); +my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local +my $GV = "gv"; +my $EVINCE = "evince"; # could also be xpdf or perhaps acroread +my $KCACHEGRIND = "kcachegrind"; +my $PS2PDF = "ps2pdf"; +# These are used for dynamic profiles +my $URL_FETCHER = "curl -s"; + +# These are the web pages that servers need to support for dynamic profiles +my $HEAP_PAGE = "/pprof/heap"; +my $PROFILE_PAGE = "/pprof/profile"; # must support cgi-param "?seconds=#" +my $PMUPROFILE_PAGE = "/pprof/pmuprofile(?:\\?.*)?"; # must support cgi-param + # ?seconds=#&event=x&period=n +my $GROWTH_PAGE = "/pprof/growth"; +my $CONTENTION_PAGE = "/pprof/contention"; +my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter +my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; +my $CENSUSPROFILE_PAGE = "/pprof/censusprofile"; # must support "?seconds=#" +my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST +my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; + +# These are the web pages that can be named on the command line. +# All the alternatives must begin with /. +my $PROFILES = "($HEAP_PAGE|$PROFILE_PAGE|$PMUPROFILE_PAGE|" . + "$GROWTH_PAGE|$CONTENTION_PAGE|$WALL_PAGE|" . + "$FILTEREDPROFILE_PAGE|$CENSUSPROFILE_PAGE)"; + +# default binary name +my $UNKNOWN_BINARY = "(unknown)"; + +# There is a pervasive dependency on the length (in hex characters, +# i.e., nibbles) of an address, distinguishing between 32-bit and +# 64-bit profiles. To err on the safe size, default to 64-bit here: +my $address_length = 16; + +# A list of paths to search for shared object files +my @prefix_list = (); + +# Special routine name that should not have any symbols. +# Used as separator to parse "addr2line -i" output. +my $sep_symbol = '_fini'; +my $sep_address = undef; + +##### Argument parsing ##### + +sub usage_string { + return < + is a space separated list of profile names. +pprof [options] + is a list of profile files where each file contains + the necessary symbol mappings as well as profile data (likely generated + with --raw). +pprof [options] + is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE + + Each name can be: + /path/to/profile - a path to a profile file + host:port[/] - a location of a service to get profile from + + The / can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, + $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, + $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. + For instance: "pprof http://myserver.com:80$HEAP_PAGE". + If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). +pprof --symbols + Maps addresses to symbol names. In this mode, stdin should be a + list of library mappings, in the same format as is found in the heap- + and cpu-profile files (this loosely matches that of /proc/self/maps + on linux), followed by a list of hex addresses to map, one per line. + + For more help with querying remote servers, including how to add the + necessary server-side support code, see this filename (or one like it): + + /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html + +Options: + --cum Sort by cumulative data + --base= Subtract from before display + --interactive Run in interactive mode (interactive "help" gives help) [default] + --seconds= Length of time for dynamic profiles [default=30 secs] + --add_lib= Read additional symbols and line info from the given library + --lib_prefix= Comma separated list of library path prefixes + +Reporting Granularity: + --addresses Report at address level + --lines Report at source line level + --functions Report at function level [default] + --files Report at source file level + +Output type: + --text Generate text report + --callgrind Generate callgrind format to stdout + --gv Generate Postscript and display + --evince Generate PDF and display + --web Generate SVG and display + --list= Generate source listing of matching routines + --disasm= Generate disassembly of matching routines + --symbols Print demangled symbol names found at given addresses + --dot Generate DOT file to stdout + --ps Generate Postcript to stdout + --pdf Generate PDF to stdout + --svg Generate SVG to stdout + --gif Generate GIF to stdout + --raw Generate symbolized pprof data (useful with remote fetch) + +Heap-Profile Options: + --inuse_space Display in-use (mega)bytes [default] + --inuse_objects Display in-use objects + --alloc_space Display allocated (mega)bytes + --alloc_objects Display allocated objects + --show_bytes Display space in bytes + --drop_negative Ignore negative differences + +Contention-profile options: + --total_delay Display total delay at each region [default] + --contentions Display number of delays at each region + --mean_delay Display mean delay at each region + +Call-graph Options: + --nodecount= Show at most so many nodes [default=80] + --nodefraction= Hide nodes below *total [default=.005] + --edgefraction= Hide edges below *total [default=.001] + --maxdegree= Max incoming/outgoing edges per node [default=8] + --focus= Focus on nodes matching + --ignore= Ignore nodes matching + --scale= Set GV scaling [default=0] + --heapcheck Make nodes with non-0 object counts + (i.e. direct leak generators) more visible + +Miscellaneous: + --tools=[,...] \$PATH for object tool pathnames + --test Run unit tests + --help This message + --version Version information + +Environment Variables: + PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof + PPROF_TOOLS Prefix for object tools pathnames + +Examples: + +pprof /bin/ls ls.prof + Enters "interactive" mode +pprof --text /bin/ls ls.prof + Outputs one line per procedure +pprof --web /bin/ls ls.prof + Displays annotated call-graph in web browser +pprof --gv /bin/ls ls.prof + Displays annotated call-graph via 'gv' +pprof --gv --focus=Mutex /bin/ls ls.prof + Restricts to code paths including a .*Mutex.* entry +pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof + Code paths including Mutex but not string +pprof --list=getdir /bin/ls ls.prof + (Per-line) annotated source listing for getdir() +pprof --disasm=getdir /bin/ls ls.prof + (Per-PC) annotated disassembly for getdir() + +pprof http://localhost:1234/ + Enters "interactive" mode +pprof --text localhost:1234 + Outputs one line per procedure for localhost:1234 +pprof --raw localhost:1234 > ./local.raw +pprof --text ./local.raw + Fetches a remote profile for later analysis and then + analyzes it in text mode. +EOF +} + +sub version_string { + return < \$main::opt_help, + "version!" => \$main::opt_version, + "cum!" => \$main::opt_cum, + "base=s" => \$main::opt_base, + "seconds=i" => \$main::opt_seconds, + "add_lib=s" => \$main::opt_lib, + "lib_prefix=s" => \$main::opt_lib_prefix, + "functions!" => \$main::opt_functions, + "lines!" => \$main::opt_lines, + "addresses!" => \$main::opt_addresses, + "files!" => \$main::opt_files, + "text!" => \$main::opt_text, + "callgrind!" => \$main::opt_callgrind, + "list=s" => \$main::opt_list, + "disasm=s" => \$main::opt_disasm, + "symbols!" => \$main::opt_symbols, + "gv!" => \$main::opt_gv, + "evince!" => \$main::opt_evince, + "web!" => \$main::opt_web, + "dot!" => \$main::opt_dot, + "ps!" => \$main::opt_ps, + "pdf!" => \$main::opt_pdf, + "svg!" => \$main::opt_svg, + "gif!" => \$main::opt_gif, + "raw!" => \$main::opt_raw, + "interactive!" => \$main::opt_interactive, + "nodecount=i" => \$main::opt_nodecount, + "nodefraction=f" => \$main::opt_nodefraction, + "edgefraction=f" => \$main::opt_edgefraction, + "maxdegree=i" => \$main::opt_maxdegree, + "focus=s" => \$main::opt_focus, + "ignore=s" => \$main::opt_ignore, + "scale=i" => \$main::opt_scale, + "heapcheck" => \$main::opt_heapcheck, + "inuse_space!" => \$main::opt_inuse_space, + "inuse_objects!" => \$main::opt_inuse_objects, + "alloc_space!" => \$main::opt_alloc_space, + "alloc_objects!" => \$main::opt_alloc_objects, + "show_bytes!" => \$main::opt_show_bytes, + "drop_negative!" => \$main::opt_drop_negative, + "total_delay!" => \$main::opt_total_delay, + "contentions!" => \$main::opt_contentions, + "mean_delay!" => \$main::opt_mean_delay, + "tools=s" => \$main::opt_tools, + "test!" => \$main::opt_test, + "debug!" => \$main::opt_debug, + # Undocumented flags used only by unittests: + "test_stride=i" => \$main::opt_test_stride, + ) || usage("Invalid option(s)"); + + # Deal with the standard --help and --version + if ($main::opt_help) { + print usage_string(); + exit(0); + } + + if ($main::opt_version) { + print version_string(); + exit(0); + } + + # Disassembly/listing/symbols mode requires address-level info + if ($main::opt_disasm || $main::opt_list || $main::opt_symbols) { + $main::opt_functions = 0; + $main::opt_lines = 0; + $main::opt_addresses = 1; + $main::opt_files = 0; + } + + # Check heap-profiling flags + if ($main::opt_inuse_space + + $main::opt_inuse_objects + + $main::opt_alloc_space + + $main::opt_alloc_objects > 1) { + usage("Specify at most on of --inuse/--alloc options"); + } + + # Check output granularities + my $grains = + $main::opt_functions + + $main::opt_lines + + $main::opt_addresses + + $main::opt_files + + 0; + if ($grains > 1) { + usage("Only specify one output granularity option"); + } + if ($grains == 0) { + $main::opt_functions = 1; + } + + # Check output modes + my $modes = + $main::opt_text + + $main::opt_callgrind + + ($main::opt_list eq '' ? 0 : 1) + + ($main::opt_disasm eq '' ? 0 : 1) + + ($main::opt_symbols == 0 ? 0 : 1) + + $main::opt_gv + + $main::opt_evince + + $main::opt_web + + $main::opt_dot + + $main::opt_ps + + $main::opt_pdf + + $main::opt_svg + + $main::opt_gif + + $main::opt_raw + + $main::opt_interactive + + 0; + if ($modes > 1) { + usage("Only specify one output mode"); + } + if ($modes == 0) { + if (-t STDOUT) { # If STDOUT is a tty, activate interactive mode + $main::opt_interactive = 1; + } else { + $main::opt_text = 1; + } + } + + if ($main::opt_test) { + RunUnitTests(); + # Should not return + exit(1); + } + + # Binary name and profile arguments list + $main::prog = ""; + @main::pfile_args = (); + + # Remote profiling without a binary (using $SYMBOL_PAGE instead) + if (IsProfileURL($ARGV[0])) { + $main::use_symbol_page = 1; + } elsif (IsSymbolizedProfileFile($ARGV[0])) { + $main::use_symbolized_profile = 1; + $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file + } + + if ($main::use_symbol_page || $main::use_symbolized_profile) { + # We don't need a binary! + my %disabled = ('--lines' => $main::opt_lines, + '--disasm' => $main::opt_disasm); + for my $option (keys %disabled) { + usage("$option cannot be used without a binary") if $disabled{$option}; + } + # Set $main::prog later... + scalar(@ARGV) || usage("Did not specify profile file"); + } elsif ($main::opt_symbols) { + # --symbols needs a binary-name (to run nm on, etc) but not profiles + $main::prog = shift(@ARGV) || usage("Did not specify program"); + } else { + $main::prog = shift(@ARGV) || usage("Did not specify program"); + scalar(@ARGV) || usage("Did not specify profile file"); + } + + # Parse profile file/location arguments + foreach my $farg (@ARGV) { + if ($farg =~ m/(.*)\@([0-9]+)(|\/.*)$/ ) { + my $machine = $1; + my $num_machines = $2; + my $path = $3; + for (my $i = 0; $i < $num_machines; $i++) { + unshift(@main::pfile_args, "$i.$machine$path"); + } + } else { + unshift(@main::pfile_args, $farg); + } + } + + if ($main::use_symbol_page) { + unless (IsProfileURL($main::pfile_args[0])) { + error("The first profile should be a remote form to use $SYMBOL_PAGE\n"); + } + CheckSymbolPage(); + $main::prog = FetchProgramName(); + } elsif (!$main::use_symbolized_profile) { # may not need objtools! + ConfigureObjTools($main::prog) + } + + # Break the opt_list_prefix into the prefix_list array + @prefix_list = split (',', $main::opt_lib_prefix); + + # Remove trailing / from the prefixes, in the list to prevent + # searching things like /my/path//lib/mylib.so + foreach (@prefix_list) { + s|/+$||; + } +} + +sub Main() { + Init(); + $main::collected_profile = undef; + @main::profile_files = (); + $main::op_time = time(); + + # Printing symbols is special and requires a lot less info that most. + if ($main::opt_symbols) { + PrintSymbols(*STDIN); # Get /proc/maps and symbols output from stdin + return; + } + + # Fetch all profile data + FetchDynamicProfiles(); + + # this will hold symbols that we read from the profile files + my $symbol_map = {}; + + # Read one profile, pick the last item on the list + my $data = ReadProfile($main::prog, pop(@main::profile_files)); + my $profile = $data->{profile}; + my $pcs = $data->{pcs}; + my $libs = $data->{libs}; # Info about main program and shared libraries + $symbol_map = MergeSymbols($symbol_map, $data->{symbols}); + + # Add additional profiles, if available. + if (scalar(@main::profile_files) > 0) { + foreach my $pname (@main::profile_files) { + my $data2 = ReadProfile($main::prog, $pname); + $profile = AddProfile($profile, $data2->{profile}); + $pcs = AddPcs($pcs, $data2->{pcs}); + $symbol_map = MergeSymbols($symbol_map, $data2->{symbols}); + } + } + + # Subtract base from profile, if specified + if ($main::opt_base ne '') { + my $base = ReadProfile($main::prog, $main::opt_base); + $profile = SubtractProfile($profile, $base->{profile}); + $pcs = AddPcs($pcs, $base->{pcs}); + $symbol_map = MergeSymbols($symbol_map, $base->{symbols}); + } + + # Get total data in profile + my $total = TotalProfile($profile); + + # Collect symbols + my $symbols; + if ($main::use_symbolized_profile) { + $symbols = FetchSymbols($pcs, $symbol_map); + } elsif ($main::use_symbol_page) { + $symbols = FetchSymbols($pcs); + } else { + # TODO(csilvers): $libs uses the /proc/self/maps data from profile1, + # which may differ from the data from subsequent profiles, especially + # if they were run on different machines. Use appropriate libs for + # each pc somehow. + $symbols = ExtractSymbols($libs, $pcs); + } + + # Remove uniniteresting stack items + $profile = RemoveUninterestingFrames($symbols, $profile); + + # Focus? + if ($main::opt_focus ne '') { + $profile = FocusProfile($symbols, $profile, $main::opt_focus); + } + + # Ignore? + if ($main::opt_ignore ne '') { + $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); + } + + my $calls = ExtractCalls($symbols, $profile); + + # Reduce profiles to required output granularity, and also clean + # each stack trace so a given entry exists at most once. + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + # Print + if (!$main::opt_interactive) { + if ($main::opt_disasm) { + PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); + } elsif ($main::opt_list) { + PrintListing($libs, $flat, $cumulative, $main::opt_list); + } elsif ($main::opt_text) { + # Make sure the output is empty when have nothing to report + # (only matters when --heapcheck is given but we must be + # compatible with old branches that did not pass --heapcheck always): + if ($total != 0) { + printf("Total: %s %s\n", Unparse($total), Units()); + } + PrintText($symbols, $flat, $cumulative, $total, -1); + } elsif ($main::opt_raw) { + PrintSymbolizedProfile($symbols, $profile, $main::prog); + } elsif ($main::opt_callgrind) { + PrintCallgrind($calls); + } else { + if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { + if ($main::opt_gv) { + RunGV(TempName($main::next_tmpfile, "ps"), ""); + } elsif ($main::opt_evince) { + RunEvince(TempName($main::next_tmpfile, "pdf"), ""); + } elsif ($main::opt_web) { + my $tmp = TempName($main::next_tmpfile, "svg"); + RunWeb($tmp); + # The command we run might hand the file name off + # to an already running browser instance and then exit. + # Normally, we'd remove $tmp on exit (right now), + # but fork a child to remove $tmp a little later, so that the + # browser has time to load it first. + delete $main::tempnames{$tmp}; + if (fork() == 0) { + sleep 5; + unlink($tmp); + exit(0); + } + } + } else { + cleanup(); + exit(1); + } + } + } else { + InteractiveMode($profile, $symbols, $libs, $total); + } + + cleanup(); + exit(0); +} + +##### Entry Point ##### + +Main(); + +# Temporary code to detect if we're running on a Goobuntu system. +# These systems don't have the right stuff installed for the special +# Readline libraries to work, so as a temporary workaround, we default +# to using the normal stdio code, rather than the fancier readline-based +# code +sub ReadlineMightFail { + if (-e '/lib/libtermcap.so.2') { + return 0; # libtermcap exists, so readline should be okay + } else { + return 1; + } +} + +sub RunGV { + my $fname = shift; + my $bg = shift; # "" or " &" if we should run in background + if (!system("$GV --version >/dev/null 2>&1")) { + # Options using double dash are supported by this gv version. + # Also, turn on noantialias to better handle bug in gv for + # postscript files with large dimensions. + # TODO: Maybe we should not pass the --noantialias flag + # if the gv version is known to work properly without the flag. + system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); + } else { + # Old gv version - only supports options that use single dash. + print STDERR "$GV -scale $main::opt_scale\n"; + system("$GV -scale $main::opt_scale " . $fname . $bg); + } +} + +sub RunEvince { + my $fname = shift; + my $bg = shift; # "" or " &" if we should run in background + system("$EVINCE " . $fname . $bg); +} + +sub RunWeb { + my $fname = shift; + print STDERR "Loading web page file:///$fname\n"; + + if (`uname` =~ /Darwin/) { + # OS X: open will use standard preference for SVG files. + system("/usr/bin/open", $fname); + return; + } + + # Some kind of Unix; try generic symlinks, then specific browsers. + # (Stop once we find one.) + # Works best if the browser is already running. + my @alt = ( + "/etc/alternatives/gnome-www-browser", + "/etc/alternatives/x-www-browser", + "google-chrome", + "firefox", + ); + foreach my $b (@alt) { + if (system($b, $fname) == 0) { + return; + } + } + + print STDERR "Could not load web browser.\n"; +} + +sub RunKcachegrind { + my $fname = shift; + my $bg = shift; # "" or " &" if we should run in background + print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; + system("$KCACHEGRIND " . $fname . $bg); +} + + +##### Interactive helper routines ##### + +sub InteractiveMode { + $| = 1; # Make output unbuffered for interactive mode + my ($orig_profile, $symbols, $libs, $total) = @_; + + print STDERR "Welcome to pprof! For help, type 'help'.\n"; + + # Use ReadLine if it's installed and input comes from a console. + if ( -t STDIN && + !ReadlineMightFail() && + defined(eval {require Term::ReadLine}) ) { + my $term = new Term::ReadLine 'pprof'; + while ( defined ($_ = $term->readline('(pprof) '))) { + $term->addhistory($_) if /\S/; + if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { + last; # exit when we get an interactive command to quit + } + } + } else { # don't have readline + while (1) { + print STDERR "(pprof) "; + $_ = ; + last if ! defined $_ ; + s/\r//g; # turn windows-looking lines into unix-looking lines + + # Save some flags that might be reset by InteractiveCommand() + my $save_opt_lines = $main::opt_lines; + + if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { + last; # exit when we get an interactive command to quit + } + + # Restore flags + $main::opt_lines = $save_opt_lines; + } + } +} + +# Takes two args: orig profile, and command to run. +# Returns 1 if we should keep going, or 0 if we were asked to quit +sub InteractiveCommand { + my($orig_profile, $symbols, $libs, $total, $command) = @_; + $_ = $command; # just to make future m//'s easier + if (!defined($_)) { + print STDERR "\n"; + return 0; + } + if (m/^\s*quit/) { + return 0; + } + if (m/^\s*help/) { + InteractiveHelpMessage(); + return 1; + } + # Clear all the mode options -- mode is controlled by "$command" + $main::opt_text = 0; + $main::opt_callgrind = 0; + $main::opt_disasm = 0; + $main::opt_list = 0; + $main::opt_gv = 0; + $main::opt_evince = 0; + $main::opt_cum = 0; + + if (m/^\s*(text|top)(\d*)\s*(.*)/) { + $main::opt_text = 1; + + my $line_limit = ($2 ne "") ? int($2) : 10; + + my $routine; + my $ignore; + ($routine, $ignore) = ParseInteractiveArgs($3); + + my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + PrintText($symbols, $flat, $cumulative, $total, $line_limit); + return 1; + } + if (m/^\s*callgrind\s*([^ \n]*)/) { + $main::opt_callgrind = 1; + + # Get derived profiles + my $calls = ExtractCalls($symbols, $orig_profile); + my $filename = $1; + if ( $1 eq '' ) { + $filename = TempName($main::next_tmpfile, "callgrind"); + } + PrintCallgrind($calls, $filename); + if ( $1 eq '' ) { + RunKcachegrind($filename, " & "); + $main::next_tmpfile++; + } + + return 1; + } + if (m/^\s*list\s*(.+)/) { + $main::opt_list = 1; + + my $routine; + my $ignore; + ($routine, $ignore) = ParseInteractiveArgs($1); + + my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + PrintListing($libs, $flat, $cumulative, $routine); + return 1; + } + if (m/^\s*disasm\s*(.+)/) { + $main::opt_disasm = 1; + + my $routine; + my $ignore; + ($routine, $ignore) = ParseInteractiveArgs($1); + + # Process current profile to account for various settings + my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + PrintDisassembly($libs, $flat, $cumulative, $routine, $total); + return 1; + } + if (m/^\s*(gv|web|evince)\s*(.*)/) { + $main::opt_gv = 0; + $main::opt_evince = 0; + $main::opt_web = 0; + if ($1 eq "gv") { + $main::opt_gv = 1; + } elsif ($1 eq "evince") { + $main::opt_evince = 1; + } elsif ($1 eq "web") { + $main::opt_web = 1; + } + + my $focus; + my $ignore; + ($focus, $ignore) = ParseInteractiveArgs($2); + + # Process current profile to account for various settings + my $profile = ProcessProfile($orig_profile, $symbols, $focus, $ignore); + my $reduced = ReduceProfile($symbols, $profile); + + # Get derived profiles + my $flat = FlatProfile($reduced); + my $cumulative = CumulativeProfile($reduced); + + if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { + if ($main::opt_gv) { + RunGV(TempName($main::next_tmpfile, "ps"), " &"); + } elsif ($main::opt_evince) { + RunEvince(TempName($main::next_tmpfile, "pdf"), " &"); + } elsif ($main::opt_web) { + RunWeb(TempName($main::next_tmpfile, "svg")); + } + $main::next_tmpfile++; + } + return 1; + } + if (m/^\s*$/) { + return 1; + } + print STDERR "Unknown command: try 'help'.\n"; + return 1; +} + + +sub ProcessProfile { + my $orig_profile = shift; + my $symbols = shift; + my $focus = shift; + my $ignore = shift; + + # Process current profile to account for various settings + my $profile = $orig_profile; + my $total_count = TotalProfile($profile); + printf("Total: %s %s\n", Unparse($total_count), Units()); + if ($focus ne '') { + $profile = FocusProfile($symbols, $profile, $focus); + my $focus_count = TotalProfile($profile); + printf("After focusing on '%s': %s %s of %s (%0.1f%%)\n", + $focus, + Unparse($focus_count), Units(), + Unparse($total_count), ($focus_count*100.0) / $total_count); + } + if ($ignore ne '') { + $profile = IgnoreProfile($symbols, $profile, $ignore); + my $ignore_count = TotalProfile($profile); + printf("After ignoring '%s': %s %s of %s (%0.1f%%)\n", + $ignore, + Unparse($ignore_count), Units(), + Unparse($total_count), + ($ignore_count*100.0) / $total_count); + } + + return $profile; +} + +sub InteractiveHelpMessage { + print STDERR <{$k}; + my @addrs = split(/\n/, $k); + if ($#addrs >= 0) { + my $depth = $#addrs + 1; + # int(foo / 2**32) is the only reliable way to get rid of bottom + # 32 bits on both 32- and 64-bit systems. + print pack('L*', $count & 0xFFFFFFFF, int($count / 2**32)); + print pack('L*', $depth & 0xFFFFFFFF, int($depth / 2**32)); + + foreach my $full_addr (@addrs) { + my $addr = $full_addr; + $addr =~ s/0x0*//; # strip off leading 0x, zeroes + if (length($addr) > 16) { + print STDERR "Invalid address in profile: $full_addr\n"; + next; + } + my $low_addr = substr($addr, -8); # get last 8 hex chars + my $high_addr = substr($addr, -16, 8); # get up to 8 more hex chars + print pack('L*', hex('0x' . $low_addr), hex('0x' . $high_addr)); + } + } + } +} + +# Print symbols and profile data +sub PrintSymbolizedProfile { + my $symbols = shift; + my $profile = shift; + my $prog = shift; + + $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $symbol_marker = $&; + + print '--- ', $symbol_marker, "\n"; + if (defined($prog)) { + print 'binary=', $prog, "\n"; + } + while (my ($pc, $name) = each(%{$symbols})) { + my $sep = ' '; + print '0x', $pc; + # We have a list of function names, which include the inlined + # calls. They are separated (and terminated) by --, which is + # illegal in function names. + for (my $j = 2; $j <= $#{$name}; $j += 3) { + print $sep, $name->[$j]; + $sep = '--'; + } + print "\n"; + } + print '---', "\n"; + + $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $profile_marker = $&; + print '--- ', $profile_marker, "\n"; + if (defined($main::collected_profile)) { + # if used with remote fetch, simply dump the collected profile to output. + open(SRC, "<$main::collected_profile"); + while () { + print $_; + } + close(SRC); + } else { + # dump a cpu-format profile to standard out + PrintProfileData($profile); + } +} + +# Print text output +sub PrintText { + my $symbols = shift; + my $flat = shift; + my $cumulative = shift; + my $total = shift; + my $line_limit = shift; + + # Which profile to sort by? + my $s = $main::opt_cum ? $cumulative : $flat; + + my $running_sum = 0; + my $lines = 0; + foreach my $k (sort { GetEntry($s, $b) <=> GetEntry($s, $a) || $a cmp $b } + keys(%{$cumulative})) { + my $f = GetEntry($flat, $k); + my $c = GetEntry($cumulative, $k); + $running_sum += $f; + + my $sym = $k; + if (exists($symbols->{$k})) { + $sym = $symbols->{$k}->[0] . " " . $symbols->{$k}->[1]; + if ($main::opt_addresses) { + $sym = $k . " " . $sym; + } + } + + if ($f != 0 || $c != 0) { + printf("%8s %6s %6s %8s %6s %s\n", + Unparse($f), + Percent($f, $total), + Percent($running_sum, $total), + Unparse($c), + Percent($c, $total), + $sym); + } + $lines++; + last if ($line_limit >= 0 && $lines > $line_limit); + } +} + +# Print the call graph in a way that's suiteable for callgrind. +sub PrintCallgrind { + my $calls = shift; + my $filename; + if ($main::opt_interactive) { + $filename = shift; + print STDERR "Writing callgrind file to '$filename'.\n" + } else { + $filename = "&STDOUT"; + } + open(CG, ">".$filename ); + printf CG ("events: Hits\n\n"); + foreach my $call ( map { $_->[0] } + sort { $a->[1] cmp $b ->[1] || + $a->[2] <=> $b->[2] } + map { /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; + [$_, $1, $2] } + keys %$calls ) { + my $count = int($calls->{$call}); + $call =~ /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; + my ( $caller_file, $caller_line, $caller_function, + $callee_file, $callee_line, $callee_function ) = + ( $1, $2, $3, $5, $6, $7 ); + + + printf CG ("fl=$caller_file\nfn=$caller_function\n"); + if (defined $6) { + printf CG ("cfl=$callee_file\n"); + printf CG ("cfn=$callee_function\n"); + printf CG ("calls=$count $callee_line\n"); + } + printf CG ("$caller_line $count\n\n"); + } +} + +# Print disassembly for all all routines that match $main::opt_disasm +sub PrintDisassembly { + my $libs = shift; + my $flat = shift; + my $cumulative = shift; + my $disasm_opts = shift; + my $total = shift; + + foreach my $lib (@{$libs}) { + my $symbol_table = GetProcedureBoundaries($lib->[0], $disasm_opts); + my $offset = AddressSub($lib->[1], $lib->[3]); + foreach my $routine (sort ByName keys(%{$symbol_table})) { + my $start_addr = $symbol_table->{$routine}->[0]; + my $end_addr = $symbol_table->{$routine}->[1]; + # See if there are any samples in this routine + my $length = hex(AddressSub($end_addr, $start_addr)); + my $addr = AddressAdd($start_addr, $offset); + for (my $i = 0; $i < $length; $i++) { + if (defined($cumulative->{$addr})) { + PrintDisassembledFunction($lib->[0], $offset, + $routine, $flat, $cumulative, + $start_addr, $end_addr, $total); + last; + } + $addr = AddressInc($addr); + } + } + } +} + +# Return reference to array of tuples of the form: +# [start_address, filename, linenumber, instruction, limit_address] +# E.g., +# ["0x806c43d", "/foo/bar.cc", 131, "ret", "0x806c440"] +sub Disassemble { + my $prog = shift; + my $offset = shift; + my $start_addr = shift; + my $end_addr = shift; + + my $objdump = $obj_tool_map{"objdump"}; + my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . + "--start-address=0x$start_addr " . + "--stop-address=0x$end_addr $prog"); + open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); + my @result = (); + my $filename = ""; + my $linenumber = -1; + my $last = ["", "", "", ""]; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + chop; + if (m|\s*([^:\s]+):(\d+)\s*$|) { + # Location line of the form: + # : + $filename = $1; + $linenumber = $2; + } elsif (m/^ +([0-9a-f]+):\s*(.*)/) { + # Disassembly line -- zero-extend address to full length + my $addr = HexExtend($1); + my $k = AddressAdd($addr, $offset); + $last->[4] = $k; # Store ending address for previous instruction + $last = [$k, $filename, $linenumber, $2, $end_addr]; + push(@result, $last); + } + } + close(OBJDUMP); + return @result; +} + +# The input file should contain lines of the form /proc/maps-like +# output (same format as expected from the profiles) or that looks +# like hex addresses (like "0xDEADBEEF"). We will parse all +# /proc/maps output, and for all the hex addresses, we will output +# "short" symbol names, one per line, in the same order as the input. +sub PrintSymbols { + my $maps_and_symbols_file = shift; + + # ParseLibraries expects pcs to be in a set. Fine by us... + my @pclist = (); # pcs in sorted order + my $pcs = {}; + my $map = ""; + foreach my $line (<$maps_and_symbols_file>) { + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + if ($line =~ /\b(0x[0-9a-f]+)\b/i) { + push(@pclist, HexExtend($1)); + $pcs->{$pclist[-1]} = 1; + } else { + $map .= $line; + } + } + + my $libs = ParseLibraries($main::prog, $map, $pcs); + my $symbols = ExtractSymbols($libs, $pcs); + + foreach my $pc (@pclist) { + # ->[0] is the shortname, ->[2] is the full name + print(($symbols->{$pc}->[0] || "??") . "\n"); + } +} + + +# For sorting functions by name +sub ByName { + return ShortFunctionName($a) cmp ShortFunctionName($b); +} + +# Print source-listing for all all routines that match $main::opt_list +sub PrintListing { + my $libs = shift; + my $flat = shift; + my $cumulative = shift; + my $list_opts = shift; + + foreach my $lib (@{$libs}) { + my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); + my $offset = AddressSub($lib->[1], $lib->[3]); + foreach my $routine (sort ByName keys(%{$symbol_table})) { + # Print if there are any samples in this routine + my $start_addr = $symbol_table->{$routine}->[0]; + my $end_addr = $symbol_table->{$routine}->[1]; + my $length = hex(AddressSub($end_addr, $start_addr)); + my $addr = AddressAdd($start_addr, $offset); + for (my $i = 0; $i < $length; $i++) { + if (defined($cumulative->{$addr})) { + PrintSource($lib->[0], $offset, + $routine, $flat, $cumulative, + $start_addr, $end_addr); + last; + } + $addr = AddressInc($addr); + } + } + } +} + +# Returns the indentation of the line, if it has any non-whitespace +# characters. Otherwise, returns -1. +sub Indentation { + my $line = shift; + if (m/^(\s*)\S/) { + return length($1); + } else { + return -1; + } +} + +# Print source-listing for one routine +sub PrintSource { + my $prog = shift; + my $offset = shift; + my $routine = shift; + my $flat = shift; + my $cumulative = shift; + my $start_addr = shift; + my $end_addr = shift; + + # Disassemble all instructions (just to get line numbers) + my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + + # Hack 1: assume that the first source file encountered in the + # disassembly contains the routine + my $filename = undef; + for (my $i = 0; $i <= $#instructions; $i++) { + if ($instructions[$i]->[2] >= 0) { + $filename = $instructions[$i]->[1]; + last; + } + } + if (!defined($filename)) { + print STDERR "no filename found in $routine\n"; + return; + } + + # Hack 2: assume that the largest line number from $filename is the + # end of the procedure. This is typically safe since if P1 contains + # an inlined call to P2, then P2 usually occurs earlier in the + # source file. If this does not work, we might have to compute a + # density profile or just print all regions we find. + my $lastline = 0; + for (my $i = 0; $i <= $#instructions; $i++) { + my $f = $instructions[$i]->[1]; + my $l = $instructions[$i]->[2]; + if (($f eq $filename) && ($l > $lastline)) { + $lastline = $l; + } + } + + # Hack 3: assume the first source location from "filename" is the start of + # the source code. + my $firstline = 1; + for (my $i = 0; $i <= $#instructions; $i++) { + if ($instructions[$i]->[1] eq $filename) { + $firstline = $instructions[$i]->[2]; + last; + } + } + + # Hack 4: Extend last line forward until its indentation is less than + # the indentation we saw on $firstline + my $oldlastline = $lastline; + { + if (!open(FILE, "<$filename")) { + print STDERR "$filename: $!\n"; + return; + } + my $l = 0; + my $first_indentation = -1; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + $l++; + my $indent = Indentation($_); + if ($l >= $firstline) { + if ($first_indentation < 0 && $indent >= 0) { + $first_indentation = $indent; + last if ($first_indentation == 0); + } + } + if ($l >= $lastline && $indent >= 0) { + if ($indent >= $first_indentation) { + $lastline = $l+1; + } else { + last; + } + } + } + close(FILE); + } + + # Assign all samples to the range $firstline,$lastline, + # Hack 4: If an instruction does not occur in the range, its samples + # are moved to the next instruction that occurs in the range. + my $samples1 = {}; + my $samples2 = {}; + my $running1 = 0; # Unassigned flat counts + my $running2 = 0; # Unassigned cumulative counts + my $total1 = 0; # Total flat counts + my $total2 = 0; # Total cumulative counts + foreach my $e (@instructions) { + # Add up counts for all address that fall inside this instruction + my $c1 = 0; + my $c2 = 0; + for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { + $c1 += GetEntry($flat, $a); + $c2 += GetEntry($cumulative, $a); + } + $running1 += $c1; + $running2 += $c2; + $total1 += $c1; + $total2 += $c2; + my $file = $e->[1]; + my $line = $e->[2]; + if (($file eq $filename) && + ($line >= $firstline) && + ($line <= $lastline)) { + # Assign all accumulated samples to this line + AddEntry($samples1, $line, $running1); + AddEntry($samples2, $line, $running2); + $running1 = 0; + $running2 = 0; + } + } + + # Assign any leftover samples to $lastline + AddEntry($samples1, $lastline, $running1); + AddEntry($samples2, $lastline, $running2); + + printf("ROUTINE ====================== %s in %s\n" . + "%6s %6s Total %s (flat / cumulative)\n", + ShortFunctionName($routine), + $filename, + Units(), + Unparse($total1), + Unparse($total2)); + if (!open(FILE, "<$filename")) { + print STDERR "$filename: $!\n"; + return; + } + my $l = 0; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + $l++; + if ($l >= $firstline - 5 && + (($l <= $oldlastline + 5) || ($l <= $lastline))) { + chop; + my $text = $_; + if ($l == $firstline) { printf("---\n"); } + printf("%6s %6s %4d: %s\n", + UnparseAlt(GetEntry($samples1, $l)), + UnparseAlt(GetEntry($samples2, $l)), + $l, + $text); + if ($l == $lastline) { printf("---\n"); } + }; + } + close(FILE); +} + +# Return the source line for the specified file/linenumber. +# Returns undef if not found. +sub SourceLine { + my $file = shift; + my $line = shift; + + # Look in cache + if (!defined($main::source_cache{$file})) { + if (100 < scalar keys(%main::source_cache)) { + # Clear the cache when it gets too big + $main::source_cache = (); + } + + # Read all lines from the file + if (!open(FILE, "<$file")) { + print STDERR "$file: $!\n"; + $main::source_cache{$file} = []; # Cache the negative result + return undef; + } + my $lines = []; + push(@{$lines}, ""); # So we can use 1-based line numbers as indices + while () { + push(@{$lines}, $_); + } + close(FILE); + + # Save the lines in the cache + $main::source_cache{$file} = $lines; + } + + my $lines = $main::source_cache{$file}; + if (($line < 0) || ($line > $#{$lines})) { + return undef; + } else { + return $lines->[$line]; + } +} + +# Print disassembly for one routine with interspersed source if available +sub PrintDisassembledFunction { + my $prog = shift; + my $offset = shift; + my $routine = shift; + my $flat = shift; + my $cumulative = shift; + my $start_addr = shift; + my $end_addr = shift; + my $total = shift; + + # Disassemble all instructions + my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + + # Make array of counts per instruction + my @flat_count = (); + my @cum_count = (); + my $flat_total = 0; + my $cum_total = 0; + foreach my $e (@instructions) { + # Add up counts for all address that fall inside this instruction + my $c1 = 0; + my $c2 = 0; + for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { + $c1 += GetEntry($flat, $a); + $c2 += GetEntry($cumulative, $a); + } + push(@flat_count, $c1); + push(@cum_count, $c2); + $flat_total += $c1; + $cum_total += $c2; + } + + # Print header with total counts + printf("ROUTINE ====================== %s\n" . + "%6s %6s %s (flat, cumulative) %.1f%% of total\n", + ShortFunctionName($routine), + Unparse($flat_total), + Unparse($cum_total), + Units(), + ($cum_total * 100.0) / $total); + + # Process instructions in order + my $current_file = ""; + for (my $i = 0; $i <= $#instructions; ) { + my $e = $instructions[$i]; + + # Print the new file name whenever we switch files + if ($e->[1] ne $current_file) { + $current_file = $e->[1]; + my $fname = $current_file; + $fname =~ s|^\./||; # Trim leading "./" + + # Shorten long file names + if (length($fname) >= 58) { + $fname = "..." . substr($fname, -55); + } + printf("-------------------- %s\n", $fname); + } + + # TODO: Compute range of lines to print together to deal with + # small reorderings. + my $first_line = $e->[2]; + my $last_line = $first_line; + my %flat_sum = (); + my %cum_sum = (); + for (my $l = $first_line; $l <= $last_line; $l++) { + $flat_sum{$l} = 0; + $cum_sum{$l} = 0; + } + + # Find run of instructions for this range of source lines + my $first_inst = $i; + while (($i <= $#instructions) && + ($instructions[$i]->[2] >= $first_line) && + ($instructions[$i]->[2] <= $last_line)) { + $e = $instructions[$i]; + $flat_sum{$e->[2]} += $flat_count[$i]; + $cum_sum{$e->[2]} += $cum_count[$i]; + $i++; + } + my $last_inst = $i - 1; + + # Print source lines + for (my $l = $first_line; $l <= $last_line; $l++) { + my $line = SourceLine($current_file, $l); + if (!defined($line)) { + $line = "?\n"; + next; + } else { + $line =~ s/^\s+//; + } + printf("%6s %6s %5d: %s", + UnparseAlt($flat_sum{$l}), + UnparseAlt($cum_sum{$l}), + $l, + $line); + } + + # Print disassembly + for (my $x = $first_inst; $x <= $last_inst; $x++) { + my $e = $instructions[$x]; + my $address = $e->[0]; + $address = AddressSub($address, $offset); # Make relative to section + $address =~ s/^0x//; + $address =~ s/^0*//; + + # Trim symbols + my $d = $e->[3]; + while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments + + printf("%6s %6s %8s: %6s\n", + UnparseAlt($flat_count[$x]), + UnparseAlt($cum_count[$x]), + $address, + $d); + } + } +} + +# Print DOT graph +sub PrintDot { + my $prog = shift; + my $symbols = shift; + my $raw = shift; + my $flat = shift; + my $cumulative = shift; + my $overall_total = shift; + + # Get total + my $local_total = TotalProfile($flat); + my $nodelimit = int($main::opt_nodefraction * $local_total); + my $edgelimit = int($main::opt_edgefraction * $local_total); + my $nodecount = $main::opt_nodecount; + + # Find nodes to include + my @list = (sort { abs(GetEntry($cumulative, $b)) <=> + abs(GetEntry($cumulative, $a)) + || $a cmp $b } + keys(%{$cumulative})); + my $last = $nodecount - 1; + if ($last > $#list) { + $last = $#list; + } + while (($last >= 0) && + (abs(GetEntry($cumulative, $list[$last])) <= $nodelimit)) { + $last--; + } + if ($last < 0) { + print STDERR "No nodes to print\n"; + return 0; + } + + if ($nodelimit > 0 || $edgelimit > 0) { + printf STDERR ("Dropping nodes with <= %s %s; edges with <= %s abs(%s)\n", + Unparse($nodelimit), Units(), + Unparse($edgelimit), Units()); + } + + # Open DOT output file + my $output; + if ($main::opt_gv) { + $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); + } elsif ($main::opt_evince) { + $output = "| $DOT -Tps2 | $PS2PDF - " . TempName($main::next_tmpfile, "pdf"); + } elsif ($main::opt_ps) { + $output = "| $DOT -Tps2"; + } elsif ($main::opt_pdf) { + $output = "| $DOT -Tps2 | $PS2PDF - -"; + } elsif ($main::opt_web || $main::opt_svg) { + # We need to post-process the SVG, so write to a temporary file always. + $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); + } elsif ($main::opt_gif) { + $output = "| $DOT -Tgif"; + } else { + $output = ">&STDOUT"; + } + open(DOT, $output) || error("$output: $!\n"); + + # Title + printf DOT ("digraph \"%s; %s %s\" {\n", + $prog, + Unparse($overall_total), + Units()); + if ($main::opt_pdf) { + # The output is more printable if we set the page size for dot. + printf DOT ("size=\"8,11\"\n"); + } + printf DOT ("node [width=0.375,height=0.25];\n"); + + # Print legend + printf DOT ("Legend [shape=box,fontsize=24,shape=plaintext," . + "label=\"%s\\l%s\\l%s\\l%s\\l%s\\l\"];\n", + $prog, + sprintf("Total %s: %s", Units(), Unparse($overall_total)), + sprintf("Focusing on: %s", Unparse($local_total)), + sprintf("Dropped nodes with <= %s abs(%s)", + Unparse($nodelimit), Units()), + sprintf("Dropped edges with <= %s %s", + Unparse($edgelimit), Units()) + ); + + # Print nodes + my %node = (); + my $nextnode = 1; + foreach my $a (@list[0..$last]) { + # Pick font size + my $f = GetEntry($flat, $a); + my $c = GetEntry($cumulative, $a); + + my $fs = 8; + if ($local_total > 0) { + $fs = 8 + (50.0 * sqrt(abs($f * 1.0 / $local_total))); + } + + $node{$a} = $nextnode++; + my $sym = $a; + $sym =~ s/\s+/\\n/g; + $sym =~ s/::/\\n/g; + + # Extra cumulative info to print for non-leaves + my $extra = ""; + if ($f != $c) { + $extra = sprintf("\\rof %s (%s)", + Unparse($c), + Percent($c, $overall_total)); + } + my $style = ""; + if ($main::opt_heapcheck) { + if ($f > 0) { + # make leak-causing nodes more visible (add a background) + $style = ",style=filled,fillcolor=gray" + } elsif ($f < 0) { + # make anti-leak-causing nodes (which almost never occur) + # stand out as well (triple border) + $style = ",peripheries=3" + } + } + + printf DOT ("N%d [label=\"%s\\n%s (%s)%s\\r" . + "\",shape=box,fontsize=%.1f%s];\n", + $node{$a}, + $sym, + Unparse($f), + Percent($f, $overall_total), + $extra, + $fs, + $style, + ); + } + + # Get edges and counts per edge + my %edge = (); + my $n; + foreach my $k (keys(%{$raw})) { + # TODO: omit low %age edges + $n = $raw->{$k}; + my @translated = TranslateStack($symbols, $k); + for (my $i = 1; $i <= $#translated; $i++) { + my $src = $translated[$i]; + my $dst = $translated[$i-1]; + #next if ($src eq $dst); # Avoid self-edges? + if (exists($node{$src}) && exists($node{$dst})) { + my $edge_label = "$src\001$dst"; + if (!exists($edge{$edge_label})) { + $edge{$edge_label} = 0; + } + $edge{$edge_label} += $n; + } + } + } + + # Print edges (process in order of decreasing counts) + my %indegree = (); # Number of incoming edges added per node so far + my %outdegree = (); # Number of outgoing edges added per node so far + foreach my $e (sort { $edge{$b} <=> $edge{$a} } keys(%edge)) { + my @x = split(/\001/, $e); + $n = $edge{$e}; + + # Initialize degree of kept incoming and outgoing edges if necessary + my $src = $x[0]; + my $dst = $x[1]; + if (!exists($outdegree{$src})) { $outdegree{$src} = 0; } + if (!exists($indegree{$dst})) { $indegree{$dst} = 0; } + + my $keep; + if ($indegree{$dst} == 0) { + # Keep edge if needed for reachability + $keep = 1; + } elsif (abs($n) <= $edgelimit) { + # Drop if we are below --edgefraction + $keep = 0; + } elsif ($outdegree{$src} >= $main::opt_maxdegree || + $indegree{$dst} >= $main::opt_maxdegree) { + # Keep limited number of in/out edges per node + $keep = 0; + } else { + $keep = 1; + } + + if ($keep) { + $outdegree{$src}++; + $indegree{$dst}++; + + # Compute line width based on edge count + my $fraction = abs($local_total ? (3 * ($n / $local_total)) : 0); + if ($fraction > 1) { $fraction = 1; } + my $w = $fraction * 2; + if ($w < 1 && ($main::opt_web || $main::opt_svg)) { + # SVG output treats line widths < 1 poorly. + $w = 1; + } + + # Dot sometimes segfaults if given edge weights that are too large, so + # we cap the weights at a large value + my $edgeweight = abs($n) ** 0.7; + if ($edgeweight > 100000) { $edgeweight = 100000; } + $edgeweight = int($edgeweight); + + my $style = sprintf("setlinewidth(%f)", $w); + if ($x[1] =~ m/\(inline\)/) { + $style .= ",dashed"; + } + + # Use a slightly squashed function of the edge count as the weight + printf DOT ("N%s -> N%s [label=%s, weight=%d, style=\"%s\"];\n", + $node{$x[0]}, + $node{$x[1]}, + Unparse($n), + $edgeweight, + $style); + } + } + + print DOT ("}\n"); + close(DOT); + + if ($main::opt_web || $main::opt_svg) { + # Rewrite SVG to be more usable inside web browser. + RewriteSvg(TempName($main::next_tmpfile, "svg")); + } + + return 1; +} + +sub RewriteSvg { + my $svgfile = shift; + + open(SVG, $svgfile) || die "open temp svg: $!"; + my @svg = ; + close(SVG); + unlink $svgfile; + my $svg = join('', @svg); + + # Dot's SVG output is + # + # + # + # ... + # + # + # + # Change it to + # + # + # $svg_javascript + # + # + # ... + # + # + # + + # Fix width, height; drop viewBox. + $svg =~ s/(?s) above first + my $svg_javascript = SvgJavascript(); + my $viewport = "\n"; + $svg =~ s/ above . + $svg =~ s/(.*)(<\/svg>)/$1<\/g>$2/; + $svg =~ s/$svgfile") || die "open $svgfile: $!"; + print SVG $svg; + close(SVG); + } +} + +sub SvgJavascript { + return <<'EOF'; + +EOF +} + +# Return a small number that identifies the argument. +# Multiple calls with the same argument will return the same number. +# Calls with different arguments will return different numbers. +sub ShortIdFor { + my $key = shift; + my $id = $main::uniqueid{$key}; + if (!defined($id)) { + $id = keys(%main::uniqueid) + 1; + $main::uniqueid{$key} = $id; + } + return $id; +} + +# Translate a stack of addresses into a stack of symbols +sub TranslateStack { + my $symbols = shift; + my $k = shift; + + my @addrs = split(/\n/, $k); + my @result = (); + for (my $i = 0; $i <= $#addrs; $i++) { + my $a = $addrs[$i]; + + # Skip large addresses since they sometimes show up as fake entries on RH9 + if (length($a) > 8 && $a gt "7fffffffffffffff") { + next; + } + + if ($main::opt_disasm || $main::opt_list) { + # We want just the address for the key + push(@result, $a); + next; + } + + my $symlist = $symbols->{$a}; + if (!defined($symlist)) { + $symlist = [$a, "", $a]; + } + + # We can have a sequence of symbols for a particular entry + # (more than one symbol in the case of inlining). Callers + # come before callees in symlist, so walk backwards since + # the translated stack should contain callees before callers. + for (my $j = $#{$symlist}; $j >= 2; $j -= 3) { + my $func = $symlist->[$j-2]; + my $fileline = $symlist->[$j-1]; + my $fullfunc = $symlist->[$j]; + if ($j > 2) { + $func = "$func (inline)"; + } + + # Do not merge nodes corresponding to Callback::Run since that + # causes confusing cycles in dot display. Instead, we synthesize + # a unique name for this frame per caller. + if ($func =~ m/Callback.*::Run$/) { + my $caller = ($i > 0) ? $addrs[$i-1] : 0; + $func = "Run#" . ShortIdFor($caller); + } + + if ($main::opt_addresses) { + push(@result, "$a $func $fileline"); + } elsif ($main::opt_lines) { + if ($func eq '??' && $fileline eq '??:0') { + push(@result, "$a"); + } else { + push(@result, "$func $fileline"); + } + } elsif ($main::opt_functions) { + if ($func eq '??') { + push(@result, "$a"); + } else { + push(@result, $func); + } + } elsif ($main::opt_files) { + if ($fileline eq '??:0' || $fileline eq '') { + push(@result, "$a"); + } else { + my $f = $fileline; + $f =~ s/:\d+$//; + push(@result, $f); + } + } else { + push(@result, $a); + last; # Do not print inlined info + } + } + } + + # print join(",", @addrs), " => ", join(",", @result), "\n"; + return @result; +} + +# Generate percent string for a number and a total +sub Percent { + my $num = shift; + my $tot = shift; + if ($tot != 0) { + return sprintf("%.1f%%", $num * 100.0 / $tot); + } else { + return ($num == 0) ? "nan" : (($num > 0) ? "+inf" : "-inf"); + } +} + +# Generate pretty-printed form of number +sub Unparse { + my $num = shift; + if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { + if ($main::opt_inuse_objects || $main::opt_alloc_objects) { + return sprintf("%d", $num); + } else { + if ($main::opt_show_bytes) { + return sprintf("%d", $num); + } else { + return sprintf("%.1f", $num / 1048576.0); + } + } + } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { + return sprintf("%.3f", $num / 1e9); # Convert nanoseconds to seconds + } else { + return sprintf("%d", $num); + } +} + +# Alternate pretty-printed form: 0 maps to "." +sub UnparseAlt { + my $num = shift; + if ($num == 0) { + return "."; + } else { + return Unparse($num); + } +} + +# Return output units +sub Units { + if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { + if ($main::opt_inuse_objects || $main::opt_alloc_objects) { + return "objects"; + } else { + if ($main::opt_show_bytes) { + return "B"; + } else { + return "MB"; + } + } + } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { + return "seconds"; + } else { + return "samples"; + } +} + +##### Profile manipulation code ##### + +# Generate flattened profile: +# If count is charged to stack [a,b,c,d], in generated profile, +# it will be charged to [a] +sub FlatProfile { + my $profile = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + if ($#addrs >= 0) { + AddEntry($result, $addrs[0], $count); + } + } + return $result; +} + +# Generate cumulative profile: +# If count is charged to stack [a,b,c,d], in generated profile, +# it will be charged to [a], [b], [c], [d] +sub CumulativeProfile { + my $profile = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + foreach my $a (@addrs) { + AddEntry($result, $a, $count); + } + } + return $result; +} + +# If the second-youngest PC on the stack is always the same, returns +# that pc. Otherwise, returns undef. +sub IsSecondPcAlwaysTheSame { + my $profile = shift; + + my $second_pc = undef; + foreach my $k (keys(%{$profile})) { + my @addrs = split(/\n/, $k); + if ($#addrs < 1) { + return undef; + } + if (not defined $second_pc) { + $second_pc = $addrs[1]; + } else { + if ($second_pc ne $addrs[1]) { + return undef; + } + } + } + return $second_pc; +} + +sub ExtractSymbolLocation { + my $symbols = shift; + my $address = shift; + # 'addr2line' outputs "??:0" for unknown locations; we do the + # same to be consistent. + my $location = "??:0:unknown"; + if (exists $symbols->{$address}) { + my $file = $symbols->{$address}->[1]; + if ($file eq "?") { + $file = "??:0" + } + $location = $file . ":" . $symbols->{$address}->[0]; + } + return $location; +} + +# Extracts a graph of calls. +sub ExtractCalls { + my $symbols = shift; + my $profile = shift; + + my $calls = {}; + while( my ($stack_trace, $count) = each %$profile ) { + my @address = split(/\n/, $stack_trace); + my $destination = ExtractSymbolLocation($symbols, $address[0]); + AddEntry($calls, $destination, $count); + for (my $i = 1; $i <= $#address; $i++) { + my $source = ExtractSymbolLocation($symbols, $address[$i]); + my $call = "$source -> $destination"; + AddEntry($calls, $call, $count); + $destination = $source; + } + } + + return $calls; +} + +sub RemoveUninterestingFrames { + my $symbols = shift; + my $profile = shift; + + # List of function names to skip + my %skip = (); + my $skip_regexp = 'NOMATCH'; + if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { + foreach my $name ('calloc', + 'cfree', + 'malloc', + 'free', + 'memalign', + 'posix_memalign', + 'pvalloc', + 'valloc', + 'realloc', + 'tc_calloc', + 'tc_cfree', + 'tc_malloc', + 'tc_free', + 'tc_memalign', + 'tc_posix_memalign', + 'tc_pvalloc', + 'tc_valloc', + 'tc_realloc', + 'tc_new', + 'tc_delete', + 'tc_newarray', + 'tc_deletearray', + 'tc_new_nothrow', + 'tc_newarray_nothrow', + 'do_malloc', + '::do_malloc', # new name -- got moved to an unnamed ns + '::do_malloc_or_cpp_alloc', + 'DoSampledAllocation', + 'simple_alloc::allocate', + '__malloc_alloc_template::allocate', + '__builtin_delete', + '__builtin_new', + '__builtin_vec_delete', + '__builtin_vec_new', + 'operator new', + 'operator new[]', + # These mark the beginning/end of our custom sections + '__start_google_malloc', + '__stop_google_malloc', + '__start_malloc_hook', + '__stop_malloc_hook') { + $skip{$name} = 1; + $skip{"_" . $name} = 1; # Mach (OS X) adds a _ prefix to everything + } + # TODO: Remove TCMalloc once everything has been + # moved into the tcmalloc:: namespace and we have flushed + # old code out of the system. + $skip_regexp = "TCMalloc|^tcmalloc::"; + } elsif ($main::profile_type eq 'contention') { + foreach my $vname ('base::RecordLockProfileData', + 'base::SubmitMutexProfileData', + 'base::SubmitSpinLockProfileData', + 'Mutex::Unlock', + 'Mutex::UnlockSlow', + 'Mutex::ReaderUnlock', + 'MutexLock::~MutexLock', + 'SpinLock::Unlock', + 'SpinLock::SlowUnlock', + 'SpinLockHolder::~SpinLockHolder') { + $skip{$vname} = 1; + } + } elsif ($main::profile_type eq 'cpu') { + # Drop signal handlers used for CPU profile collection + # TODO(dpeng): this should not be necessary; it's taken + # care of by the general 2nd-pc mechanism below. + foreach my $name ('ProfileData::Add', # historical + 'ProfileData::prof_handler', # historical + 'CpuProfiler::prof_handler', + '__FRAME_END__', + '__pthread_sighandler', + '__restore') { + $skip{$name} = 1; + } + } else { + # Nothing skipped for unknown types + } + + if ($main::profile_type eq 'cpu') { + # If all the second-youngest program counters are the same, + # this STRONGLY suggests that it is an artifact of measurement, + # i.e., stack frames pushed by the CPU profiler signal handler. + # Hence, we delete them. + # (The topmost PC is read from the signal structure, not from + # the stack, so it does not get involved.) + while (my $second_pc = IsSecondPcAlwaysTheSame($profile)) { + my $result = {}; + my $func = ''; + if (exists($symbols->{$second_pc})) { + $second_pc = $symbols->{$second_pc}->[0]; + } + print STDERR "Removing $second_pc from all stack traces.\n"; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + splice @addrs, 1, 1; + my $reduced_path = join("\n", @addrs); + AddEntry($result, $reduced_path, $count); + } + $profile = $result; + } + } + + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + my @path = (); + foreach my $a (@addrs) { + if (exists($symbols->{$a})) { + my $func = $symbols->{$a}->[0]; + if ($skip{$func} || ($func =~ m/$skip_regexp/)) { + next; + } + } + push(@path, $a); + } + my $reduced_path = join("\n", @path); + AddEntry($result, $reduced_path, $count); + } + return $result; +} + +# Reduce profile to granularity given by user +sub ReduceProfile { + my $symbols = shift; + my $profile = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @translated = TranslateStack($symbols, $k); + my @path = (); + my %seen = (); + $seen{''} = 1; # So that empty keys are skipped + foreach my $e (@translated) { + # To avoid double-counting due to recursion, skip a stack-trace + # entry if it has already been seen + if (!$seen{$e}) { + $seen{$e} = 1; + push(@path, $e); + } + } + my $reduced_path = join("\n", @path); + AddEntry($result, $reduced_path, $count); + } + return $result; +} + +# Does the specified symbol array match the regexp? +sub SymbolMatches { + my $sym = shift; + my $re = shift; + if (defined($sym)) { + for (my $i = 0; $i < $#{$sym}; $i += 3) { + if ($sym->[$i] =~ m/$re/ || $sym->[$i+1] =~ m/$re/) { + return 1; + } + } + } + return 0; +} + +# Focus only on paths involving specified regexps +sub FocusProfile { + my $symbols = shift; + my $profile = shift; + my $focus = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + foreach my $a (@addrs) { + # Reply if it matches either the address/shortname/fileline + if (($a =~ m/$focus/) || SymbolMatches($symbols->{$a}, $focus)) { + AddEntry($result, $k, $count); + last; + } + } + } + return $result; +} + +# Focus only on paths not involving specified regexps +sub IgnoreProfile { + my $symbols = shift; + my $profile = shift; + my $ignore = shift; + my $result = {}; + foreach my $k (keys(%{$profile})) { + my $count = $profile->{$k}; + my @addrs = split(/\n/, $k); + my $matched = 0; + foreach my $a (@addrs) { + # Reply if it matches either the address/shortname/fileline + if (($a =~ m/$ignore/) || SymbolMatches($symbols->{$a}, $ignore)) { + $matched = 1; + last; + } + } + if (!$matched) { + AddEntry($result, $k, $count); + } + } + return $result; +} + +# Get total count in profile +sub TotalProfile { + my $profile = shift; + my $result = 0; + foreach my $k (keys(%{$profile})) { + $result += $profile->{$k}; + } + return $result; +} + +# Add A to B +sub AddProfile { + my $A = shift; + my $B = shift; + + my $R = {}; + # add all keys in A + foreach my $k (keys(%{$A})) { + my $v = $A->{$k}; + AddEntry($R, $k, $v); + } + # add all keys in B + foreach my $k (keys(%{$B})) { + my $v = $B->{$k}; + AddEntry($R, $k, $v); + } + return $R; +} + +# Merges symbol maps +sub MergeSymbols { + my $A = shift; + my $B = shift; + + my $R = {}; + foreach my $k (keys(%{$A})) { + $R->{$k} = $A->{$k}; + } + if (defined($B)) { + foreach my $k (keys(%{$B})) { + $R->{$k} = $B->{$k}; + } + } + return $R; +} + + +# Add A to B +sub AddPcs { + my $A = shift; + my $B = shift; + + my $R = {}; + # add all keys in A + foreach my $k (keys(%{$A})) { + $R->{$k} = 1 + } + # add all keys in B + foreach my $k (keys(%{$B})) { + $R->{$k} = 1 + } + return $R; +} + +# Subtract B from A +sub SubtractProfile { + my $A = shift; + my $B = shift; + + my $R = {}; + foreach my $k (keys(%{$A})) { + my $v = $A->{$k} - GetEntry($B, $k); + if ($v < 0 && $main::opt_drop_negative) { + $v = 0; + } + AddEntry($R, $k, $v); + } + if (!$main::opt_drop_negative) { + # Take care of when subtracted profile has more entries + foreach my $k (keys(%{$B})) { + if (!exists($A->{$k})) { + AddEntry($R, $k, 0 - $B->{$k}); + } + } + } + return $R; +} + +# Get entry from profile; zero if not present +sub GetEntry { + my $profile = shift; + my $k = shift; + if (exists($profile->{$k})) { + return $profile->{$k}; + } else { + return 0; + } +} + +# Add entry to specified profile +sub AddEntry { + my $profile = shift; + my $k = shift; + my $n = shift; + if (!exists($profile->{$k})) { + $profile->{$k} = 0; + } + $profile->{$k} += $n; +} + +# Add a stack of entries to specified profile, and add them to the $pcs +# list. +sub AddEntries { + my $profile = shift; + my $pcs = shift; + my $stack = shift; + my $count = shift; + my @k = (); + + foreach my $e (split(/\s+/, $stack)) { + my $pc = HexExtend($e); + $pcs->{$pc} = 1; + push @k, $pc; + } + AddEntry($profile, (join "\n", @k), $count); +} + +##### Code to profile a server dynamically ##### + +sub CheckSymbolPage { + my $url = SymbolPageURL(); + open(SYMBOL, "$URL_FETCHER '$url' |"); + my $line = ; + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + close(SYMBOL); + unless (defined($line)) { + error("$url doesn't exist\n"); + } + + if ($line =~ /^num_symbols:\s+(\d+)$/) { + if ($1 == 0) { + error("Stripped binary. No symbols available.\n"); + } + } else { + error("Failed to get the number of symbols from $url\n"); + } +} + +sub IsProfileURL { + my $profile_name = shift; + if (-f $profile_name) { + printf STDERR "Using local file $profile_name.\n"; + return 0; + } + return 1; +} + +sub ParseProfileURL { + my $profile_name = shift; + + if (!defined($profile_name) || $profile_name eq "") { + return (); + } + + # Split profile URL - matches all non-empty strings, so no test. + $profile_name =~ m,^(https?://)?([^/]+)(.*?)(/|$PROFILES)?$,; + + my $proto = $1 || "http://"; + my $hostport = $2; + my $prefix = $3; + my $profile = $4 || "/"; + + my $host = $hostport; + $host =~ s/:.*//; + + my $baseurl = "$proto$hostport$prefix"; + return ($host, $baseurl, $profile); +} + +# We fetch symbols from the first profile argument. +sub SymbolPageURL { + my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); + return "$baseURL$SYMBOL_PAGE"; +} + +sub FetchProgramName() { + my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); + my $url = "$baseURL$PROGRAM_NAME_PAGE"; + my $command_line = "$URL_FETCHER '$url'"; + open(CMDLINE, "$command_line |") or error($command_line); + my $cmdline = ; + $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines + close(CMDLINE); + error("Failed to get program name from $url\n") unless defined($cmdline); + $cmdline =~ s/\x00.+//; # Remove argv[1] and latters. + $cmdline =~ s!\n!!g; # Remove LFs. + return $cmdline; +} + +# Gee, curl's -L (--location) option isn't reliable at least +# with its 7.12.3 version. Curl will forget to post data if +# there is a redirection. This function is a workaround for +# curl. Redirection happens on borg hosts. +sub ResolveRedirectionForCurl { + my $url = shift; + my $command_line = "$URL_FETCHER --head '$url'"; + open(CMDLINE, "$command_line |") or error($command_line); + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + if (/^Location: (.*)/) { + $url = $1; + } + } + close(CMDLINE); + return $url; +} + +# Add a timeout flat to URL_FETCHER +sub AddFetchTimeout { + my $fetcher = shift; + my $timeout = shift; + if (defined($timeout)) { + if ($fetcher =~ m/\bcurl -s/) { + $fetcher .= sprintf(" --max-time %d", $timeout); + } elsif ($fetcher =~ m/\brpcget\b/) { + $fetcher .= sprintf(" --deadline=%d", $timeout); + } + } + return $fetcher; +} + +# Reads a symbol map from the file handle name given as $1, returning +# the resulting symbol map. Also processes variables relating to symbols. +# Currently, the only variable processed is 'binary=' which updates +# $main::prog to have the correct program name. +sub ReadSymbols { + my $in = shift; + my $map = {}; + while (<$in>) { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Removes all the leading zeroes from the symbols, see comment below. + if (m/^0x0*([0-9a-f]+)\s+(.+)/) { + $map->{$1} = $2; + } elsif (m/^---/) { + last; + } elsif (m/^([a-z][^=]*)=(.*)$/ ) { + my ($variable, $value) = ($1, $2); + for ($variable, $value) { + s/^\s+//; + s/\s+$//; + } + if ($variable eq "binary") { + if ($main::prog ne $UNKNOWN_BINARY && $main::prog ne $value) { + printf STDERR ("Warning: Mismatched binary name '%s', using '%s'.\n", + $main::prog, $value); + } + $main::prog = $value; + } else { + printf STDERR ("Ignoring unknown variable in symbols list: " . + "'%s' = '%s'\n", $variable, $value); + } + } + } + return $map; +} + +# Fetches and processes symbols to prepare them for use in the profile output +# code. If the optional 'symbol_map' arg is not given, fetches symbols from +# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols +# are assumed to have already been fetched into 'symbol_map' and are simply +# extracted and processed. +sub FetchSymbols { + my $pcset = shift; + my $symbol_map = shift; + + my %seen = (); + my @pcs = grep { !$seen{$_}++ } keys(%$pcset); # uniq + + if (!defined($symbol_map)) { + my $post_data = join("+", sort((map {"0x" . "$_"} @pcs))); + + open(POSTFILE, ">$main::tmpfile_sym"); + print POSTFILE $post_data; + close(POSTFILE); + + my $url = SymbolPageURL(); + + my $command_line; + if ($URL_FETCHER =~ m/\bcurl -s/) { + $url = ResolveRedirectionForCurl($url); + $command_line = "$URL_FETCHER -d '\@$main::tmpfile_sym' '$url'"; + } else { + $command_line = "$URL_FETCHER --post '$url' < '$main::tmpfile_sym'"; + } + # We use c++filt in case $SYMBOL_PAGE gives us mangled symbols. + my $cppfilt = $obj_tool_map{"c++filt"}; + open(SYMBOL, "$command_line | $cppfilt |") or error($command_line); + $symbol_map = ReadSymbols(*SYMBOL{IO}); + close(SYMBOL); + } + + my $symbols = {}; + foreach my $pc (@pcs) { + my $fullname; + # For 64 bits binaries, symbols are extracted with 8 leading zeroes. + # Then /symbol reads the long symbols in as uint64, and outputs + # the result with a "0x%08llx" format which get rid of the zeroes. + # By removing all the leading zeroes in both $pc and the symbols from + # /symbol, the symbols match and are retrievable from the map. + my $shortpc = $pc; + $shortpc =~ s/^0*//; + # Each line may have a list of names, which includes the function + # and also other functions it has inlined. They are separated + # (in PrintSymbolizedFile), by --, which is illegal in function names. + my $fullnames; + if (defined($symbol_map->{$shortpc})) { + $fullnames = $symbol_map->{$shortpc}; + } else { + $fullnames = "0x" . $pc; # Just use addresses + } + my $sym = []; + $symbols->{$pc} = $sym; + foreach my $fullname (split("--", $fullnames)) { + my $name = ShortFunctionName($fullname); + push(@{$sym}, $name, "?", $fullname); + } + } + return $symbols; +} + +sub BaseName { + my $file_name = shift; + $file_name =~ s!^.*/!!; # Remove directory name + return $file_name; +} + +sub MakeProfileBaseName { + my ($binary_name, $profile_name) = @_; + my ($host, $baseURL, $path) = ParseProfileURL($profile_name); + my $binary_shortname = BaseName($binary_name); + return sprintf("%s.%s.%s", + $binary_shortname, $main::op_time, $host); +} + +sub FetchDynamicProfile { + my $binary_name = shift; + my $profile_name = shift; + my $fetch_name_only = shift; + my $encourage_patience = shift; + + if (!IsProfileURL($profile_name)) { + return $profile_name; + } else { + my ($host, $baseURL, $path) = ParseProfileURL($profile_name); + if ($path eq "" || $path eq "/") { + # Missing type specifier defaults to cpu-profile + $path = $PROFILE_PAGE; + } + + my $profile_file = MakeProfileBaseName($binary_name, $profile_name); + + my $url = "$baseURL$path"; + my $fetch_timeout = undef; + if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE/) { + if ($path =~ m/[?]/) { + $url .= "&"; + } else { + $url .= "?"; + } + $url .= sprintf("seconds=%d", $main::opt_seconds); + $fetch_timeout = $main::opt_seconds * 1.01 + 60; + } else { + # For non-CPU profiles, we add a type-extension to + # the target profile file name. + my $suffix = $path; + $suffix =~ s,/,.,g; + $profile_file .= $suffix; + } + + my $profile_dir = $ENV{"PPROF_TMPDIR"} || ($ENV{HOME} . "/pprof"); + if (! -d $profile_dir) { + mkdir($profile_dir) + || die("Unable to create profile directory $profile_dir: $!\n"); + } + my $tmp_profile = "$profile_dir/.tmp.$profile_file"; + my $real_profile = "$profile_dir/$profile_file"; + + if ($fetch_name_only > 0) { + return $real_profile; + } + + my $fetcher = AddFetchTimeout($URL_FETCHER, $fetch_timeout); + my $cmd = "$fetcher '$url' > '$tmp_profile'"; + if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE|$CENSUSPROFILE_PAGE/){ + print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; + if ($encourage_patience) { + print STDERR "Be patient...\n"; + } + } else { + print STDERR "Fetching $path profile from $url to\n ${real_profile}\n"; + } + + (system($cmd) == 0) || error("Failed to get profile: $cmd: $!\n"); + (system("mv $tmp_profile $real_profile") == 0) || error("Unable to rename profile\n"); + print STDERR "Wrote profile to $real_profile\n"; + $main::collected_profile = $real_profile; + return $main::collected_profile; + } +} + +# Collect profiles in parallel +sub FetchDynamicProfiles { + my $items = scalar(@main::pfile_args); + my $levels = log($items) / log(2); + + if ($items == 1) { + $main::profile_files[0] = FetchDynamicProfile($main::prog, $main::pfile_args[0], 0, 1); + } else { + # math rounding issues + if ((2 ** $levels) < $items) { + $levels++; + } + my $count = scalar(@main::pfile_args); + for (my $i = 0; $i < $count; $i++) { + $main::profile_files[$i] = FetchDynamicProfile($main::prog, $main::pfile_args[$i], 1, 0); + } + print STDERR "Fetching $count profiles, Be patient...\n"; + FetchDynamicProfilesRecurse($levels, 0, 0); + $main::collected_profile = join(" \\\n ", @main::profile_files); + } +} + +# Recursively fork a process to get enough processes +# collecting profiles +sub FetchDynamicProfilesRecurse { + my $maxlevel = shift; + my $level = shift; + my $position = shift; + + if (my $pid = fork()) { + $position = 0 | ($position << 1); + TryCollectProfile($maxlevel, $level, $position); + wait; + } else { + $position = 1 | ($position << 1); + TryCollectProfile($maxlevel, $level, $position); + cleanup(); + exit(0); + } +} + +# Collect a single profile +sub TryCollectProfile { + my $maxlevel = shift; + my $level = shift; + my $position = shift; + + if ($level >= ($maxlevel - 1)) { + if ($position < scalar(@main::pfile_args)) { + FetchDynamicProfile($main::prog, $main::pfile_args[$position], 0, 0); + } + } else { + FetchDynamicProfilesRecurse($maxlevel, $level+1, $position); + } +} + +##### Parsing code ##### + +# Provide a small streaming-read module to handle very large +# cpu-profile files. Stream in chunks along a sliding window. +# Provides an interface to get one 'slot', correctly handling +# endian-ness differences. A slot is one 32-bit or 64-bit word +# (depending on the input profile). We tell endianness and bit-size +# for the profile by looking at the first 8 bytes: in cpu profiles, +# the second slot is always 3 (we'll accept anything that's not 0). +BEGIN { + package CpuProfileStream; + + sub new { + my ($class, $file, $fname) = @_; + my $self = { file => $file, + base => 0, + stride => 512 * 1024, # must be a multiple of bitsize/8 + slots => [], + unpack_code => "", # N for big-endian, V for little + perl_is_64bit => 1, # matters if profile is 64-bit + }; + bless $self, $class; + # Let unittests adjust the stride + if ($main::opt_test_stride > 0) { + $self->{stride} = $main::opt_test_stride; + } + # Read the first two slots to figure out bitsize and endianness. + my $slots = $self->{slots}; + my $str; + read($self->{file}, $str, 8); + # Set the global $address_length based on what we see here. + # 8 is 32-bit (8 hexadecimal chars); 16 is 64-bit (16 hexadecimal chars). + $address_length = ($str eq (chr(0)x8)) ? 16 : 8; + if ($address_length == 8) { + if (substr($str, 6, 2) eq chr(0)x2) { + $self->{unpack_code} = 'V'; # Little-endian. + } elsif (substr($str, 4, 2) eq chr(0)x2) { + $self->{unpack_code} = 'N'; # Big-endian + } else { + ::error("$fname: header size >= 2**16\n"); + } + @$slots = unpack($self->{unpack_code} . "*", $str); + } else { + # If we're a 64-bit profile, check if we're a 64-bit-capable + # perl. Otherwise, each slot will be represented as a float + # instead of an int64, losing precision and making all the + # 64-bit addresses wrong. We won't complain yet, but will + # later if we ever see a value that doesn't fit in 32 bits. + my $has_q = 0; + eval { $has_q = pack("Q", "1") ? 1 : 1; }; + if (!$has_q) { + $self->{perl_is_64bit} = 0; + } + read($self->{file}, $str, 8); + if (substr($str, 4, 4) eq chr(0)x4) { + # We'd love to use 'Q', but it's a) not universal, b) not endian-proof. + $self->{unpack_code} = 'V'; # Little-endian. + } elsif (substr($str, 0, 4) eq chr(0)x4) { + $self->{unpack_code} = 'N'; # Big-endian + } else { + ::error("$fname: header size >= 2**32\n"); + } + my @pair = unpack($self->{unpack_code} . "*", $str); + # Since we know one of the pair is 0, it's fine to just add them. + @$slots = (0, $pair[0] + $pair[1]); + } + return $self; + } + + # Load more data when we access slots->get(X) which is not yet in memory. + sub overflow { + my ($self) = @_; + my $slots = $self->{slots}; + $self->{base} += $#$slots + 1; # skip over data we're replacing + my $str; + read($self->{file}, $str, $self->{stride}); + if ($address_length == 8) { # the 32-bit case + # This is the easy case: unpack provides 32-bit unpacking primitives. + @$slots = unpack($self->{unpack_code} . "*", $str); + } else { + # We need to unpack 32 bits at a time and combine. + my @b32_values = unpack($self->{unpack_code} . "*", $str); + my @b64_values = (); + for (my $i = 0; $i < $#b32_values; $i += 2) { + # TODO(csilvers): if this is a 32-bit perl, the math below + # could end up in a too-large int, which perl will promote + # to a double, losing necessary precision. Deal with that. + # Right now, we just die. + my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); + if ($self->{unpack_code} eq 'N') { # big-endian + ($lo, $hi) = ($hi, $lo); + } + my $value = $lo + $hi * (2**32); + if (!$self->{perl_is_64bit} && # check value is exactly represented + (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { + ::error("Need a 64-bit perl to process this 64-bit profile.\n"); + } + push(@b64_values, $value); + } + @$slots = @b64_values; + } + } + + # Access the i-th long in the file (logically), or -1 at EOF. + sub get { + my ($self, $idx) = @_; + my $slots = $self->{slots}; + while ($#$slots >= 0) { + if ($idx < $self->{base}) { + # The only time we expect a reference to $slots[$i - something] + # after referencing $slots[$i] is reading the very first header. + # Since $stride > |header|, that shouldn't cause any lookback + # errors. And everything after the header is sequential. + print STDERR "Unexpected look-back reading CPU profile"; + return -1; # shrug, don't know what better to return + } elsif ($idx > $self->{base} + $#$slots) { + $self->overflow(); + } else { + return $slots->[$idx - $self->{base}]; + } + } + # If we get here, $slots is [], which means we've reached EOF + return -1; # unique since slots is supposed to hold unsigned numbers + } +} + +# Reads the top, 'header' section of a profile, and returns the last +# line of the header, commonly called a 'header line'. The header +# section of a profile consists of zero or more 'command' lines that +# are instructions to pprof, which pprof executes when reading the +# header. All 'command' lines start with a %. After the command +# lines is the 'header line', which is a profile-specific line that +# indicates what type of profile it is, and perhaps other global +# information about the profile. For instance, here's a header line +# for a heap profile: +# heap profile: 53: 38236 [ 5525: 1284029] @ heapprofile +# For historical reasons, the CPU profile does not contain a text- +# readable header line. If the profile looks like a CPU profile, +# this function returns "". If no header line could be found, this +# function returns undef. +# +# The following commands are recognized: +# %warn -- emit the rest of this line to stderr, prefixed by 'WARNING:' +# +# The input file should be in binmode. +sub ReadProfileHeader { + local *PROFILE = shift; + my $firstchar = ""; + my $line = ""; + read(PROFILE, $firstchar, 1); + seek(PROFILE, -1, 1); # unread the firstchar + if ($firstchar !~ /[[:print:]]/) { # is not a text character + return ""; + } + while (defined($line = )) { + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + if ($line =~ /^%warn\s+(.*)/) { # 'warn' command + # Note this matches both '%warn blah\n' and '%warn\n'. + print STDERR "WARNING: $1\n"; # print the rest of the line + } elsif ($line =~ /^%/) { + print STDERR "Ignoring unknown command from profile header: $line"; + } else { + # End of commands, must be the header line. + return $line; + } + } + return undef; # got to EOF without seeing a header line +} + +sub IsSymbolizedProfileFile { + my $file_name = shift; + if (!(-e $file_name) || !(-r $file_name)) { + return 0; + } + # Check if the file contains a symbol-section marker. + open(TFILE, "<$file_name"); + binmode TFILE; + my $firstline = ReadProfileHeader(*TFILE); + close(TFILE); + if (!$firstline) { + return 0; + } + $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $symbol_marker = $&; + return $firstline =~ /^--- *$symbol_marker/; +} + +# Parse profile generated by common/profiler.cc and return a reference +# to a map: +# $result->{version} Version number of profile file +# $result->{period} Sampling period (in microseconds) +# $result->{profile} Profile object +# $result->{map} Memory map info from profile +# $result->{pcs} Hash of all PC values seen, key is hex address +sub ReadProfile { + my $prog = shift; + my $fname = shift; + my $result; # return value + + $CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $contention_marker = $&; + $GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $growth_marker = $&; + $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $symbol_marker = $&; + $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash + my $profile_marker = $&; + + # Look at first line to see if it is a heap or a CPU profile. + # CPU profile may start with no header at all, and just binary data + # (starting with \0\0\0\0) -- in that case, don't try to read the + # whole firstline, since it may be gigabytes(!) of data. + open(PROFILE, "<$fname") || error("$fname: $!\n"); + binmode PROFILE; # New perls do UTF-8 processing + my $header = ReadProfileHeader(*PROFILE); + if (!defined($header)) { # means "at EOF" + error("Profile is empty.\n"); + } + + my $symbols; + if ($header =~ m/^--- *$symbol_marker/o) { + # Verify that the user asked for a symbolized profile + if (!$main::use_symbolized_profile) { + # we have both a binary and symbolized profiles, abort + error("FATAL ERROR: Symbolized profile\n $fname\ncannot be used with " . + "a binary arg. Try again without passing\n $prog\n"); + } + # Read the symbol section of the symbolized profile file. + $symbols = ReadSymbols(*PROFILE{IO}); + # Read the next line to get the header for the remaining profile. + $header = ReadProfileHeader(*PROFILE) || ""; + } + + $main::profile_type = ''; + if ($header =~ m/^heap profile:.*$growth_marker/o) { + $main::profile_type = 'growth'; + $result = ReadHeapProfile($prog, *PROFILE, $header); + } elsif ($header =~ m/^heap profile:/) { + $main::profile_type = 'heap'; + $result = ReadHeapProfile($prog, *PROFILE, $header); + } elsif ($header =~ m/^--- *$contention_marker/o) { + $main::profile_type = 'contention'; + $result = ReadSynchProfile($prog, *PROFILE); + } elsif ($header =~ m/^--- *Stacks:/) { + print STDERR + "Old format contention profile: mistakenly reports " . + "condition variable signals as lock contentions.\n"; + $main::profile_type = 'contention'; + $result = ReadSynchProfile($prog, *PROFILE); + } elsif ($header =~ m/^--- *$profile_marker/) { + # the binary cpu profile data starts immediately after this line + $main::profile_type = 'cpu'; + $result = ReadCPUProfile($prog, $fname, *PROFILE); + } else { + if (defined($symbols)) { + # a symbolized profile contains a format we don't recognize, bail out + error("$fname: Cannot recognize profile section after symbols.\n"); + } + # no ascii header present -- must be a CPU profile + $main::profile_type = 'cpu'; + $result = ReadCPUProfile($prog, $fname, *PROFILE); + } + + close(PROFILE); + + # if we got symbols along with the profile, return those as well + if (defined($symbols)) { + $result->{symbols} = $symbols; + } + + return $result; +} + +# Subtract one from caller pc so we map back to call instr. +# However, don't do this if we're reading a symbolized profile +# file, in which case the subtract-one was done when the file +# was written. +# +# We apply the same logic to all readers, though ReadCPUProfile uses an +# independent implementation. +sub FixCallerAddresses { + my $stack = shift; + if ($main::use_symbolized_profile) { + return $stack; + } else { + $stack =~ /(\s)/; + my $delimiter = $1; + my @addrs = split(' ', $stack); + my @fixedaddrs; + $#fixedaddrs = $#addrs; + if ($#addrs >= 0) { + $fixedaddrs[0] = $addrs[0]; + } + for (my $i = 1; $i <= $#addrs; $i++) { + $fixedaddrs[$i] = AddressSub($addrs[$i], "0x1"); + } + return join $delimiter, @fixedaddrs; + } +} + +# CPU profile reader +sub ReadCPUProfile { + my $prog = shift; + my $fname = shift; # just used for logging + local *PROFILE = shift; + my $version; + my $period; + my $i; + my $profile = {}; + my $pcs = {}; + + # Parse string into array of slots. + my $slots = CpuProfileStream->new(*PROFILE, $fname); + + # Read header. The current header version is a 5-element structure + # containing: + # 0: header count (always 0) + # 1: header "words" (after this one: 3) + # 2: format version (0) + # 3: sampling period (usec) + # 4: unused padding (always 0) + if ($slots->get(0) != 0 ) { + error("$fname: not a profile file, or old format profile file\n"); + } + $i = 2 + $slots->get(1); + $version = $slots->get(2); + $period = $slots->get(3); + # Do some sanity checking on these header values. + if ($version > (2**32) || $period > (2**32) || $i > (2**32) || $i < 5) { + error("$fname: not a profile file, or corrupted profile file\n"); + } + + # Parse profile + while ($slots->get($i) != -1) { + my $n = $slots->get($i++); + my $d = $slots->get($i++); + if ($d > (2**16)) { # TODO(csilvers): what's a reasonable max-stack-depth? + my $addr = sprintf("0%o", $i * ($address_length == 8 ? 4 : 8)); + print STDERR "At index $i (address $addr):\n"; + error("$fname: stack trace depth >= 2**32\n"); + } + if ($slots->get($i) == 0) { + # End of profile data marker + $i += $d; + last; + } + + # Make key out of the stack entries + my @k = (); + for (my $j = 0; $j < $d; $j++) { + my $pc = $slots->get($i+$j); + # Subtract one from caller pc so we map back to call instr. + # However, don't do this if we're reading a symbolized profile + # file, in which case the subtract-one was done when the file + # was written. + if ($j > 0 && !$main::use_symbolized_profile) { + $pc--; + } + $pc = sprintf("%0*x", $address_length, $pc); + $pcs->{$pc} = 1; + push @k, $pc; + } + + AddEntry($profile, (join "\n", @k), $n); + $i += $d; + } + + # Parse map + my $map = ''; + seek(PROFILE, $i * 4, 0); + read(PROFILE, $map, (stat PROFILE)[7]); + + my $r = {}; + $r->{version} = $version; + $r->{period} = $period; + $r->{profile} = $profile; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + + return $r; +} + +sub ReadHeapProfile { + my $prog = shift; + local *PROFILE = shift; + my $header = shift; + + my $index = 1; + if ($main::opt_inuse_space) { + $index = 1; + } elsif ($main::opt_inuse_objects) { + $index = 0; + } elsif ($main::opt_alloc_space) { + $index = 3; + } elsif ($main::opt_alloc_objects) { + $index = 2; + } + + # Find the type of this profile. The header line looks like: + # heap profile: 1246: 8800744 [ 1246: 8800744] @ /266053 + # There are two pairs , the first inuse objects/space, and the + # second allocated objects/space. This is followed optionally by a profile + # type, and if that is present, optionally by a sampling frequency. + # For remote heap profiles (v1): + # The interpretation of the sampling frequency is that the profiler, for + # each sample, calculates a uniformly distributed random integer less than + # the given value, and records the next sample after that many bytes have + # been allocated. Therefore, the expected sample interval is half of the + # given frequency. By default, if not specified, the expected sample + # interval is 128KB. Only remote-heap-page profiles are adjusted for + # sample size. + # For remote heap profiles (v2): + # The sampling frequency is the rate of a Poisson process. This means that + # the probability of sampling an allocation of size X with sampling rate Y + # is 1 - exp(-X/Y) + # For version 2, a typical header line might look like this: + # heap profile: 1922: 127792360 [ 1922: 127792360] @ _v2/524288 + # the trailing number (524288) is the sampling rate. (Version 1 showed + # double the 'rate' here) + my $sampling_algorithm = 0; + my $sample_adjustment = 0; + chomp($header); + my $type = "unknown"; + if ($header =~ m"^heap profile:\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\](\s*@\s*([^/]*)(/(\d+))?)?") { + if (defined($6) && ($6 ne '')) { + $type = $6; + my $sample_period = $8; + # $type is "heapprofile" for profiles generated by the + # heap-profiler, and either "heap" or "heap_v2" for profiles + # generated by sampling directly within tcmalloc. It can also + # be "growth" for heap-growth profiles. The first is typically + # found for profiles generated locally, and the others for + # remote profiles. + if (($type eq "heapprofile") || ($type !~ /heap/) ) { + # No need to adjust for the sampling rate with heap-profiler-derived data + $sampling_algorithm = 0; + } elsif ($type =~ /_v2/) { + $sampling_algorithm = 2; # version 2 sampling + if (defined($sample_period) && ($sample_period ne '')) { + $sample_adjustment = int($sample_period); + } + } else { + $sampling_algorithm = 1; # version 1 sampling + if (defined($sample_period) && ($sample_period ne '')) { + $sample_adjustment = int($sample_period)/2; + } + } + } else { + # We detect whether or not this is a remote-heap profile by checking + # that the total-allocated stats ($n2,$s2) are exactly the + # same as the in-use stats ($n1,$s1). It is remotely conceivable + # that a non-remote-heap profile may pass this check, but it is hard + # to imagine how that could happen. + # In this case it's so old it's guaranteed to be remote-heap version 1. + my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); + if (($n1 == $n2) && ($s1 == $s2)) { + # This is likely to be a remote-heap based sample profile + $sampling_algorithm = 1; + } + } + } + + if ($sampling_algorithm > 0) { + # For remote-heap generated profiles, adjust the counts and sizes to + # account for the sample rate (we sample once every 128KB by default). + if ($sample_adjustment == 0) { + # Turn on profile adjustment. + $sample_adjustment = 128*1024; + print STDERR "Adjusting heap profiles for 1-in-128KB sampling rate\n"; + } else { + printf STDERR ("Adjusting heap profiles for 1-in-%d sampling rate\n", + $sample_adjustment); + } + if ($sampling_algorithm > 1) { + # We don't bother printing anything for the original version (version 1) + printf STDERR "Heap version $sampling_algorithm\n"; + } + } + + my $profile = {}; + my $pcs = {}; + my $map = ""; + + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + if (/^MAPPED_LIBRARIES:/) { + # Read the /proc/self/maps data + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + $map .= $_; + } + last; + } + + if (/^--- Memory map:/) { + # Read /proc/self/maps data as formatted by DumpAddressMap() + my $buildvar = ""; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Parse "build=" specification if supplied + if (m/^\s*build=(.*)\n/) { + $buildvar = $1; + } + + # Expand "$build" variable if available + $_ =~ s/\$build\b/$buildvar/g; + + $map .= $_; + } + last; + } + + # Read entry of the form: + # : [: ] @ a1 a2 a3 ... an + s/^\s*//; + s/\s*$//; + if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { + my $stack = $5; + my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); + + if ($sample_adjustment) { + if ($sampling_algorithm == 2) { + # Remote-heap version 2 + # The sampling frequency is the rate of a Poisson process. + # This means that the probability of sampling an allocation of + # size X with sampling rate Y is 1 - exp(-X/Y) + if ($n1 != 0) { + my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n1 *= $scale_factor; + $s1 *= $scale_factor; + } + if ($n2 != 0) { + my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n2 *= $scale_factor; + $s2 *= $scale_factor; + } + } else { + # Remote-heap version 1 + my $ratio; + $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + if ($ratio < 1) { + $n1 /= $ratio; + $s1 /= $ratio; + } + $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + if ($ratio < 1) { + $n2 /= $ratio; + $s2 /= $ratio; + } + } + } + + my @counts = ($n1, $s1, $n2, $s2); + AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); + } + } + + my $r = {}; + $r->{version} = "heap"; + $r->{period} = 1; + $r->{profile} = $profile; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + return $r; +} + +sub ReadSynchProfile { + my $prog = shift; + local *PROFILE = shift; + my $header = shift; + + my $map = ''; + my $profile = {}; + my $pcs = {}; + my $sampling_period = 1; + my $cyclespernanosec = 2.8; # Default assumption for old binaries + my $seen_clockrate = 0; + my $line; + + my $index = 0; + if ($main::opt_total_delay) { + $index = 0; + } elsif ($main::opt_contentions) { + $index = 1; + } elsif ($main::opt_mean_delay) { + $index = 2; + } + + while ( $line = ) { + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + if ( $line =~ /^\s*(\d+)\s+(\d+) \@\s*(.*?)\s*$/ ) { + my ($cycles, $count, $stack) = ($1, $2, $3); + + # Convert cycles to nanoseconds + $cycles /= $cyclespernanosec; + + # Adjust for sampling done by application + $cycles *= $sampling_period; + $count *= $sampling_period; + + my @values = ($cycles, $count, $cycles / $count); + AddEntries($profile, $pcs, FixCallerAddresses($stack), $values[$index]); + + } elsif ( $line =~ /^(slow release).*thread \d+ \@\s*(.*?)\s*$/ || + $line =~ /^\s*(\d+) \@\s*(.*?)\s*$/ ) { + my ($cycles, $stack) = ($1, $2); + if ($cycles !~ /^\d+$/) { + next; + } + + # Convert cycles to nanoseconds + $cycles /= $cyclespernanosec; + + # Adjust for sampling done by application + $cycles *= $sampling_period; + + AddEntries($profile, $pcs, FixCallerAddresses($stack), $cycles); + + } elsif ( $line =~ m/^([a-z][^=]*)=(.*)$/ ) { + my ($variable, $value) = ($1,$2); + for ($variable, $value) { + s/^\s+//; + s/\s+$//; + } + if ($variable eq "cycles/second") { + $cyclespernanosec = $value / 1e9; + $seen_clockrate = 1; + } elsif ($variable eq "sampling period") { + $sampling_period = $value; + } elsif ($variable eq "ms since reset") { + # Currently nothing is done with this value in pprof + # So we just silently ignore it for now + } elsif ($variable eq "discarded samples") { + # Currently nothing is done with this value in pprof + # So we just silently ignore it for now + } else { + printf STDERR ("Ignoring unnknown variable in /contention output: " . + "'%s' = '%s'\n",$variable,$value); + } + } else { + # Memory map entry + $map .= $line; + } + } + + if (!$seen_clockrate) { + printf STDERR ("No cycles/second entry in profile; Guessing %.1f GHz\n", + $cyclespernanosec); + } + + my $r = {}; + $r->{version} = 0; + $r->{period} = $sampling_period; + $r->{profile} = $profile; + $r->{libs} = ParseLibraries($prog, $map, $pcs); + $r->{pcs} = $pcs; + return $r; +} + +# Given a hex value in the form "0x1abcd" return "0001abcd" or +# "000000000001abcd", depending on the current address length. +# There's probably a more idiomatic (or faster) way to do this... +sub HexExtend { + my $addr = shift; + + $addr =~ s/^0x//; + + if (length $addr > $address_length) { + printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + } + + return substr("000000000000000".$addr, -$address_length); +} + +##### Symbol extraction ##### + +# Aggressively search the lib_prefix values for the given library +# If all else fails, just return the name of the library unmodified. +# If the lib_prefix is "/my/path,/other/path" and $file is "/lib/dir/mylib.so" +# it will search the following locations in this order, until it finds a file: +# /my/path/lib/dir/mylib.so +# /other/path/lib/dir/mylib.so +# /my/path/dir/mylib.so +# /other/path/dir/mylib.so +# /my/path/mylib.so +# /other/path/mylib.so +# /lib/dir/mylib.so (returned as last resort) +sub FindLibrary { + my $file = shift; + my $suffix = $file; + + # Search for the library as described above + do { + foreach my $prefix (@prefix_list) { + my $fullpath = $prefix . $suffix; + if (-e $fullpath) { + return $fullpath; + } + } + } while ($suffix =~ s|^/[^/]+/|/|); + return $file; +} + +# Return path to library with debugging symbols. +# For libc libraries, the copy in /usr/lib/debug contains debugging symbols +sub DebuggingLibrary { + my $file = shift; + if ($file =~ m|^/| && -f "/usr/lib/debug$file") { + return "/usr/lib/debug$file"; + } + return undef; +} + +# Parse text section header of a library using objdump +sub ParseTextSectionHeaderFromObjdump { + my $lib = shift; + + my $size = undef; + my $vma; + my $file_offset; + # Get objdump output from the library file to figure out how to + # map between mapped addresses and addresses in the library. + my $objdump = $obj_tool_map{"objdump"}; + open(OBJDUMP, "$objdump -h $lib |") + || error("$objdump $lib: $!\n"); + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + # Idx Name Size VMA LMA File off Algn + # 10 .text 00104b2c 420156f0 420156f0 000156f0 2**4 + # For 64-bit objects, VMA and LMA will be 16 hex digits, size and file + # offset may still be 8. But AddressSub below will still handle that. + my @x = split; + if (($#x >= 6) && ($x[1] eq '.text')) { + $size = $x[2]; + $vma = $x[3]; + $file_offset = $x[5]; + last; + } + } + close(OBJDUMP); + + if (!defined($size)) { + return undef; + } + + my $r = {}; + $r->{size} = $size; + $r->{vma} = $vma; + $r->{file_offset} = $file_offset; + + return $r; +} + +# Parse text section header of a library using otool (on OS X) +sub ParseTextSectionHeaderFromOtool { + my $lib = shift; + + my $size = undef; + my $vma = undef; + my $file_offset = undef; + # Get otool output from the library file to figure out how to + # map between mapped addresses and addresses in the library. + my $otool = $obj_tool_map{"otool"}; + open(OTOOL, "$otool -l $lib |") + || error("$otool $lib: $!\n"); + my $cmd = ""; + my $sectname = ""; + my $segname = ""; + foreach my $line () { + $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines + # Load command <#> + # cmd LC_SEGMENT + # [...] + # Section + # sectname __text + # segname __TEXT + # addr 0x000009f8 + # size 0x00018b9e + # offset 2552 + # align 2^2 (4) + # We will need to strip off the leading 0x from the hex addresses, + # and convert the offset into hex. + if ($line =~ /Load command/) { + $cmd = ""; + $sectname = ""; + $segname = ""; + } elsif ($line =~ /Section/) { + $sectname = ""; + $segname = ""; + } elsif ($line =~ /cmd (\w+)/) { + $cmd = $1; + } elsif ($line =~ /sectname (\w+)/) { + $sectname = $1; + } elsif ($line =~ /segname (\w+)/) { + $segname = $1; + } elsif (!(($cmd eq "LC_SEGMENT" || $cmd eq "LC_SEGMENT_64") && + $sectname eq "__text" && + $segname eq "__TEXT")) { + next; + } elsif ($line =~ /\baddr 0x([0-9a-fA-F]+)/) { + $vma = $1; + } elsif ($line =~ /\bsize 0x([0-9a-fA-F]+)/) { + $size = $1; + } elsif ($line =~ /\boffset ([0-9]+)/) { + $file_offset = sprintf("%016x", $1); + } + if (defined($vma) && defined($size) && defined($file_offset)) { + last; + } + } + close(OTOOL); + + if (!defined($vma) || !defined($size) || !defined($file_offset)) { + return undef; + } + + my $r = {}; + $r->{size} = $size; + $r->{vma} = $vma; + $r->{file_offset} = $file_offset; + + return $r; +} + +sub ParseTextSectionHeader { + # obj_tool_map("otool") is only defined if we're in a Mach-O environment + if (defined($obj_tool_map{"otool"})) { + my $r = ParseTextSectionHeaderFromOtool(@_); + if (defined($r)){ + return $r; + } + } + # If otool doesn't work, or we don't have it, fall back to objdump + return ParseTextSectionHeaderFromObjdump(@_); +} + +# Split /proc/pid/maps dump into a list of libraries +sub ParseLibraries { + return if $main::use_symbol_page; # We don't need libraries info. + my $prog = shift; + my $map = shift; + my $pcs = shift; + + my $result = []; + my $h = "[a-f0-9]+"; + my $zero_offset = HexExtend("0"); + + my $buildvar = ""; + foreach my $l (split("\n", $map)) { + if ($l =~ m/^\s*build=(.*)$/) { + $buildvar = $1; + } + + my $start; + my $finish; + my $offset; + my $lib; + if ($l =~ /^($h)-($h)\s+..x.\s+($h)\s+\S+:\S+\s+\d+\s+(\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?)$/i) { + # Full line from /proc/self/maps. Example: + # 40000000-40015000 r-xp 00000000 03:01 12845071 /lib/ld-2.3.2.so + $start = HexExtend($1); + $finish = HexExtend($2); + $offset = HexExtend($3); + $lib = $4; + $lib =~ s|\\|/|g; # turn windows-style paths into unix-style paths + } elsif ($l =~ /^\s*($h)-($h):\s*(\S+\.so(\.\d+)*)/) { + # Cooked line from DumpAddressMap. Example: + # 40000000-40015000: /lib/ld-2.3.2.so + $start = HexExtend($1); + $finish = HexExtend($2); + $offset = $zero_offset; + $lib = $3; + } else { + next; + } + + # Expand "$build" variable if available + $lib =~ s/\$build\b/$buildvar/g; + + $lib = FindLibrary($lib); + + # Check for pre-relocated libraries, which use pre-relocated symbol tables + # and thus require adjusting the offset that we'll use to translate + # VM addresses into symbol table addresses. + # Only do this if we're not going to fetch the symbol table from a + # debugging copy of the library. + if (!DebuggingLibrary($lib)) { + my $text = ParseTextSectionHeader($lib); + if (defined($text)) { + my $vma_offset = AddressSub($text->{vma}, $text->{file_offset}); + $offset = AddressAdd($offset, $vma_offset); + } + } + + push(@{$result}, [$lib, $start, $finish, $offset]); + } + + # Append special entry for additional library (not relocated) + if ($main::opt_lib ne "") { + my $text = ParseTextSectionHeader($main::opt_lib); + if (defined($text)) { + my $start = $text->{vma}; + my $finish = AddressAdd($start, $text->{size}); + + push(@{$result}, [$main::opt_lib, $start, $finish, $start]); + } + } + + # Append special entry for the main program. This covers + # 0..max_pc_value_seen, so that we assume pc values not found in one + # of the library ranges will be treated as coming from the main + # program binary. + my $min_pc = HexExtend("0"); + my $max_pc = $min_pc; # find the maximal PC value in any sample + foreach my $pc (keys(%{$pcs})) { + if (HexExtend($pc) gt $max_pc) { $max_pc = HexExtend($pc); } + } + push(@{$result}, [$prog, $min_pc, $max_pc, $zero_offset]); + + return $result; +} + +# Add two hex addresses of length $address_length. +# Run pprof --test for unit test if this is changed. +sub AddressAdd { + my $addr1 = shift; + my $addr2 = shift; + my $sum; + + if ($address_length == 8) { + # Perl doesn't cope with wraparound arithmetic, so do it explicitly: + $sum = (hex($addr1)+hex($addr2)) % (0x10000000 * 16); + return sprintf("%08x", $sum); + + } else { + # Do the addition in 7-nibble chunks to trivialize carry handling. + + if ($main::opt_debug and $main::opt_test) { + print STDERR "AddressAdd $addr1 + $addr2 = "; + } + + my $a1 = substr($addr1,-7); + $addr1 = substr($addr1,0,-7); + my $a2 = substr($addr2,-7); + $addr2 = substr($addr2,0,-7); + $sum = hex($a1) + hex($a2); + my $c = 0; + if ($sum > 0xfffffff) { + $c = 1; + $sum -= 0x10000000; + } + my $r = sprintf("%07x", $sum); + + $a1 = substr($addr1,-7); + $addr1 = substr($addr1,0,-7); + $a2 = substr($addr2,-7); + $addr2 = substr($addr2,0,-7); + $sum = hex($a1) + hex($a2) + $c; + $c = 0; + if ($sum > 0xfffffff) { + $c = 1; + $sum -= 0x10000000; + } + $r = sprintf("%07x", $sum) . $r; + + $sum = hex($addr1) + hex($addr2) + $c; + if ($sum > 0xff) { $sum -= 0x100; } + $r = sprintf("%02x", $sum) . $r; + + if ($main::opt_debug and $main::opt_test) { print STDERR "$r\n"; } + + return $r; + } +} + + +# Subtract two hex addresses of length $address_length. +# Run pprof --test for unit test if this is changed. +sub AddressSub { + my $addr1 = shift; + my $addr2 = shift; + my $diff; + + if ($address_length == 8) { + # Perl doesn't cope with wraparound arithmetic, so do it explicitly: + $diff = (hex($addr1)-hex($addr2)) % (0x10000000 * 16); + return sprintf("%08x", $diff); + + } else { + # Do the addition in 7-nibble chunks to trivialize borrow handling. + # if ($main::opt_debug) { print STDERR "AddressSub $addr1 - $addr2 = "; } + + my $a1 = hex(substr($addr1,-7)); + $addr1 = substr($addr1,0,-7); + my $a2 = hex(substr($addr2,-7)); + $addr2 = substr($addr2,0,-7); + my $b = 0; + if ($a2 > $a1) { + $b = 1; + $a1 += 0x10000000; + } + $diff = $a1 - $a2; + my $r = sprintf("%07x", $diff); + + $a1 = hex(substr($addr1,-7)); + $addr1 = substr($addr1,0,-7); + $a2 = hex(substr($addr2,-7)) + $b; + $addr2 = substr($addr2,0,-7); + $b = 0; + if ($a2 > $a1) { + $b = 1; + $a1 += 0x10000000; + } + $diff = $a1 - $a2; + $r = sprintf("%07x", $diff) . $r; + + $a1 = hex($addr1); + $a2 = hex($addr2) + $b; + if ($a2 > $a1) { $a1 += 0x100; } + $diff = $a1 - $a2; + $r = sprintf("%02x", $diff) . $r; + + # if ($main::opt_debug) { print STDERR "$r\n"; } + + return $r; + } +} + +# Increment a hex addresses of length $address_length. +# Run pprof --test for unit test if this is changed. +sub AddressInc { + my $addr = shift; + my $sum; + + if ($address_length == 8) { + # Perl doesn't cope with wraparound arithmetic, so do it explicitly: + $sum = (hex($addr)+1) % (0x10000000 * 16); + return sprintf("%08x", $sum); + + } else { + # Do the addition in 7-nibble chunks to trivialize carry handling. + # We are always doing this to step through the addresses in a function, + # and will almost never overflow the first chunk, so we check for this + # case and exit early. + + # if ($main::opt_debug) { print STDERR "AddressInc $addr1 = "; } + + my $a1 = substr($addr,-7); + $addr = substr($addr,0,-7); + $sum = hex($a1) + 1; + my $r = sprintf("%07x", $sum); + if ($sum <= 0xfffffff) { + $r = $addr . $r; + # if ($main::opt_debug) { print STDERR "$r\n"; } + return HexExtend($r); + } else { + $r = "0000000"; + } + + $a1 = substr($addr,-7); + $addr = substr($addr,0,-7); + $sum = hex($a1) + 1; + $r = sprintf("%07x", $sum) . $r; + if ($sum <= 0xfffffff) { + $r = $addr . $r; + # if ($main::opt_debug) { print STDERR "$r\n"; } + return HexExtend($r); + } else { + $r = "00000000000000"; + } + + $sum = hex($addr) + 1; + if ($sum > 0xff) { $sum -= 0x100; } + $r = sprintf("%02x", $sum) . $r; + + # if ($main::opt_debug) { print STDERR "$r\n"; } + return $r; + } +} + +# Extract symbols for all PC values found in profile +sub ExtractSymbols { + my $libs = shift; + my $pcset = shift; + + my $symbols = {}; + + # Map each PC value to the containing library. To make this faster, + # we sort libraries by their starting pc value (highest first), and + # advance through the libraries as we advance the pc. Sometimes the + # addresses of libraries may overlap with the addresses of the main + # binary, so to make sure the libraries 'win', we iterate over the + # libraries in reverse order (which assumes the binary doesn't start + # in the middle of a library, which seems a fair assumption). + my @pcs = (sort { $a cmp $b } keys(%{$pcset})); # pcset is 0-extended strings + foreach my $lib (sort {$b->[1] cmp $a->[1]} @{$libs}) { + my $libname = $lib->[0]; + my $start = $lib->[1]; + my $finish = $lib->[2]; + my $offset = $lib->[3]; + + # Get list of pcs that belong in this library. + my $contained = []; + my ($start_pc_index, $finish_pc_index); + # Find smallest finish_pc_index such that $finish < $pc[$finish_pc_index]. + for ($finish_pc_index = $#pcs + 1; $finish_pc_index > 0; + $finish_pc_index--) { + last if $pcs[$finish_pc_index - 1] le $finish; + } + # Find smallest start_pc_index such that $start <= $pc[$start_pc_index]. + for ($start_pc_index = $finish_pc_index; $start_pc_index > 0; + $start_pc_index--) { + last if $pcs[$start_pc_index - 1] lt $start; + } + # This keeps PC values higher than $pc[$finish_pc_index] in @pcs, + # in case there are overlaps in libraries and the main binary. + @{$contained} = splice(@pcs, $start_pc_index, + $finish_pc_index - $start_pc_index); + # Map to symbols + MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); + } + + return $symbols; +} + +# Map list of PC values to symbols for a given image +sub MapToSymbols { + my $image = shift; + my $offset = shift; + my $pclist = shift; + my $symbols = shift; + + my $debug = 0; + + # Ignore empty binaries + if ($#{$pclist} < 0) { return; } + + # Figure out the addr2line command to use + my $addr2line = $obj_tool_map{"addr2line"}; + my $cmd = "$addr2line -f -C -e $image"; + if (exists $obj_tool_map{"addr2line_pdb"}) { + $addr2line = $obj_tool_map{"addr2line_pdb"}; + $cmd = "$addr2line --demangle -f -C -e $image"; + } + + # If "addr2line" isn't installed on the system at all, just use + # nm to get what info we can (function names, but not line numbers). + if (system("$addr2line --help >/dev/null 2>&1") != 0) { + MapSymbolsWithNM($image, $offset, $pclist, $symbols); + return; + } + + # "addr2line -i" can produce a variable number of lines per input + # address, with no separator that allows us to tell when data for + # the next address starts. So we find the address for a special + # symbol (_fini) and interleave this address between all real + # addresses passed to addr2line. The name of this special symbol + # can then be used as a separator. + $sep_address = undef; # May be filled in by MapSymbolsWithNM() + my $nm_symbols = {}; + MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); + # TODO(csilvers): only add '-i' if addr2line supports it. + if (defined($sep_address)) { + # Only add " -i" to addr2line if the binary supports it. + # addr2line --help returns 0, but not if it sees an unknown flag first. + if (system("$cmd -i --help >/dev/null 2>&1") == 0) { + $cmd .= " -i"; + } else { + $sep_address = undef; # no need for sep_address if we don't support -i + } + } + + # Make file with all PC values with intervening 'sep_address' so + # that we can reliably detect the end of inlined function list + open(ADDRESSES, ">$main::tmpfile_sym") || error("$main::tmpfile_sym: $!\n"); + if ($debug) { print("---- $image ---\n"); } + for (my $i = 0; $i <= $#{$pclist}; $i++) { + # addr2line always reads hex addresses, and does not need '0x' prefix. + if ($debug) { printf STDERR ("%s\n", $pclist->[$i]); } + printf ADDRESSES ("%s\n", AddressSub($pclist->[$i], $offset)); + if (defined($sep_address)) { + printf ADDRESSES ("%s\n", $sep_address); + } + } + close(ADDRESSES); + if ($debug) { + print("----\n"); + system("cat $main::tmpfile_sym"); + print("----\n"); + system("$cmd <$main::tmpfile_sym"); + print("----\n"); + } + + open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); + my $count = 0; # Index in pclist + while () { + # Read fullfunction and filelineinfo from next pair of lines + s/\r?\n$//g; + my $fullfunction = $_; + $_ = ; + s/\r?\n$//g; + my $filelinenum = $_; + + if (defined($sep_address) && $fullfunction eq $sep_symbol) { + # Terminating marker for data for this address + $count++; + next; + } + + $filelinenum =~ s|\\|/|g; # turn windows-style paths into unix-style paths + + my $pcstr = $pclist->[$count]; + my $function = ShortFunctionName($fullfunction); + if ($fullfunction eq '??') { + # See if nm found a symbol + my $nms = $nm_symbols->{$pcstr}; + if (defined($nms)) { + $function = $nms->[0]; + $fullfunction = $nms->[2]; + } + } + + # Prepend to accumulated symbols for pcstr + # (so that caller comes before callee) + my $sym = $symbols->{$pcstr}; + if (!defined($sym)) { + $sym = []; + $symbols->{$pcstr} = $sym; + } + unshift(@{$sym}, $function, $filelinenum, $fullfunction); + if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } + if (!defined($sep_address)) { + # Inlining is off, se this entry ends immediately + $count++; + } + } + close(SYMBOLS); +} + +# Use nm to map the list of referenced PCs to symbols. Return true iff we +# are able to read procedure information via nm. +sub MapSymbolsWithNM { + my $image = shift; + my $offset = shift; + my $pclist = shift; + my $symbols = shift; + + # Get nm output sorted by increasing address + my $symbol_table = GetProcedureBoundaries($image, "."); + if (!%{$symbol_table}) { + return 0; + } + # Start addresses are already the right length (8 or 16 hex digits). + my @names = sort { $symbol_table->{$a}->[0] cmp $symbol_table->{$b}->[0] } + keys(%{$symbol_table}); + + if ($#names < 0) { + # No symbols: just use addresses + foreach my $pc (@{$pclist}) { + my $pcstr = "0x" . $pc; + $symbols->{$pc} = [$pcstr, "?", $pcstr]; + } + return 0; + } + + # Sort addresses so we can do a join against nm output + my $index = 0; + my $fullname = $names[0]; + my $name = ShortFunctionName($fullname); + foreach my $pc (sort { $a cmp $b } @{$pclist}) { + # Adjust for mapped offset + my $mpc = AddressSub($pc, $offset); + while (($index < $#names) && ($mpc ge $symbol_table->{$fullname}->[1])){ + $index++; + $fullname = $names[$index]; + $name = ShortFunctionName($fullname); + } + if ($mpc lt $symbol_table->{$fullname}->[1]) { + $symbols->{$pc} = [$name, "?", $fullname]; + } else { + my $pcstr = "0x" . $pc; + $symbols->{$pc} = [$pcstr, "?", $pcstr]; + } + } + return 1; +} + +sub ShortFunctionName { + my $function = shift; + while ($function =~ s/\([^()]*\)(\s*const)?//g) { } # Argument types + while ($function =~ s/<[^<>]*>//g) { } # Remove template arguments + $function =~ s/^.*\s+(\w+::)/$1/; # Remove leading type + return $function; +} + +##### Miscellaneous ##### + +# Find the right versions of the above object tools to use. The +# argument is the program file being analyzed, and should be an ELF +# 32-bit or ELF 64-bit executable file. The location of the tools +# is determined by considering the following options in this order: +# 1) --tools option, if set +# 2) PPROF_TOOLS environment variable, if set +# 3) the environment +sub ConfigureObjTools { + my $prog_file = shift; + + # Check for the existence of $prog_file because /usr/bin/file does not + # predictably return error status in prod. + (-e $prog_file) || error("$prog_file does not exist.\n"); + + # Follow symlinks (at least for systems where "file" supports that) + my $file_type = `/usr/bin/file -L $prog_file 2>/dev/null || /usr/bin/file $prog_file`; + if ($file_type =~ /64-bit/) { + # Change $address_length to 16 if the program file is ELF 64-bit. + # We can't detect this from many (most?) heap or lock contention + # profiles, since the actual addresses referenced are generally in low + # memory even for 64-bit programs. + $address_length = 16; + } + + if ($file_type =~ /MS Windows/) { + # For windows, we provide a version of nm and addr2line as part of + # the opensource release, which is capable of parsing + # Windows-style PDB executables. It should live in the path, or + # in the same directory as pprof. + $obj_tool_map{"nm_pdb"} = "nm-pdb"; + $obj_tool_map{"addr2line_pdb"} = "addr2line-pdb"; + } + + if ($file_type =~ /Mach-O/) { + # OS X uses otool to examine Mach-O files, rather than objdump. + $obj_tool_map{"otool"} = "otool"; + $obj_tool_map{"addr2line"} = "false"; # no addr2line + $obj_tool_map{"objdump"} = "false"; # no objdump + } + + # Go fill in %obj_tool_map with the pathnames to use: + foreach my $tool (keys %obj_tool_map) { + $obj_tool_map{$tool} = ConfigureTool($obj_tool_map{$tool}); + } +} + +# Returns the path of a caller-specified object tool. If --tools or +# PPROF_TOOLS are specified, then returns the full path to the tool +# with that prefix. Otherwise, returns the path unmodified (which +# means we will look for it on PATH). +sub ConfigureTool { + my $tool = shift; + my $path; + + # --tools (or $PPROF_TOOLS) is a comma separated list, where each + # item is either a) a pathname prefix, or b) a map of the form + # :. First we look for an entry of type (b) for our + # tool. If one is found, we use it. Otherwise, we consider all the + # pathname prefixes in turn, until one yields an existing file. If + # none does, we use a default path. + my $tools = $main::opt_tools || $ENV{"PPROF_TOOLS"} || ""; + if ($tools =~ m/(,|^)\Q$tool\E:([^,]*)/) { + $path = $2; + # TODO(csilvers): sanity-check that $path exists? Hard if it's relative. + } elsif ($tools ne '') { + foreach my $prefix (split(',', $tools)) { + next if ($prefix =~ /:/); # ignore "tool:fullpath" entries in the list + if (-x $prefix . $tool) { + $path = $prefix . $tool; + last; + } + } + if (!$path) { + error("No '$tool' found with prefix specified by " . + "--tools (or \$PPROF_TOOLS) '$tools'\n"); + } + } else { + # ... otherwise use the version that exists in the same directory as + # pprof. If there's nothing there, use $PATH. + $0 =~ m,[^/]*$,; # this is everything after the last slash + my $dirname = $`; # this is everything up to and including the last slash + if (-x "$dirname$tool") { + $path = "$dirname$tool"; + } else { + $path = $tool; + } + } + if ($main::opt_debug) { print STDERR "Using '$path' for '$tool'.\n"; } + return $path; +} + +sub cleanup { + unlink($main::tmpfile_sym); + unlink(keys %main::tempnames); + + # We leave any collected profiles in $HOME/pprof in case the user wants + # to look at them later. We print a message informing them of this. + if ((scalar(@main::profile_files) > 0) && + defined($main::collected_profile)) { + if (scalar(@main::profile_files) == 1) { + print STDERR "Dynamically gathered profile is in $main::collected_profile\n"; + } + print STDERR "If you want to investigate this profile further, you can do:\n"; + print STDERR "\n"; + print STDERR " pprof \\\n"; + print STDERR " $main::prog \\\n"; + print STDERR " $main::collected_profile\n"; + print STDERR "\n"; + } +} + +sub sighandler { + cleanup(); + exit(1); +} + +sub error { + my $msg = shift; + print STDERR $msg; + cleanup(); + exit(1); +} + + +# Run $nm_command and get all the resulting procedure boundaries whose +# names match "$regexp" and returns them in a hashtable mapping from +# procedure name to a two-element vector of [start address, end address] +sub GetProcedureBoundariesViaNm { + my $nm_command = shift; + my $regexp = shift; + + my $symbol_table = {}; + open(NM, "$nm_command |") || error("$nm_command: $!\n"); + my $last_start = "0"; + my $routine = ""; + while () { + s/\r//g; # turn windows-looking lines into unix-looking lines + if (m/^\s*([0-9a-f]+) (.) (..*)/) { + my $start_val = $1; + my $type = $2; + my $this_routine = $3; + + # It's possible for two symbols to share the same address, if + # one is a zero-length variable (like __start_google_malloc) or + # one symbol is a weak alias to another (like __libc_malloc). + # In such cases, we want to ignore all values except for the + # actual symbol, which in nm-speak has type "T". The logic + # below does this, though it's a bit tricky: what happens when + # we have a series of lines with the same address, is the first + # one gets queued up to be processed. However, it won't + # *actually* be processed until later, when we read a line with + # a different address. That means that as long as we're reading + # lines with the same address, we have a chance to replace that + # item in the queue, which we do whenever we see a 'T' entry -- + # that is, a line with type 'T'. If we never see a 'T' entry, + # we'll just go ahead and process the first entry (which never + # got touched in the queue), and ignore the others. + if ($start_val eq $last_start && $type =~ /t/i) { + # We are the 'T' symbol at this address, replace previous symbol. + $routine = $this_routine; + next; + } elsif ($start_val eq $last_start) { + # We're not the 'T' symbol at this address, so ignore us. + next; + } + + if ($this_routine eq $sep_symbol) { + $sep_address = HexExtend($start_val); + } + + # Tag this routine with the starting address in case the image + # has multiple occurrences of this routine. We use a syntax + # that resembles template paramters that are automatically + # stripped out by ShortFunctionName() + $this_routine .= "<$start_val>"; + + if (defined($routine) && $routine =~ m/$regexp/) { + $symbol_table->{$routine} = [HexExtend($last_start), + HexExtend($start_val)]; + } + $last_start = $start_val; + $routine = $this_routine; + } elsif (m/^Loaded image name: (.+)/) { + # The win32 nm workalike emits information about the binary it is using. + if ($main::opt_debug) { print STDERR "Using Image $1\n"; } + } elsif (m/^PDB file name: (.+)/) { + # The win32 nm workalike emits information about the pdb it is using. + if ($main::opt_debug) { print STDERR "Using PDB $1\n"; } + } + } + close(NM); + # Handle the last line in the nm output. Unfortunately, we don't know + # how big this last symbol is, because we don't know how big the file + # is. For now, we just give it a size of 0. + # TODO(csilvers): do better here. + if (defined($routine) && $routine =~ m/$regexp/) { + $symbol_table->{$routine} = [HexExtend($last_start), + HexExtend($last_start)]; + } + return $symbol_table; +} + +# Gets the procedure boundaries for all routines in "$image" whose names +# match "$regexp" and returns them in a hashtable mapping from procedure +# name to a two-element vector of [start address, end address]. +# Will return an empty map if nm is not installed or not working properly. +sub GetProcedureBoundaries { + my $image = shift; + my $regexp = shift; + + # For libc libraries, the copy in /usr/lib/debug contains debugging symbols + my $debugging = DebuggingLibrary($image); + if ($debugging) { + $image = $debugging; + } + + my $nm = $obj_tool_map{"nm"}; + my $cppfilt = $obj_tool_map{"c++filt"}; + + # nm can fail for two reasons: 1) $image isn't a debug library; 2) nm + # binary doesn't support --demangle. In addition, for OS X we need + # to use the -f flag to get 'flat' nm output (otherwise we don't sort + # properly and get incorrect results). Unfortunately, GNU nm uses -f + # in an incompatible way. So first we test whether our nm supports + # --demangle and -f. + my $demangle_flag = ""; + my $cppfilt_flag = ""; + if (system("$nm --demangle $image >/dev/null 2>&1") == 0) { + # In this mode, we do "nm --demangle " + $demangle_flag = "--demangle"; + $cppfilt_flag = ""; + } elsif (system("$cppfilt $image >/dev/null 2>&1") == 0) { + # In this mode, we do "nm | c++filt" + $cppfilt_flag = " | $cppfilt"; + }; + my $flatten_flag = ""; + if (system("$nm -f $image >/dev/null 2>&1") == 0) { + $flatten_flag = "-f"; + } + + # Finally, in the case $imagie isn't a debug library, we try again with + # -D to at least get *exported* symbols. If we can't use --demangle, + # we use c++filt instead, if it exists on this system. + my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . + " $image 2>/dev/null $cppfilt_flag", + "$nm -D -n $flatten_flag $demangle_flag" . + " $image 2>/dev/null $cppfilt_flag", + # 6nm is for Go binaries + "6nm $image 2>/dev/null | sort", + ); + + # If the executable is an MS Windows PDB-format executable, we'll + # have set up obj_tool_map("nm_pdb"). In this case, we actually + # want to use both unix nm and windows-specific nm_pdb, since + # PDB-format executables can apparently include dwarf .o files. + if (exists $obj_tool_map{"nm_pdb"}) { + my $nm_pdb = $obj_tool_map{"nm_pdb"}; + push(@nm_commands, "$nm_pdb --demangle $image 2>/dev/null"); + } + + foreach my $nm_command (@nm_commands) { + my $symbol_table = GetProcedureBoundariesViaNm($nm_command, $regexp); + return $symbol_table if (%{$symbol_table}); + } + my $symbol_table = {}; + return $symbol_table; +} + + +# The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings. +# To make them more readable, we add underscores at interesting places. +# This routine removes the underscores, producing the canonical representation +# used by pprof to represent addresses, particularly in the tested routines. +sub CanonicalHex { + my $arg = shift; + return join '', (split '_',$arg); +} + + +# Unit test for AddressAdd: +sub AddressAddUnitTest { + my $test_data_8 = shift; + my $test_data_16 = shift; + my $error_count = 0; + my $fail_count = 0; + my $pass_count = 0; + # print STDERR "AddressAddUnitTest: ", 1+$#{$test_data_8}, " tests\n"; + + # First a few 8-nibble addresses. Note that this implementation uses + # plain old arithmetic, so a quick sanity check along with verifying what + # happens to overflow (we want it to wrap): + $address_length = 8; + foreach my $row (@{$test_data_8}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressAdd ($row->[0], $row->[1]); + if ($sum ne $row->[2]) { + printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, + $row->[0], $row->[1], $row->[2]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressAdd 32-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count = $fail_count; + $fail_count = 0; + $pass_count = 0; + + # Now 16-nibble addresses. + $address_length = 16; + foreach my $row (@{$test_data_16}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressAdd (CanonicalHex($row->[0]), CanonicalHex($row->[1])); + my $expected = join '', (split '_',$row->[2]); + if ($sum ne CanonicalHex($row->[2])) { + printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, + $row->[0], $row->[1], $row->[2]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressAdd 64-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count += $fail_count; + + return $error_count; +} + + +# Unit test for AddressSub: +sub AddressSubUnitTest { + my $test_data_8 = shift; + my $test_data_16 = shift; + my $error_count = 0; + my $fail_count = 0; + my $pass_count = 0; + # print STDERR "AddressSubUnitTest: ", 1+$#{$test_data_8}, " tests\n"; + + # First a few 8-nibble addresses. Note that this implementation uses + # plain old arithmetic, so a quick sanity check along with verifying what + # happens to overflow (we want it to wrap): + $address_length = 8; + foreach my $row (@{$test_data_8}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressSub ($row->[0], $row->[1]); + if ($sum ne $row->[3]) { + printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, + $row->[0], $row->[1], $row->[3]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressSub 32-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count = $fail_count; + $fail_count = 0; + $pass_count = 0; + + # Now 16-nibble addresses. + $address_length = 16; + foreach my $row (@{$test_data_16}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressSub (CanonicalHex($row->[0]), CanonicalHex($row->[1])); + if ($sum ne CanonicalHex($row->[3])) { + printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, + $row->[0], $row->[1], $row->[3]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressSub 64-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count += $fail_count; + + return $error_count; +} + + +# Unit test for AddressInc: +sub AddressIncUnitTest { + my $test_data_8 = shift; + my $test_data_16 = shift; + my $error_count = 0; + my $fail_count = 0; + my $pass_count = 0; + # print STDERR "AddressIncUnitTest: ", 1+$#{$test_data_8}, " tests\n"; + + # First a few 8-nibble addresses. Note that this implementation uses + # plain old arithmetic, so a quick sanity check along with verifying what + # happens to overflow (we want it to wrap): + $address_length = 8; + foreach my $row (@{$test_data_8}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressInc ($row->[0]); + if ($sum ne $row->[4]) { + printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, + $row->[0], $row->[4]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressInc 32-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count = $fail_count; + $fail_count = 0; + $pass_count = 0; + + # Now 16-nibble addresses. + $address_length = 16; + foreach my $row (@{$test_data_16}) { + if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } + my $sum = AddressInc (CanonicalHex($row->[0])); + if ($sum ne CanonicalHex($row->[4])) { + printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, + $row->[0], $row->[4]; + ++$fail_count; + } else { + ++$pass_count; + } + } + printf STDERR "AddressInc 64-bit tests: %d passes, %d failures\n", + $pass_count, $fail_count; + $error_count += $fail_count; + + return $error_count; +} + + +# Driver for unit tests. +# Currently just the address add/subtract/increment routines for 64-bit. +sub RunUnitTests { + my $error_count = 0; + + # This is a list of tuples [a, b, a+b, a-b, a+1] + my $unit_test_data_8 = [ + [qw(aaaaaaaa 50505050 fafafafa 5a5a5a5a aaaaaaab)], + [qw(50505050 aaaaaaaa fafafafa a5a5a5a6 50505051)], + [qw(ffffffff aaaaaaaa aaaaaaa9 55555555 00000000)], + [qw(00000001 ffffffff 00000000 00000002 00000002)], + [qw(00000001 fffffff0 fffffff1 00000011 00000002)], + ]; + my $unit_test_data_16 = [ + # The implementation handles data in 7-nibble chunks, so those are the + # interesting boundaries. + [qw(aaaaaaaa 50505050 + 00_000000f_afafafa 00_0000005_a5a5a5a 00_000000a_aaaaaab)], + [qw(50505050 aaaaaaaa + 00_000000f_afafafa ff_ffffffa_5a5a5a6 00_0000005_0505051)], + [qw(ffffffff aaaaaaaa + 00_000001a_aaaaaa9 00_0000005_5555555 00_0000010_0000000)], + [qw(00000001 ffffffff + 00_0000010_0000000 ff_ffffff0_0000002 00_0000000_0000002)], + [qw(00000001 fffffff0 + 00_000000f_ffffff1 ff_ffffff0_0000011 00_0000000_0000002)], + + [qw(00_a00000a_aaaaaaa 50505050 + 00_a00000f_afafafa 00_a000005_a5a5a5a 00_a00000a_aaaaaab)], + [qw(0f_fff0005_0505050 aaaaaaaa + 0f_fff000f_afafafa 0f_ffefffa_5a5a5a6 0f_fff0005_0505051)], + [qw(00_000000f_fffffff 01_800000a_aaaaaaa + 01_800001a_aaaaaa9 fe_8000005_5555555 00_0000010_0000000)], + [qw(00_0000000_0000001 ff_fffffff_fffffff + 00_0000000_0000000 00_0000000_0000002 00_0000000_0000002)], + [qw(00_0000000_0000001 ff_fffffff_ffffff0 + ff_fffffff_ffffff1 00_0000000_0000011 00_0000000_0000002)], + ]; + + $error_count += AddressAddUnitTest($unit_test_data_8, $unit_test_data_16); + $error_count += AddressSubUnitTest($unit_test_data_8, $unit_test_data_16); + $error_count += AddressIncUnitTest($unit_test_data_8, $unit_test_data_16); + if ($error_count > 0) { + print STDERR $error_count, " errors: FAILED\n"; + } else { + print STDERR "PASS\n"; + } + exit ($error_count); +} diff --git a/deps/jemalloc.orig/config.guess b/deps/jemalloc.orig/config.guess new file mode 100755 index 00000000000..0773d0f6312 --- /dev/null +++ b/deps/jemalloc.orig/config.guess @@ -0,0 +1,1456 @@ +#! /bin/sh +# Attempt to guess a canonical system name. +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, +# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. + +timestamp='2004-03-03' + +# This file is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# Originally written by Per Bothner . +# Please send patches to . Submit a context +# diff and a properly formatted ChangeLog entry. +# +# This script attempts to guess a canonical system name similar to +# config.sub. If it succeeds, it prints the system name on stdout, and +# exits with 0. Otherwise, it exits with 1. +# +# The plan is that this can be called by configure scripts if you +# don't specify an explicit build system type. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] + +Output the configuration name of the system \`$me' is run on. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.guess ($timestamp) + +Originally written by Per Bothner. +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 +Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit 0 ;; + --version | -v ) + echo "$version" ; exit 0 ;; + --help | --h* | -h ) + echo "$usage"; exit 0 ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" >&2 + exit 1 ;; + * ) + break ;; + esac +done + +if test $# != 0; then + echo "$me: too many arguments$help" >&2 + exit 1 +fi + +trap 'exit 1' 1 2 15 + +# CC_FOR_BUILD -- compiler used by this script. Note that the use of a +# compiler to aid in system detection is discouraged as it requires +# temporary files to be created and, as you can see below, it is a +# headache to deal with in a portable fashion. + +# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still +# use `HOST_CC' if defined, but it is deprecated. + +# Portable tmp directory creation inspired by the Autoconf team. + +set_cc_for_build=' +trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; +trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; +: ${TMPDIR=/tmp} ; + { tmp=`(umask 077 && mktemp -d -q "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || + { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || + { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; +dummy=$tmp/dummy ; +tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; +case $CC_FOR_BUILD,$HOST_CC,$CC in + ,,) echo "int x;" > $dummy.c ; + for c in cc gcc c89 c99 ; do + if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then + CC_FOR_BUILD="$c"; break ; + fi ; + done ; + if test x"$CC_FOR_BUILD" = x ; then + CC_FOR_BUILD=no_compiler_found ; + fi + ;; + ,,*) CC_FOR_BUILD=$CC ;; + ,*,*) CC_FOR_BUILD=$HOST_CC ;; +esac ;' + +# This is needed to find uname on a Pyramid OSx when run in the BSD universe. +# (ghazi@noc.rutgers.edu 1994-08-24) +if (test -f /.attbin/uname) >/dev/null 2>&1 ; then + PATH=$PATH:/.attbin ; export PATH +fi + +UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown +UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown +UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown +UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown + +# Note: order is significant - the case branches are not exclusive. + +case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in + *:NetBSD:*:*) + # NetBSD (nbsd) targets should (where applicable) match one or + # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, + # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently + # switched to ELF, *-*-netbsd* would select the old + # object file format. This provides both forward + # compatibility and a consistent mechanism for selecting the + # object file format. + # + # Note: NetBSD doesn't particularly care about the vendor + # portion of the name. We always set it to "unknown". + sysctl="sysctl -n hw.machine_arch" + UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + case "${UNAME_MACHINE_ARCH}" in + armeb) machine=armeb-unknown ;; + arm*) machine=arm-unknown ;; + sh3el) machine=shl-unknown ;; + sh3eb) machine=sh-unknown ;; + *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + esac + # The Operating System including object format, if it has switched + # to ELF recently, or will in the future. + case "${UNAME_MACHINE_ARCH}" in + arm*|i386|m68k|ns32k|sh3*|sparc|vax) + eval $set_cc_for_build + if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep __ELF__ >/dev/null + then + # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). + # Return netbsd for either. FIX? + os=netbsd + else + os=netbsdelf + fi + ;; + *) + os=netbsd + ;; + esac + # The OS release + # Debian GNU/NetBSD machines have a different userland, and + # thus, need a distinct triplet. However, they do not need + # kernel version information, so it can be replaced with a + # suitable tag, in the style of linux-gnu. + case "${UNAME_VERSION}" in + Debian*) + release='-gnu' + ;; + *) + release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + ;; + esac + # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: + # contains redundant information, the shorter form: + # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. + echo "${machine}-${os}${release}" + exit 0 ;; + amd64:OpenBSD:*:*) + echo x86_64-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + amiga:OpenBSD:*:*) + echo m68k-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + arc:OpenBSD:*:*) + echo mipsel-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + cats:OpenBSD:*:*) + echo arm-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + hp300:OpenBSD:*:*) + echo m68k-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + mac68k:OpenBSD:*:*) + echo m68k-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + macppc:OpenBSD:*:*) + echo powerpc-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + mvme68k:OpenBSD:*:*) + echo m68k-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + mvme88k:OpenBSD:*:*) + echo m88k-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + mvmeppc:OpenBSD:*:*) + echo powerpc-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + pegasos:OpenBSD:*:*) + echo powerpc-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + pmax:OpenBSD:*:*) + echo mipsel-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + sgi:OpenBSD:*:*) + echo mipseb-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + sun3:OpenBSD:*:*) + echo m68k-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + wgrisc:OpenBSD:*:*) + echo mipsel-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + *:OpenBSD:*:*) + echo ${UNAME_MACHINE}-unknown-openbsd${UNAME_RELEASE} + exit 0 ;; + *:ekkoBSD:*:*) + echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} + exit 0 ;; + macppc:MirBSD:*:*) + echo powerppc-unknown-mirbsd${UNAME_RELEASE} + exit 0 ;; + *:MirBSD:*:*) + echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} + exit 0 ;; + alpha:OSF1:*:*) + case $UNAME_RELEASE in + *4.0) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` + ;; + *5.*) + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + ;; + esac + # According to Compaq, /usr/sbin/psrinfo has been available on + # OSF/1 and Tru64 systems produced since 1995. I hope that + # covers most systems running today. This code pipes the CPU + # types through head -n 1, so we only detect the type of CPU 0. + ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` + case "$ALPHA_CPU_TYPE" in + "EV4 (21064)") + UNAME_MACHINE="alpha" ;; + "EV4.5 (21064)") + UNAME_MACHINE="alpha" ;; + "LCA4 (21066/21068)") + UNAME_MACHINE="alpha" ;; + "EV5 (21164)") + UNAME_MACHINE="alphaev5" ;; + "EV5.6 (21164A)") + UNAME_MACHINE="alphaev56" ;; + "EV5.6 (21164PC)") + UNAME_MACHINE="alphapca56" ;; + "EV5.7 (21164PC)") + UNAME_MACHINE="alphapca57" ;; + "EV6 (21264)") + UNAME_MACHINE="alphaev6" ;; + "EV6.7 (21264A)") + UNAME_MACHINE="alphaev67" ;; + "EV6.8CB (21264C)") + UNAME_MACHINE="alphaev68" ;; + "EV6.8AL (21264B)") + UNAME_MACHINE="alphaev68" ;; + "EV6.8CX (21264D)") + UNAME_MACHINE="alphaev68" ;; + "EV6.9A (21264/EV69A)") + UNAME_MACHINE="alphaev69" ;; + "EV7 (21364)") + UNAME_MACHINE="alphaev7" ;; + "EV7.9 (21364A)") + UNAME_MACHINE="alphaev79" ;; + esac + # A Pn.n version is a patched version. + # A Vn.n version is a released version. + # A Tn.n version is a released field test version. + # A Xn.n version is an unreleased experimental baselevel. + # 1.2 uses "1.2" for uname -r. + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + exit 0 ;; + Alpha*:OpenVMS:*:*) + echo alpha-hp-vms + exit 0 ;; + Alpha\ *:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # Should we change UNAME_MACHINE based on the output of uname instead + # of the specific Alpha model? + echo alpha-pc-interix + exit 0 ;; + 21064:Windows_NT:50:3) + echo alpha-dec-winnt3.5 + exit 0 ;; + Amiga*:UNIX_System_V:4.0:*) + echo m68k-unknown-sysv4 + exit 0;; + *:[Aa]miga[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-amigaos + exit 0 ;; + *:[Mm]orph[Oo][Ss]:*:*) + echo ${UNAME_MACHINE}-unknown-morphos + exit 0 ;; + *:OS/390:*:*) + echo i370-ibm-openedition + exit 0 ;; + *:OS400:*:*) + echo powerpc-ibm-os400 + exit 0 ;; + arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) + echo arm-acorn-riscix${UNAME_RELEASE} + exit 0;; + SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) + echo hppa1.1-hitachi-hiuxmpp + exit 0;; + Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) + # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. + if test "`(/bin/universe) 2>/dev/null`" = att ; then + echo pyramid-pyramid-sysv3 + else + echo pyramid-pyramid-bsd + fi + exit 0 ;; + NILE*:*:*:dcosx) + echo pyramid-pyramid-svr4 + exit 0 ;; + DRS?6000:unix:4.0:6*) + echo sparc-icl-nx6 + exit 0 ;; + DRS?6000:UNIX_SV:4.2*:7*) + case `/usr/bin/uname -p` in + sparc) echo sparc-icl-nx7 && exit 0 ;; + esac ;; + sun4H:SunOS:5.*:*) + echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit 0 ;; + sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) + echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit 0 ;; + i86pc:SunOS:5.*:*) + echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit 0 ;; + sun4*:SunOS:6*:*) + # According to config.sub, this is the proper way to canonicalize + # SunOS6. Hard to guess exactly what SunOS6 will be like, but + # it's likely to be more like Solaris than SunOS4. + echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit 0 ;; + sun4*:SunOS:*:*) + case "`/usr/bin/arch -k`" in + Series*|S4*) + UNAME_RELEASE=`uname -v` + ;; + esac + # Japanese Language versions have a version number like `4.1.3-JL'. + echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` + exit 0 ;; + sun3*:SunOS:*:*) + echo m68k-sun-sunos${UNAME_RELEASE} + exit 0 ;; + sun*:*:4.2BSD:*) + UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` + test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + case "`/bin/arch`" in + sun3) + echo m68k-sun-sunos${UNAME_RELEASE} + ;; + sun4) + echo sparc-sun-sunos${UNAME_RELEASE} + ;; + esac + exit 0 ;; + aushp:SunOS:*:*) + echo sparc-auspex-sunos${UNAME_RELEASE} + exit 0 ;; + # The situation for MiNT is a little confusing. The machine name + # can be virtually everything (everything which is not + # "atarist" or "atariste" at least should have a processor + # > m68000). The system name ranges from "MiNT" over "FreeMiNT" + # to the lowercase version "mint" (or "freemint"). Finally + # the system name "TOS" denotes a system which is actually not + # MiNT. But MiNT is downward compatible to TOS, so this should + # be no problem. + atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit 0 ;; + atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit 0 ;; + *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) + echo m68k-atari-mint${UNAME_RELEASE} + exit 0 ;; + milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) + echo m68k-milan-mint${UNAME_RELEASE} + exit 0 ;; + hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) + echo m68k-hades-mint${UNAME_RELEASE} + exit 0 ;; + *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) + echo m68k-unknown-mint${UNAME_RELEASE} + exit 0 ;; + m68k:machten:*:*) + echo m68k-apple-machten${UNAME_RELEASE} + exit 0 ;; + powerpc:machten:*:*) + echo powerpc-apple-machten${UNAME_RELEASE} + exit 0 ;; + RISC*:Mach:*:*) + echo mips-dec-mach_bsd4.3 + exit 0 ;; + RISC*:ULTRIX:*:*) + echo mips-dec-ultrix${UNAME_RELEASE} + exit 0 ;; + VAX*:ULTRIX*:*:*) + echo vax-dec-ultrix${UNAME_RELEASE} + exit 0 ;; + 2020:CLIX:*:* | 2430:CLIX:*:*) + echo clipper-intergraph-clix${UNAME_RELEASE} + exit 0 ;; + mips:*:*:UMIPS | mips:*:*:RISCos) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c +#ifdef __cplusplus +#include /* for printf() prototype */ + int main (int argc, char *argv[]) { +#else + int main (argc, argv) int argc; char *argv[]; { +#endif + #if defined (host_mips) && defined (MIPSEB) + #if defined (SYSTYPE_SYSV) + printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_SVR4) + printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); + #endif + #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) + printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); + #endif + #endif + exit (-1); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c \ + && $dummy `echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` \ + && exit 0 + echo mips-mips-riscos${UNAME_RELEASE} + exit 0 ;; + Motorola:PowerMAX_OS:*:*) + echo powerpc-motorola-powermax + exit 0 ;; + Motorola:*:4.3:PL8-*) + echo powerpc-harris-powermax + exit 0 ;; + Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) + echo powerpc-harris-powermax + exit 0 ;; + Night_Hawk:Power_UNIX:*:*) + echo powerpc-harris-powerunix + exit 0 ;; + m88k:CX/UX:7*:*) + echo m88k-harris-cxux7 + exit 0 ;; + m88k:*:4*:R4*) + echo m88k-motorola-sysv4 + exit 0 ;; + m88k:*:3*:R3*) + echo m88k-motorola-sysv3 + exit 0 ;; + AViiON:dgux:*:*) + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` + if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] + then + if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ + [ ${TARGET_BINARY_INTERFACE}x = x ] + then + echo m88k-dg-dgux${UNAME_RELEASE} + else + echo m88k-dg-dguxbcs${UNAME_RELEASE} + fi + else + echo i586-dg-dgux${UNAME_RELEASE} + fi + exit 0 ;; + M88*:DolphinOS:*:*) # DolphinOS (SVR3) + echo m88k-dolphin-sysv3 + exit 0 ;; + M88*:*:R3*:*) + # Delta 88k system running SVR3 + echo m88k-motorola-sysv3 + exit 0 ;; + XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) + echo m88k-tektronix-sysv3 + exit 0 ;; + Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) + echo m68k-tektronix-bsd + exit 0 ;; + *:IRIX*:*:*) + echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` + exit 0 ;; + ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit 0 ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + i*86:AIX:*:*) + echo i386-ibm-aix + exit 0 ;; + ia64:AIX:*:*) + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} + exit 0 ;; + *:AIX:2:3) + if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + + main() + { + if (!__power_pc()) + exit(1); + puts("powerpc-ibm-aix3.2.5"); + exit(0); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 + echo rs6000-ibm-aix3.2.5 + elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then + echo rs6000-ibm-aix3.2.4 + else + echo rs6000-ibm-aix3.2 + fi + exit 0 ;; + *:AIX:*:[45]) + IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` + if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then + IBM_ARCH=rs6000 + else + IBM_ARCH=powerpc + fi + if [ -x /usr/bin/oslevel ] ; then + IBM_REV=`/usr/bin/oslevel` + else + IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} + fi + echo ${IBM_ARCH}-ibm-aix${IBM_REV} + exit 0 ;; + *:AIX:*:*) + echo rs6000-ibm-aix + exit 0 ;; + ibmrt:4.4BSD:*|romp-ibm:BSD:*) + echo romp-ibm-bsd4.4 + exit 0 ;; + ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and + echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to + exit 0 ;; # report: romp-ibm BSD 4.3 + *:BOSX:*:*) + echo rs6000-bull-bosx + exit 0 ;; + DPX/2?00:B.O.S.:*:*) + echo m68k-bull-sysv3 + exit 0 ;; + 9000/[34]??:4.3bsd:1.*:*) + echo m68k-hp-bsd + exit 0 ;; + hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) + echo m68k-hp-bsd4.4 + exit 0 ;; + 9000/[34678]??:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + case "${UNAME_MACHINE}" in + 9000/31? ) HP_ARCH=m68000 ;; + 9000/[34]?? ) HP_ARCH=m68k ;; + 9000/[678][0-9][0-9]) + if [ -x /usr/bin/getconf ]; then + sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; + '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + esac ;; + esac + fi + if [ "${HP_ARCH}" = "" ]; then + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + + #define _HPUX_SOURCE + #include + #include + + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); + + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } +EOF + (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + test -z "$HP_ARCH" && HP_ARCH=hppa + fi ;; + esac + if [ ${HP_ARCH} = "hppa2.0w" ] + then + # avoid double evaluation of $set_cc_for_build + test -n "$CC_FOR_BUILD" || eval $set_cc_for_build + if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E -) | grep __LP64__ >/dev/null + then + HP_ARCH="hppa2.0w" + else + HP_ARCH="hppa64" + fi + fi + echo ${HP_ARCH}-hp-hpux${HPUX_REV} + exit 0 ;; + ia64:HP-UX:*:*) + HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` + echo ia64-hp-hpux${HPUX_REV} + exit 0 ;; + 3050*:HI-UX:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + int + main () + { + long cpu = sysconf (_SC_CPU_VERSION); + /* The order matters, because CPU_IS_HP_MC68K erroneously returns + true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct + results, however. */ + if (CPU_IS_PA_RISC (cpu)) + { + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; + case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; + default: puts ("hppa-hitachi-hiuxwe2"); break; + } + } + else if (CPU_IS_HP_MC68K (cpu)) + puts ("m68k-hitachi-hiuxwe2"); + else puts ("unknown-hitachi-hiuxwe2"); + exit (0); + } +EOF + $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 + echo unknown-hitachi-hiuxwe2 + exit 0 ;; + 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) + echo hppa1.1-hp-bsd + exit 0 ;; + 9000/8??:4.3bsd:*:*) + echo hppa1.0-hp-bsd + exit 0 ;; + *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) + echo hppa1.0-hp-mpeix + exit 0 ;; + hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) + echo hppa1.1-hp-osf + exit 0 ;; + hp8??:OSF1:*:*) + echo hppa1.0-hp-osf + exit 0 ;; + i*86:OSF1:*:*) + if [ -x /usr/sbin/sysversion ] ; then + echo ${UNAME_MACHINE}-unknown-osf1mk + else + echo ${UNAME_MACHINE}-unknown-osf1 + fi + exit 0 ;; + parisc*:Lites*:*:*) + echo hppa1.1-hp-lites + exit 0 ;; + C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) + echo c1-convex-bsd + exit 0 ;; + C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit 0 ;; + C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) + echo c34-convex-bsd + exit 0 ;; + C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) + echo c38-convex-bsd + exit 0 ;; + C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) + echo c4-convex-bsd + exit 0 ;; + CRAY*Y-MP:*:*:*) + echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit 0 ;; + CRAY*[A-Z]90:*:*:*) + echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ + | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ + -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ + -e 's/\.[^.]*$/.X/' + exit 0 ;; + CRAY*TS:*:*:*) + echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit 0 ;; + CRAY*T3E:*:*:*) + echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit 0 ;; + CRAY*SV1:*:*:*) + echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit 0 ;; + *:UNICOS/mp:*:*) + echo nv1-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit 0 ;; + F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) + FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit 0 ;; + 5000:UNIX_System_V:4.*:*) + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit 0 ;; + i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) + echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} + exit 0 ;; + sparc*:BSD/OS:*:*) + echo sparc-unknown-bsdi${UNAME_RELEASE} + exit 0 ;; + *:BSD/OS:*:*) + echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} + exit 0 ;; + *:FreeBSD:*:*) + # Determine whether the default compiler uses glibc. + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + #if __GLIBC__ >= 2 + LIBC=gnu + #else + LIBC= + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` + # GNU/KFreeBSD systems have a "k" prefix to indicate we are using + # FreeBSD's kernel, but not the complete OS. + case ${LIBC} in gnu) kernel_only='k' ;; esac + echo ${UNAME_MACHINE}-unknown-${kernel_only}freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`${LIBC:+-$LIBC} + exit 0 ;; + i*:CYGWIN*:*) + echo ${UNAME_MACHINE}-pc-cygwin + exit 0 ;; + i*:MINGW*:*) + echo ${UNAME_MACHINE}-pc-mingw32 + exit 0 ;; + i*:PW*:*) + echo ${UNAME_MACHINE}-pc-pw32 + exit 0 ;; + x86:Interix*:[34]*) + echo i586-pc-interix${UNAME_RELEASE}|sed -e 's/\..*//' + exit 0 ;; + [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) + echo i${UNAME_MACHINE}-pc-mks + exit 0 ;; + i*:Windows_NT*:* | Pentium*:Windows_NT*:*) + # How do we know it's Interix rather than the generic POSIX subsystem? + # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we + # UNAME_MACHINE based on the output of uname instead of i386? + echo i586-pc-interix + exit 0 ;; + i*:UWIN*:*) + echo ${UNAME_MACHINE}-pc-uwin + exit 0 ;; + p*:CYGWIN*:*) + echo powerpcle-unknown-cygwin + exit 0 ;; + prep*:SunOS:5.*:*) + echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit 0 ;; + *:GNU:*:*) + # the GNU system + echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` + exit 0 ;; + *:GNU/*:*:*) + # other systems with GNU libc and userland + echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu + exit 0 ;; + i*86:Minix:*:*) + echo ${UNAME_MACHINE}-pc-minix + exit 0 ;; + arm*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit 0 ;; + cris:Linux:*:*) + echo cris-axis-linux-gnu + exit 0 ;; + ia64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit 0 ;; + m68*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit 0 ;; + mips:Linux:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #undef CPU + #undef mips + #undef mipsel + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=mipsel + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=mips + #else + CPU= + #endif + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` + test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 + ;; + mips64:Linux:*:*) + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #undef CPU + #undef mips64 + #undef mips64el + #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) + CPU=mips64el + #else + #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) + CPU=mips64 + #else + CPU= + #endif + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` + test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 + ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-gnu + exit 0 ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-gnu + exit 0 ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null + if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi + echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + exit 0 ;; + parisc:Linux:*:* | hppa:Linux:*:*) + # Look for CPU level + case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in + PA7*) echo hppa1.1-unknown-linux-gnu ;; + PA8*) echo hppa2.0-unknown-linux-gnu ;; + *) echo hppa-unknown-linux-gnu ;; + esac + exit 0 ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-gnu + exit 0 ;; + s390:Linux:*:* | s390x:Linux:*:*) + echo ${UNAME_MACHINE}-ibm-linux + exit 0 ;; + sh64*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit 0 ;; + sh*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit 0 ;; + sparc:Linux:*:* | sparc64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit 0 ;; + x86_64:Linux:*:*) + echo x86_64-unknown-linux-gnu + exit 0 ;; + i*86:Linux:*:*) + # The BFD linker knows what the default object file format is, so + # first see if it will tell us. cd to the root directory to prevent + # problems with other programs or directories called `ld' in the path. + # Set LC_ALL=C to ensure ld outputs messages in English. + ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ + | sed -ne '/supported targets:/!d + s/[ ][ ]*/ /g + s/.*supported targets: *// + s/ .*// + p'` + case "$ld_supported_targets" in + elf32-i386) + TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu" + ;; + a.out-i386-linux) + echo "${UNAME_MACHINE}-pc-linux-gnuaout" + exit 0 ;; + coff-i386) + echo "${UNAME_MACHINE}-pc-linux-gnucoff" + exit 0 ;; + "") + # Either a pre-BFD a.out linker (linux-gnuoldld) or + # one that does not give us useful --help. + echo "${UNAME_MACHINE}-pc-linux-gnuoldld" + exit 0 ;; + esac + # Determine whether the default compiler is a.out or elf + eval $set_cc_for_build + sed 's/^ //' << EOF >$dummy.c + #include + #ifdef __ELF__ + # ifdef __GLIBC__ + # if __GLIBC__ >= 2 + LIBC=gnu + # else + LIBC=gnulibc1 + # endif + # else + LIBC=gnulibc1 + # endif + #else + #ifdef __INTEL_COMPILER + LIBC=gnu + #else + LIBC=gnuaout + #endif + #endif + #ifdef __dietlibc__ + LIBC=dietlibc + #endif +EOF + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` + test x"${LIBC}" != x && echo "${UNAME_MACHINE}-pc-linux-${LIBC}" && exit 0 + test x"${TENTATIVE}" != x && echo "${TENTATIVE}" && exit 0 + ;; + i*86:DYNIX/ptx:4*:*) + # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. + # earlier versions are messed up and put the nodename in both + # sysname and nodename. + echo i386-sequent-sysv4 + exit 0 ;; + i*86:UNIX_SV:4.2MP:2.*) + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, + # I just have to hope. -- rms. + # Use sysv4.2uw... so that sysv4* matches it. + echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} + exit 0 ;; + i*86:OS/2:*:*) + # If we were able to find `uname', then EMX Unix compatibility + # is probably installed. + echo ${UNAME_MACHINE}-pc-os2-emx + exit 0 ;; + i*86:XTS-300:*:STOP) + echo ${UNAME_MACHINE}-unknown-stop + exit 0 ;; + i*86:atheos:*:*) + echo ${UNAME_MACHINE}-unknown-atheos + exit 0 ;; + i*86:syllable:*:*) + echo ${UNAME_MACHINE}-pc-syllable + exit 0 ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) + echo i386-unknown-lynxos${UNAME_RELEASE} + exit 0 ;; + i*86:*DOS:*:*) + echo ${UNAME_MACHINE}-pc-msdosdjgpp + exit 0 ;; + i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) + UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` + if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then + echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} + else + echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} + fi + exit 0 ;; + i*86:*:5:[78]*) + case `/bin/uname -X | grep "^Machine"` in + *486*) UNAME_MACHINE=i486 ;; + *Pentium) UNAME_MACHINE=i586 ;; + *Pent*|*Celeron) UNAME_MACHINE=i686 ;; + esac + echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} + exit 0 ;; + i*86:*:3.2:*) + if test -f /usr/options/cb.name; then + UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then + UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` + (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 + (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ + && UNAME_MACHINE=i586 + (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ + && UNAME_MACHINE=i686 + (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ + && UNAME_MACHINE=i686 + echo ${UNAME_MACHINE}-pc-sco$UNAME_REL + else + echo ${UNAME_MACHINE}-pc-sysv32 + fi + exit 0 ;; + pc:*:*:*) + # Left here for compatibility: + # uname -m prints for DJGPP always 'pc', but it prints nothing about + # the processor, so we play safe by assuming i386. + echo i386-pc-msdosdjgpp + exit 0 ;; + Intel:Mach:3*:*) + echo i386-pc-mach3 + exit 0 ;; + paragon:*:*:*) + echo i860-intel-osf1 + exit 0 ;; + i860:*:4.*:*) # i860-SVR4 + if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then + echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 + else # Add other i860-SVR4 vendors below as they are discovered. + echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 + fi + exit 0 ;; + mini*:CTIX:SYS*5:*) + # "miniframe" + echo m68010-convergent-sysv + exit 0 ;; + mc68k:UNIX:SYSTEM5:3.51m) + echo m68k-convergent-sysv + exit 0 ;; + M680?0:D-NIX:5.3:*) + echo m68k-diab-dnix + exit 0 ;; + M68*:*:R3V[567]*:*) + test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0) + OS_REL='' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && echo i486-ncr-sysv4.3${OS_REL} && exit 0 + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && echo i586-ncr-sysv4.3${OS_REL} && exit 0 ;; + 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && echo i486-ncr-sysv4 && exit 0 ;; + m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) + echo m68k-unknown-lynxos${UNAME_RELEASE} + exit 0 ;; + mc68030:UNIX_System_V:4.*:*) + echo m68k-atari-sysv4 + exit 0 ;; + TSUNAMI:LynxOS:2.*:*) + echo sparc-unknown-lynxos${UNAME_RELEASE} + exit 0 ;; + rs6000:LynxOS:2.*:*) + echo rs6000-unknown-lynxos${UNAME_RELEASE} + exit 0 ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) + echo powerpc-unknown-lynxos${UNAME_RELEASE} + exit 0 ;; + SM[BE]S:UNIX_SV:*:*) + echo mips-dde-sysv${UNAME_RELEASE} + exit 0 ;; + RM*:ReliantUNIX-*:*:*) + echo mips-sni-sysv4 + exit 0 ;; + RM*:SINIX-*:*:*) + echo mips-sni-sysv4 + exit 0 ;; + *:SINIX-*:*:*) + if uname -p 2>/dev/null >/dev/null ; then + UNAME_MACHINE=`(uname -p) 2>/dev/null` + echo ${UNAME_MACHINE}-sni-sysv4 + else + echo ns32k-sni-sysv + fi + exit 0 ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit 0 ;; + *:UNIX_System_V:4*:FTX*) + # From Gerald Hewes . + # How about differentiating between stratus architectures? -djm + echo hppa1.1-stratus-sysv4 + exit 0 ;; + *:*:*:FTX*) + # From seanf@swdc.stratus.com. + echo i860-stratus-sysv4 + exit 0 ;; + *:VOS:*:*) + # From Paul.Green@stratus.com. + echo hppa1.1-stratus-vos + exit 0 ;; + mc68*:A/UX:*:*) + echo m68k-apple-aux${UNAME_RELEASE} + exit 0 ;; + news*:NEWS-OS:6*:*) + echo mips-sony-newsos6 + exit 0 ;; + R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) + if [ -d /usr/nec ]; then + echo mips-nec-sysv${UNAME_RELEASE} + else + echo mips-unknown-sysv${UNAME_RELEASE} + fi + exit 0 ;; + BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. + echo powerpc-be-beos + exit 0 ;; + BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. + echo powerpc-apple-beos + exit 0 ;; + BePC:BeOS:*:*) # BeOS running on Intel PC compatible. + echo i586-pc-beos + exit 0 ;; + SX-4:SUPER-UX:*:*) + echo sx4-nec-superux${UNAME_RELEASE} + exit 0 ;; + SX-5:SUPER-UX:*:*) + echo sx5-nec-superux${UNAME_RELEASE} + exit 0 ;; + SX-6:SUPER-UX:*:*) + echo sx6-nec-superux${UNAME_RELEASE} + exit 0 ;; + Power*:Rhapsody:*:*) + echo powerpc-apple-rhapsody${UNAME_RELEASE} + exit 0 ;; + *:Rhapsody:*:*) + echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} + exit 0 ;; + *:Darwin:*:*) + case `uname -p` in + *86) UNAME_PROCESSOR=i686 ;; + powerpc) UNAME_PROCESSOR=powerpc ;; + esac + echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} + exit 0 ;; + *:procnto*:*:* | *:QNX:[0123456789]*:*) + UNAME_PROCESSOR=`uname -p` + if test "$UNAME_PROCESSOR" = "x86"; then + UNAME_PROCESSOR=i386 + UNAME_MACHINE=pc + fi + echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} + exit 0 ;; + *:QNX:*:4*) + echo i386-pc-qnx + exit 0 ;; + NSR-?:NONSTOP_KERNEL:*:*) + echo nsr-tandem-nsk${UNAME_RELEASE} + exit 0 ;; + *:NonStop-UX:*:*) + echo mips-compaq-nonstopux + exit 0 ;; + BS2000:POSIX*:*:*) + echo bs2000-siemens-sysv + exit 0 ;; + DS/*:UNIX_System_V:*:*) + echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} + exit 0 ;; + *:Plan9:*:*) + # "uname -m" is not consistent, so use $cputype instead. 386 + # is converted to i386 for consistency with other x86 + # operating systems. + if test "$cputype" = "386"; then + UNAME_MACHINE=i386 + else + UNAME_MACHINE="$cputype" + fi + echo ${UNAME_MACHINE}-unknown-plan9 + exit 0 ;; + *:TOPS-10:*:*) + echo pdp10-unknown-tops10 + exit 0 ;; + *:TENEX:*:*) + echo pdp10-unknown-tenex + exit 0 ;; + KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) + echo pdp10-dec-tops20 + exit 0 ;; + XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) + echo pdp10-xkl-tops20 + exit 0 ;; + *:TOPS-20:*:*) + echo pdp10-unknown-tops20 + exit 0 ;; + *:ITS:*:*) + echo pdp10-unknown-its + exit 0 ;; + SEI:*:*:SEIUX) + echo mips-sei-seiux${UNAME_RELEASE} + exit 0 ;; + *:DragonFly:*:*) + echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` + exit 0 ;; +esac + +#echo '(No uname command or uname output not recognized.)' 1>&2 +#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 + +eval $set_cc_for_build +cat >$dummy.c < +# include +#endif +main () +{ +#if defined (sony) +#if defined (MIPSEB) + /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, + I don't know.... */ + printf ("mips-sony-bsd\n"); exit (0); +#else +#include + printf ("m68k-sony-newsos%s\n", +#ifdef NEWSOS4 + "4" +#else + "" +#endif + ); exit (0); +#endif +#endif + +#if defined (__arm) && defined (__acorn) && defined (__unix) + printf ("arm-acorn-riscix"); exit (0); +#endif + +#if defined (hp300) && !defined (hpux) + printf ("m68k-hp-bsd\n"); exit (0); +#endif + +#if defined (NeXT) +#if !defined (__ARCHITECTURE__) +#define __ARCHITECTURE__ "m68k" +#endif + int version; + version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; + if (version < 4) + printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); + else + printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); + exit (0); +#endif + +#if defined (MULTIMAX) || defined (n16) +#if defined (UMAXV) + printf ("ns32k-encore-sysv\n"); exit (0); +#else +#if defined (CMU) + printf ("ns32k-encore-mach\n"); exit (0); +#else + printf ("ns32k-encore-bsd\n"); exit (0); +#endif +#endif +#endif + +#if defined (__386BSD__) + printf ("i386-pc-bsd\n"); exit (0); +#endif + +#if defined (sequent) +#if defined (i386) + printf ("i386-sequent-dynix\n"); exit (0); +#endif +#if defined (ns32000) + printf ("ns32k-sequent-dynix\n"); exit (0); +#endif +#endif + +#if defined (_SEQUENT_) + struct utsname un; + + uname(&un); + + if (strncmp(un.version, "V2", 2) == 0) { + printf ("i386-sequent-ptx2\n"); exit (0); + } + if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ + printf ("i386-sequent-ptx1\n"); exit (0); + } + printf ("i386-sequent-ptx\n"); exit (0); + +#endif + +#if defined (vax) +# if !defined (ultrix) +# include +# if defined (BSD) +# if BSD == 43 + printf ("vax-dec-bsd4.3\n"); exit (0); +# else +# if BSD == 199006 + printf ("vax-dec-bsd4.3reno\n"); exit (0); +# else + printf ("vax-dec-bsd\n"); exit (0); +# endif +# endif +# else + printf ("vax-dec-bsd\n"); exit (0); +# endif +# else + printf ("vax-dec-ultrix\n"); exit (0); +# endif +#endif + +#if defined (alliant) && defined (i860) + printf ("i860-alliant-bsd\n"); exit (0); +#endif + + exit (1); +} +EOF + +$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && $dummy && exit 0 + +# Apollos put the system type in the environment. + +test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; } + +# Convex versions that predate uname can use getsysinfo(1) + +if [ -x /usr/convex/getsysinfo ] +then + case `getsysinfo -f cpu_type` in + c1*) + echo c1-convex-bsd + exit 0 ;; + c2*) + if getsysinfo -f scalar_acc + then echo c32-convex-bsd + else echo c2-convex-bsd + fi + exit 0 ;; + c34*) + echo c34-convex-bsd + exit 0 ;; + c38*) + echo c38-convex-bsd + exit 0 ;; + c4*) + echo c4-convex-bsd + exit 0 ;; + esac +fi + +cat >&2 < in order to provide the needed +information to handle your system. + +config.guess timestamp = $timestamp + +uname -m = `(uname -m) 2>/dev/null || echo unknown` +uname -r = `(uname -r) 2>/dev/null || echo unknown` +uname -s = `(uname -s) 2>/dev/null || echo unknown` +uname -v = `(uname -v) 2>/dev/null || echo unknown` + +/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` +/bin/uname -X = `(/bin/uname -X) 2>/dev/null` + +hostinfo = `(hostinfo) 2>/dev/null` +/bin/universe = `(/bin/universe) 2>/dev/null` +/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` +/bin/arch = `(/bin/arch) 2>/dev/null` +/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` +/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` + +UNAME_MACHINE = ${UNAME_MACHINE} +UNAME_RELEASE = ${UNAME_RELEASE} +UNAME_SYSTEM = ${UNAME_SYSTEM} +UNAME_VERSION = ${UNAME_VERSION} +EOF + +exit 1 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/deps/jemalloc.orig/config.stamp.in b/deps/jemalloc.orig/config.stamp.in new file mode 100644 index 00000000000..e69de29bb2d diff --git a/deps/jemalloc.orig/config.sub b/deps/jemalloc.orig/config.sub new file mode 100755 index 00000000000..264f820aa55 --- /dev/null +++ b/deps/jemalloc.orig/config.sub @@ -0,0 +1,1549 @@ +#! /bin/sh +# Configuration validation subroutine script. +# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, +# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. + +timestamp='2004-02-23' + +# This file is (in principle) common to ALL GNU software. +# The presence of a machine in this file suggests that SOME GNU software +# can handle that machine. It does not imply ALL GNU software can. +# +# This file is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# As a special exception to the GNU General Public License, if you +# distribute this file as part of a program that contains a +# configuration script generated by Autoconf, you may include it under +# the same distribution terms that you use for the rest of that program. + +# Please send patches to . Submit a context +# diff and a properly formatted ChangeLog entry. +# +# Configuration subroutine to validate and canonicalize a configuration type. +# Supply the specified configuration type as an argument. +# If it is invalid, we print an error message on stderr and exit with code 1. +# Otherwise, we print the canonical config type on stdout and succeed. + +# This file is supposed to be the same for all GNU packages +# and recognize all the CPU types, system types and aliases +# that are meaningful with *any* GNU software. +# Each package is responsible for reporting which valid configurations +# it does not support. The user should be able to distinguish +# a failure to support a valid configuration from a meaningless +# configuration. + +# The goal of this file is to map all the various variations of a given +# machine specification into a single specification in the form: +# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM +# or in some cases, the newer four-part form: +# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM +# It is wrong to echo any other type of specification. + +me=`echo "$0" | sed -e 's,.*/,,'` + +usage="\ +Usage: $0 [OPTION] CPU-MFR-OPSYS + $0 [OPTION] ALIAS + +Canonicalize a configuration name. + +Operation modes: + -h, --help print this help, then exit + -t, --time-stamp print date of last modification, then exit + -v, --version print version number, then exit + +Report bugs and patches to ." + +version="\ +GNU config.sub ($timestamp) + +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 +Free Software Foundation, Inc. + +This is free software; see the source for copying conditions. There is NO +warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." + +help=" +Try \`$me --help' for more information." + +# Parse command line +while test $# -gt 0 ; do + case $1 in + --time-stamp | --time* | -t ) + echo "$timestamp" ; exit 0 ;; + --version | -v ) + echo "$version" ; exit 0 ;; + --help | --h* | -h ) + echo "$usage"; exit 0 ;; + -- ) # Stop option processing + shift; break ;; + - ) # Use stdin as input. + break ;; + -* ) + echo "$me: invalid option $1$help" + exit 1 ;; + + *local*) + # First pass through any local machine types. + echo $1 + exit 0;; + + * ) + break ;; + esac +done + +case $# in + 0) echo "$me: missing argument$help" >&2 + exit 1;; + 1) ;; + *) echo "$me: too many arguments$help" >&2 + exit 1;; +esac + +# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). +# Here we must recognize all the valid KERNEL-OS combinations. +maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` +case $maybe_os in + nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \ + kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*) + os=-$maybe_os + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` + ;; + *) + basic_machine=`echo $1 | sed 's/-[^-]*$//'` + if [ $basic_machine != $1 ] + then os=`echo $1 | sed 's/.*-/-/'` + else os=; fi + ;; +esac + +### Let's recognize common machines as not being operating systems so +### that things like config.sub decstation-3100 work. We also +### recognize some manufacturers as not being operating systems, so we +### can provide default operating systems below. +case $os in + -sun*os*) + # Prevent following clause from handling this invalid input. + ;; + -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ + -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ + -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ + -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ + -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ + -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ + -apple | -axis) + os= + basic_machine=$1 + ;; + -sim | -cisco | -oki | -wec | -winbond) + os= + basic_machine=$1 + ;; + -scout) + ;; + -wrs) + os=-vxworks + basic_machine=$1 + ;; + -chorusos*) + os=-chorusos + basic_machine=$1 + ;; + -chorusrdb) + os=-chorusrdb + basic_machine=$1 + ;; + -hiux*) + os=-hiuxwe2 + ;; + -sco5) + os=-sco3.2v5 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco4) + os=-sco3.2v4 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2.[4-9]*) + os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco3.2v[4-9]*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -sco*) + os=-sco3.2v2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -udk*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -isc) + os=-isc2.2 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -clix*) + basic_machine=clipper-intergraph + ;; + -isc*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; + -lynx*) + os=-lynxos + ;; + -ptx*) + basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` + ;; + -windowsnt*) + os=`echo $os | sed -e 's/windowsnt/winnt/'` + ;; + -psos*) + os=-psos + ;; + -mint | -mint[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; +esac + +# Decode aliases for certain CPU-COMPANY combinations. +case $basic_machine in + # Recognize the basic CPU types without company name. + # Some are omitted here because they have special meanings below. + 1750a | 580 \ + | a29k \ + | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ + | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ + | am33_2.0 \ + | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \ + | c4x | clipper \ + | d10v | d30v | dlx | dsp16xx \ + | fr30 | frv \ + | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | i370 | i860 | i960 | ia64 \ + | ip2k | iq2000 \ + | m32r | m68000 | m68k | m88k | mcore \ + | mips | mipsbe | mipseb | mipsel | mipsle \ + | mips16 \ + | mips64 | mips64el \ + | mips64vr | mips64vrel \ + | mips64orion | mips64orionel \ + | mips64vr4100 | mips64vr4100el \ + | mips64vr4300 | mips64vr4300el \ + | mips64vr5000 | mips64vr5000el \ + | mipsisa32 | mipsisa32el \ + | mipsisa32r2 | mipsisa32r2el \ + | mipsisa64 | mipsisa64el \ + | mipsisa64r2 | mipsisa64r2el \ + | mipsisa64sb1 | mipsisa64sb1el \ + | mipsisa64sr71k | mipsisa64sr71kel \ + | mipstx39 | mipstx39el \ + | mn10200 | mn10300 \ + | msp430 \ + | ns16k | ns32k \ + | openrisc | or32 \ + | pdp10 | pdp11 | pj | pjl \ + | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \ + | pyramid \ + | sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \ + | sh64 | sh64le \ + | sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \ + | strongarm \ + | tahoe | thumb | tic4x | tic80 | tron \ + | v850 | v850e \ + | we32k \ + | x86 | xscale | xstormy16 | xtensa \ + | z8k) + basic_machine=$basic_machine-unknown + ;; + m6811 | m68hc11 | m6812 | m68hc12) + # Motorola 68HC11/12. + basic_machine=$basic_machine-unknown + os=-none + ;; + m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) + ;; + + # We use `pc' rather than `unknown' + # because (1) that's what they normally are, and + # (2) the word "unknown" tends to confuse beginning users. + i*86 | x86_64) + basic_machine=$basic_machine-pc + ;; + # Object if more than one company name word. + *-*-*) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; + # Recognize the basic CPU types with company name. + 580-* \ + | a29k-* \ + | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ + | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ + | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ + | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ + | avr-* \ + | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \ + | clipper-* | cydra-* \ + | d10v-* | d30v-* | dlx-* \ + | elxsi-* \ + | f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \ + | h8300-* | h8500-* \ + | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | i*86-* | i860-* | i960-* | ia64-* \ + | ip2k-* | iq2000-* \ + | m32r-* \ + | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ + | m88110-* | m88k-* | mcore-* \ + | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ + | mips16-* \ + | mips64-* | mips64el-* \ + | mips64vr-* | mips64vrel-* \ + | mips64orion-* | mips64orionel-* \ + | mips64vr4100-* | mips64vr4100el-* \ + | mips64vr4300-* | mips64vr4300el-* \ + | mips64vr5000-* | mips64vr5000el-* \ + | mipsisa32-* | mipsisa32el-* \ + | mipsisa32r2-* | mipsisa32r2el-* \ + | mipsisa64-* | mipsisa64el-* \ + | mipsisa64r2-* | mipsisa64r2el-* \ + | mipsisa64sb1-* | mipsisa64sb1el-* \ + | mipsisa64sr71k-* | mipsisa64sr71kel-* \ + | mipstx39-* | mipstx39el-* \ + | msp430-* \ + | none-* | np1-* | nv1-* | ns16k-* | ns32k-* \ + | orion-* \ + | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \ + | pyramid-* \ + | romp-* | rs6000-* \ + | sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \ + | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ + | sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \ + | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \ + | tahoe-* | thumb-* \ + | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tron-* \ + | v850-* | v850e-* | vax-* \ + | we32k-* \ + | x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \ + | xtensa-* \ + | ymp-* \ + | z8k-*) + ;; + # Recognize the various machine names and aliases which stand + # for a CPU type and a company and sometimes even an OS. + 386bsd) + basic_machine=i386-unknown + os=-bsd + ;; + 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) + basic_machine=m68000-att + ;; + 3b*) + basic_machine=we32k-att + ;; + a29khif) + basic_machine=a29k-amd + os=-udi + ;; + abacus) + basic_machine=abacus-unknown + ;; + adobe68k) + basic_machine=m68010-adobe + os=-scout + ;; + alliant | fx80) + basic_machine=fx80-alliant + ;; + altos | altos3068) + basic_machine=m68k-altos + ;; + am29k) + basic_machine=a29k-none + os=-bsd + ;; + amd64) + basic_machine=x86_64-pc + ;; + amd64-*) + basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + amdahl) + basic_machine=580-amdahl + os=-sysv + ;; + amiga | amiga-*) + basic_machine=m68k-unknown + ;; + amigaos | amigados) + basic_machine=m68k-unknown + os=-amigaos + ;; + amigaunix | amix) + basic_machine=m68k-unknown + os=-sysv4 + ;; + apollo68) + basic_machine=m68k-apollo + os=-sysv + ;; + apollo68bsd) + basic_machine=m68k-apollo + os=-bsd + ;; + aux) + basic_machine=m68k-apple + os=-aux + ;; + balance) + basic_machine=ns32k-sequent + os=-dynix + ;; + c90) + basic_machine=c90-cray + os=-unicos + ;; + convex-c1) + basic_machine=c1-convex + os=-bsd + ;; + convex-c2) + basic_machine=c2-convex + os=-bsd + ;; + convex-c32) + basic_machine=c32-convex + os=-bsd + ;; + convex-c34) + basic_machine=c34-convex + os=-bsd + ;; + convex-c38) + basic_machine=c38-convex + os=-bsd + ;; + cray | j90) + basic_machine=j90-cray + os=-unicos + ;; + cr16c) + basic_machine=cr16c-unknown + os=-elf + ;; + crds | unos) + basic_machine=m68k-crds + ;; + cris | cris-* | etrax*) + basic_machine=cris-axis + ;; + crx) + basic_machine=crx-unknown + os=-elf + ;; + da30 | da30-*) + basic_machine=m68k-da30 + ;; + decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) + basic_machine=mips-dec + ;; + decsystem10* | dec10*) + basic_machine=pdp10-dec + os=-tops10 + ;; + decsystem20* | dec20*) + basic_machine=pdp10-dec + os=-tops20 + ;; + delta | 3300 | motorola-3300 | motorola-delta \ + | 3300-motorola | delta-motorola) + basic_machine=m68k-motorola + ;; + delta88) + basic_machine=m88k-motorola + os=-sysv3 + ;; + dpx20 | dpx20-*) + basic_machine=rs6000-bull + os=-bosx + ;; + dpx2* | dpx2*-bull) + basic_machine=m68k-bull + os=-sysv3 + ;; + ebmon29k) + basic_machine=a29k-amd + os=-ebmon + ;; + elxsi) + basic_machine=elxsi-elxsi + os=-bsd + ;; + encore | umax | mmax) + basic_machine=ns32k-encore + ;; + es1800 | OSE68k | ose68k | ose | OSE) + basic_machine=m68k-ericsson + os=-ose + ;; + fx2800) + basic_machine=i860-alliant + ;; + genix) + basic_machine=ns32k-ns + ;; + gmicro) + basic_machine=tron-gmicro + os=-sysv + ;; + go32) + basic_machine=i386-pc + os=-go32 + ;; + h3050r* | hiux*) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + h8300hms) + basic_machine=h8300-hitachi + os=-hms + ;; + h8300xray) + basic_machine=h8300-hitachi + os=-xray + ;; + h8500hms) + basic_machine=h8500-hitachi + os=-hms + ;; + harris) + basic_machine=m88k-harris + os=-sysv3 + ;; + hp300-*) + basic_machine=m68k-hp + ;; + hp300bsd) + basic_machine=m68k-hp + os=-bsd + ;; + hp300hpux) + basic_machine=m68k-hp + os=-hpux + ;; + hp3k9[0-9][0-9] | hp9[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k2[0-9][0-9] | hp9k31[0-9]) + basic_machine=m68000-hp + ;; + hp9k3[2-9][0-9]) + basic_machine=m68k-hp + ;; + hp9k6[0-9][0-9] | hp6[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hp9k7[0-79][0-9] | hp7[0-79][0-9]) + basic_machine=hppa1.1-hp + ;; + hp9k78[0-9] | hp78[0-9]) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) + # FIXME: really hppa2.0-hp + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][13679] | hp8[0-9][13679]) + basic_machine=hppa1.1-hp + ;; + hp9k8[0-9][0-9] | hp8[0-9][0-9]) + basic_machine=hppa1.0-hp + ;; + hppa-next) + os=-nextstep3 + ;; + hppaosf) + basic_machine=hppa1.1-hp + os=-osf + ;; + hppro) + basic_machine=hppa1.1-hp + os=-proelf + ;; + i370-ibm* | ibm*) + basic_machine=i370-ibm + ;; +# I'm not sure what "Sysv32" means. Should this be sysv3.2? + i*86v32) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv32 + ;; + i*86v4*) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv4 + ;; + i*86v) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-sysv + ;; + i*86sol2) + basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` + os=-solaris2 + ;; + i386mach) + basic_machine=i386-mach + os=-mach + ;; + i386-vsta | vsta) + basic_machine=i386-unknown + os=-vsta + ;; + iris | iris4d) + basic_machine=mips-sgi + case $os in + -irix*) + ;; + *) + os=-irix4 + ;; + esac + ;; + isi68 | isi) + basic_machine=m68k-isi + os=-sysv + ;; + m88k-omron*) + basic_machine=m88k-omron + ;; + magnum | m3230) + basic_machine=mips-mips + os=-sysv + ;; + merlin) + basic_machine=ns32k-utek + os=-sysv + ;; + mingw32) + basic_machine=i386-pc + os=-mingw32 + ;; + miniframe) + basic_machine=m68000-convergent + ;; + *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) + basic_machine=m68k-atari + os=-mint + ;; + mips3*-*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` + ;; + mips3*) + basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown + ;; + mmix*) + basic_machine=mmix-knuth + os=-mmixware + ;; + monitor) + basic_machine=m68k-rom68k + os=-coff + ;; + morphos) + basic_machine=powerpc-unknown + os=-morphos + ;; + msdos) + basic_machine=i386-pc + os=-msdos + ;; + mvs) + basic_machine=i370-ibm + os=-mvs + ;; + ncr3000) + basic_machine=i486-ncr + os=-sysv4 + ;; + netbsd386) + basic_machine=i386-unknown + os=-netbsd + ;; + netwinder) + basic_machine=armv4l-rebel + os=-linux + ;; + news | news700 | news800 | news900) + basic_machine=m68k-sony + os=-newsos + ;; + news1000) + basic_machine=m68030-sony + os=-newsos + ;; + news-3600 | risc-news) + basic_machine=mips-sony + os=-newsos + ;; + necv70) + basic_machine=v70-nec + os=-sysv + ;; + next | m*-next ) + basic_machine=m68k-next + case $os in + -nextstep* ) + ;; + -ns2*) + os=-nextstep2 + ;; + *) + os=-nextstep3 + ;; + esac + ;; + nh3000) + basic_machine=m68k-harris + os=-cxux + ;; + nh[45]000) + basic_machine=m88k-harris + os=-cxux + ;; + nindy960) + basic_machine=i960-intel + os=-nindy + ;; + mon960) + basic_machine=i960-intel + os=-mon960 + ;; + nonstopux) + basic_machine=mips-compaq + os=-nonstopux + ;; + np1) + basic_machine=np1-gould + ;; + nv1) + basic_machine=nv1-cray + os=-unicosmp + ;; + nsr-tandem) + basic_machine=nsr-tandem + ;; + op50n-* | op60c-*) + basic_machine=hppa1.1-oki + os=-proelf + ;; + or32 | or32-*) + basic_machine=or32-unknown + os=-coff + ;; + os400) + basic_machine=powerpc-ibm + os=-os400 + ;; + OSE68000 | ose68000) + basic_machine=m68000-ericsson + os=-ose + ;; + os68k) + basic_machine=m68k-none + os=-os68k + ;; + pa-hitachi) + basic_machine=hppa1.1-hitachi + os=-hiuxwe2 + ;; + paragon) + basic_machine=i860-intel + os=-osf + ;; + pbd) + basic_machine=sparc-tti + ;; + pbb) + basic_machine=m68k-tti + ;; + pc532 | pc532-*) + basic_machine=ns32k-pc532 + ;; + pentium | p5 | k5 | k6 | nexgen | viac3) + basic_machine=i586-pc + ;; + pentiumpro | p6 | 6x86 | athlon | athlon_*) + basic_machine=i686-pc + ;; + pentiumii | pentium2 | pentiumiii | pentium3) + basic_machine=i686-pc + ;; + pentium4) + basic_machine=i786-pc + ;; + pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) + basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumpro-* | p6-* | 6x86-* | athlon-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) + basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pentium4-*) + basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + pn) + basic_machine=pn-gould + ;; + power) basic_machine=power-ibm + ;; + ppc) basic_machine=powerpc-unknown + ;; + ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppcle | powerpclittle | ppc-le | powerpc-little) + basic_machine=powerpcle-unknown + ;; + ppcle-* | powerpclittle-*) + basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64) basic_machine=powerpc64-unknown + ;; + ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ppc64le | powerpc64little | ppc64-le | powerpc64-little) + basic_machine=powerpc64le-unknown + ;; + ppc64le-* | powerpc64little-*) + basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + ps2) + basic_machine=i386-ibm + ;; + pw32) + basic_machine=i586-unknown + os=-pw32 + ;; + rom68k) + basic_machine=m68k-rom68k + os=-coff + ;; + rm[46]00) + basic_machine=mips-siemens + ;; + rtpc | rtpc-*) + basic_machine=romp-ibm + ;; + s390 | s390-*) + basic_machine=s390-ibm + ;; + s390x | s390x-*) + basic_machine=s390x-ibm + ;; + sa29200) + basic_machine=a29k-amd + os=-udi + ;; + sb1) + basic_machine=mipsisa64sb1-unknown + ;; + sb1el) + basic_machine=mipsisa64sb1el-unknown + ;; + sei) + basic_machine=mips-sei + os=-seiux + ;; + sequent) + basic_machine=i386-sequent + ;; + sh) + basic_machine=sh-hitachi + os=-hms + ;; + sh64) + basic_machine=sh64-unknown + ;; + sparclite-wrs | simso-wrs) + basic_machine=sparclite-wrs + os=-vxworks + ;; + sps7) + basic_machine=m68k-bull + os=-sysv2 + ;; + spur) + basic_machine=spur-unknown + ;; + st2000) + basic_machine=m68k-tandem + ;; + stratus) + basic_machine=i860-stratus + os=-sysv4 + ;; + sun2) + basic_machine=m68000-sun + ;; + sun2os3) + basic_machine=m68000-sun + os=-sunos3 + ;; + sun2os4) + basic_machine=m68000-sun + os=-sunos4 + ;; + sun3os3) + basic_machine=m68k-sun + os=-sunos3 + ;; + sun3os4) + basic_machine=m68k-sun + os=-sunos4 + ;; + sun4os3) + basic_machine=sparc-sun + os=-sunos3 + ;; + sun4os4) + basic_machine=sparc-sun + os=-sunos4 + ;; + sun4sol2) + basic_machine=sparc-sun + os=-solaris2 + ;; + sun3 | sun3-*) + basic_machine=m68k-sun + ;; + sun4) + basic_machine=sparc-sun + ;; + sun386 | sun386i | roadrunner) + basic_machine=i386-sun + ;; + sv1) + basic_machine=sv1-cray + os=-unicos + ;; + symmetry) + basic_machine=i386-sequent + os=-dynix + ;; + t3e) + basic_machine=alphaev5-cray + os=-unicos + ;; + t90) + basic_machine=t90-cray + os=-unicos + ;; + tic54x | c54x*) + basic_machine=tic54x-unknown + os=-coff + ;; + tic55x | c55x*) + basic_machine=tic55x-unknown + os=-coff + ;; + tic6x | c6x*) + basic_machine=tic6x-unknown + os=-coff + ;; + tx39) + basic_machine=mipstx39-unknown + ;; + tx39el) + basic_machine=mipstx39el-unknown + ;; + toad1) + basic_machine=pdp10-xkl + os=-tops20 + ;; + tower | tower-32) + basic_machine=m68k-ncr + ;; + tpf) + basic_machine=s390x-ibm + os=-tpf + ;; + udi29k) + basic_machine=a29k-amd + os=-udi + ;; + ultra3) + basic_machine=a29k-nyu + os=-sym1 + ;; + v810 | necv810) + basic_machine=v810-nec + os=-none + ;; + vaxv) + basic_machine=vax-dec + os=-sysv + ;; + vms) + basic_machine=vax-dec + os=-vms + ;; + vpp*|vx|vx-*) + basic_machine=f301-fujitsu + ;; + vxworks960) + basic_machine=i960-wrs + os=-vxworks + ;; + vxworks68) + basic_machine=m68k-wrs + os=-vxworks + ;; + vxworks29k) + basic_machine=a29k-wrs + os=-vxworks + ;; + w65*) + basic_machine=w65-wdc + os=-none + ;; + w89k-*) + basic_machine=hppa1.1-winbond + os=-proelf + ;; + xps | xps100) + basic_machine=xps100-honeywell + ;; + ymp) + basic_machine=ymp-cray + os=-unicos + ;; + z8k-*-coff) + basic_machine=z8k-unknown + os=-sim + ;; + none) + basic_machine=none-none + os=-none + ;; + +# Here we handle the default manufacturer of certain CPU types. It is in +# some cases the only manufacturer, in others, it is the most popular. + w89k) + basic_machine=hppa1.1-winbond + ;; + op50n) + basic_machine=hppa1.1-oki + ;; + op60c) + basic_machine=hppa1.1-oki + ;; + romp) + basic_machine=romp-ibm + ;; + rs6000) + basic_machine=rs6000-ibm + ;; + vax) + basic_machine=vax-dec + ;; + pdp10) + # there are many clones, so DEC is not a safe bet + basic_machine=pdp10-unknown + ;; + pdp11) + basic_machine=pdp11-dec + ;; + we32k) + basic_machine=we32k-att + ;; + sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele) + basic_machine=sh-unknown + ;; + sh64) + basic_machine=sh64-unknown + ;; + sparc | sparcv9 | sparcv9b) + basic_machine=sparc-sun + ;; + cydra) + basic_machine=cydra-cydrome + ;; + orion) + basic_machine=orion-highlevel + ;; + orion105) + basic_machine=clipper-highlevel + ;; + mac | mpw | mac-mpw) + basic_machine=m68k-apple + ;; + pmac | pmac-mpw) + basic_machine=powerpc-apple + ;; + *-unknown) + # Make sure to match an already-canonicalized machine name. + ;; + *) + echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 + exit 1 + ;; +esac + +# Here we canonicalize certain aliases for manufacturers. +case $basic_machine in + *-digital*) + basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` + ;; + *-commodore*) + basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` + ;; + *) + ;; +esac + +# Decode manufacturer-specific aliases for certain operating systems. + +if [ x"$os" != x"" ] +then +case $os in + # First match some system type aliases + # that might get confused with valid system types. + # -solaris* is a basic system type, with this one exception. + -solaris1 | -solaris1.*) + os=`echo $os | sed -e 's|solaris1|sunos4|'` + ;; + -solaris) + os=-solaris2 + ;; + -svr4*) + os=-sysv4 + ;; + -unixware*) + os=-sysv4.2uw + ;; + -gnu/linux*) + os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` + ;; + # First accept the basic system types. + # The portable systems comes first. + # Each alternative MUST END IN A *, to match a version number. + # -sysv* is not here because it comes later, after sysvr4. + -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \ + | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ + | -aos* \ + | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ + | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \ + | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ + | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ + | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ + | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ + | -chorusos* | -chorusrdb* \ + | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \ + | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ + | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ + | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ + | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ + | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*) + # Remember, each alternative MUST END IN *, to match a version number. + ;; + -qnx*) + case $basic_machine in + x86-* | i*86-*) + ;; + *) + os=-nto$os + ;; + esac + ;; + -nto-qnx*) + ;; + -nto*) + os=`echo $os | sed -e 's|nto|nto-qnx|'` + ;; + -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* \ + | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) + ;; + -mac*) + os=`echo $os | sed -e 's|mac|macos|'` + ;; + -linux-dietlibc) + os=-linux-dietlibc + ;; + -linux*) + os=`echo $os | sed -e 's|linux|linux-gnu|'` + ;; + -sunos5*) + os=`echo $os | sed -e 's|sunos5|solaris2|'` + ;; + -sunos6*) + os=`echo $os | sed -e 's|sunos6|solaris3|'` + ;; + -opened*) + os=-openedition + ;; + -os400*) + os=-os400 + ;; + -wince*) + os=-wince + ;; + -osfrose*) + os=-osfrose + ;; + -osf*) + os=-osf + ;; + -utek*) + os=-bsd + ;; + -dynix*) + os=-bsd + ;; + -acis*) + os=-aos + ;; + -atheos*) + os=-atheos + ;; + -syllable*) + os=-syllable + ;; + -386bsd) + os=-bsd + ;; + -ctix* | -uts*) + os=-sysv + ;; + -nova*) + os=-rtmk-nova + ;; + -ns2 ) + os=-nextstep2 + ;; + -nsk*) + os=-nsk + ;; + # Preserve the version number of sinix5. + -sinix5.*) + os=`echo $os | sed -e 's|sinix|sysv|'` + ;; + -sinix*) + os=-sysv4 + ;; + -tpf*) + os=-tpf + ;; + -triton*) + os=-sysv3 + ;; + -oss*) + os=-sysv3 + ;; + -svr4) + os=-sysv4 + ;; + -svr3) + os=-sysv3 + ;; + -sysvr4) + os=-sysv4 + ;; + # This must come after -sysvr4. + -sysv*) + ;; + -ose*) + os=-ose + ;; + -es1800*) + os=-ose + ;; + -xenix) + os=-xenix + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + os=-mint + ;; + -aros*) + os=-aros + ;; + -kaos*) + os=-kaos + ;; + -none) + ;; + *) + # Get rid of the `-' at the beginning of $os. + os=`echo $os | sed 's/[^-]*-//'` + echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 + exit 1 + ;; +esac +else + +# Here we handle the default operating systems that come with various machines. +# The value should be what the vendor currently ships out the door with their +# machine or put another way, the most popular os provided with the machine. + +# Note that if you're going to try to match "-MANUFACTURER" here (say, +# "-sun"), then you have to tell the case statement up towards the top +# that MANUFACTURER isn't an operating system. Otherwise, code above +# will signal an error saying that MANUFACTURER isn't an operating +# system, and we'll never get to this point. + +case $basic_machine in + *-acorn) + os=-riscix1.2 + ;; + arm*-rebel) + os=-linux + ;; + arm*-semi) + os=-aout + ;; + c4x-* | tic4x-*) + os=-coff + ;; + # This must come before the *-dec entry. + pdp10-*) + os=-tops20 + ;; + pdp11-*) + os=-none + ;; + *-dec | vax-*) + os=-ultrix4.2 + ;; + m68*-apollo) + os=-domain + ;; + i386-sun) + os=-sunos4.0.2 + ;; + m68000-sun) + os=-sunos3 + # This also exists in the configure program, but was not the + # default. + # os=-sunos4 + ;; + m68*-cisco) + os=-aout + ;; + mips*-cisco) + os=-elf + ;; + mips*-*) + os=-elf + ;; + or32-*) + os=-coff + ;; + *-tti) # must be before sparc entry or we get the wrong os. + os=-sysv3 + ;; + sparc-* | *-sun) + os=-sunos4.1.1 + ;; + *-be) + os=-beos + ;; + *-ibm) + os=-aix + ;; + *-wec) + os=-proelf + ;; + *-winbond) + os=-proelf + ;; + *-oki) + os=-proelf + ;; + *-hp) + os=-hpux + ;; + *-hitachi) + os=-hiux + ;; + i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) + os=-sysv + ;; + *-cbm) + os=-amigaos + ;; + *-dg) + os=-dgux + ;; + *-dolphin) + os=-sysv3 + ;; + m68k-ccur) + os=-rtu + ;; + m88k-omron*) + os=-luna + ;; + *-next ) + os=-nextstep + ;; + *-sequent) + os=-ptx + ;; + *-crds) + os=-unos + ;; + *-ns) + os=-genix + ;; + i370-*) + os=-mvs + ;; + *-next) + os=-nextstep3 + ;; + *-gould) + os=-sysv + ;; + *-highlevel) + os=-bsd + ;; + *-encore) + os=-bsd + ;; + *-sgi) + os=-irix + ;; + *-siemens) + os=-sysv4 + ;; + *-masscomp) + os=-rtu + ;; + f30[01]-fujitsu | f700-fujitsu) + os=-uxpv + ;; + *-rom68k) + os=-coff + ;; + *-*bug) + os=-coff + ;; + *-apple) + os=-macos + ;; + *-atari*) + os=-mint + ;; + *) + os=-none + ;; +esac +fi + +# Here we handle the case where we know the os, and the CPU type, but not the +# manufacturer. We pick the logical manufacturer. +vendor=unknown +case $basic_machine in + *-unknown) + case $os in + -riscix*) + vendor=acorn + ;; + -sunos*) + vendor=sun + ;; + -aix*) + vendor=ibm + ;; + -beos*) + vendor=be + ;; + -hpux*) + vendor=hp + ;; + -mpeix*) + vendor=hp + ;; + -hiux*) + vendor=hitachi + ;; + -unos*) + vendor=crds + ;; + -dgux*) + vendor=dg + ;; + -luna*) + vendor=omron + ;; + -genix*) + vendor=ns + ;; + -mvs* | -opened*) + vendor=ibm + ;; + -os400*) + vendor=ibm + ;; + -ptx*) + vendor=sequent + ;; + -tpf*) + vendor=ibm + ;; + -vxsim* | -vxworks* | -windiss*) + vendor=wrs + ;; + -aux*) + vendor=apple + ;; + -hms*) + vendor=hitachi + ;; + -mpw* | -macos*) + vendor=apple + ;; + -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) + vendor=atari + ;; + -vos*) + vendor=stratus + ;; + esac + basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` + ;; +esac + +echo $basic_machine$os +exit 0 + +# Local variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "timestamp='" +# time-stamp-format: "%:y-%02m-%02d" +# time-stamp-end: "'" +# End: diff --git a/deps/jemalloc.orig/configure.ac b/deps/jemalloc.orig/configure.ac new file mode 100644 index 00000000000..b58aa520998 --- /dev/null +++ b/deps/jemalloc.orig/configure.ac @@ -0,0 +1,938 @@ +dnl Process this file with autoconf to produce a configure script. +AC_INIT([Makefile.in]) + +dnl ============================================================================ +dnl Custom macro definitions. + +dnl JE_CFLAGS_APPEND(cflag) +AC_DEFUN([JE_CFLAGS_APPEND], +[ +AC_MSG_CHECKING([whether compiler supports $1]) +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="$1" +else + CFLAGS="${CFLAGS} $1" +fi +AC_RUN_IFELSE([AC_LANG_PROGRAM( +[[ +]], [[ + return 0; +]])], + AC_MSG_RESULT([yes]), + AC_MSG_RESULT([no]) + [CFLAGS="${TCFLAGS}"] +) +]) + +dnl JE_COMPILABLE(label, hcode, mcode, rvar) +AC_DEFUN([JE_COMPILABLE], +[ +AC_MSG_CHECKING([whether $1 is compilable]) +AC_RUN_IFELSE([AC_LANG_PROGRAM( +[$2], [$3])], + AC_MSG_RESULT([yes]) + [$4="yes"], + AC_MSG_RESULT([no]) + [$4="no"] +) +]) + +dnl ============================================================================ + +srcroot=$srcdir +if test "x${srcroot}" = "x." ; then + srcroot="" +else + srcroot="${srcroot}/" +fi +AC_SUBST([srcroot]) +abs_srcroot="`cd \"${srcdir}\"; pwd`/" +AC_SUBST([abs_srcroot]) + +objroot="" +AC_SUBST([objroot]) +abs_objroot="`pwd`/" +AC_SUBST([abs_objroot]) + +dnl Munge install path variables. +if test "x$prefix" = "xNONE" ; then + prefix="/usr/local" +fi +if test "x$exec_prefix" = "xNONE" ; then + exec_prefix=$prefix +fi +PREFIX=$prefix +AC_SUBST([PREFIX]) +BINDIR=`eval echo $bindir` +BINDIR=`eval echo $BINDIR` +AC_SUBST([BINDIR]) +INCLUDEDIR=`eval echo $includedir` +INCLUDEDIR=`eval echo $INCLUDEDIR` +AC_SUBST([INCLUDEDIR]) +LIBDIR=`eval echo $libdir` +LIBDIR=`eval echo $LIBDIR` +AC_SUBST([LIBDIR]) +DATADIR=`eval echo $datadir` +DATADIR=`eval echo $DATADIR` +AC_SUBST([DATADIR]) +MANDIR=`eval echo $mandir` +MANDIR=`eval echo $MANDIR` +AC_SUBST([MANDIR]) + +dnl Support for building documentation. +AC_PATH_PROG([XSLTPROC], [xsltproc], , [$PATH]) +AC_ARG_WITH([xslroot], + [AS_HELP_STRING([--with-xslroot=], [XSL stylesheet root path])], +if test "x$with_xslroot" = "xno" ; then + XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +else + XSLROOT="${with_xslroot}" +fi, + XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +) +AC_SUBST([XSLROOT]) + +dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, +dnl just prevent autoconf from molesting CFLAGS. +CFLAGS=$CFLAGS +AC_PROG_CC +if test "x$CFLAGS" = "x" ; then + no_CFLAGS="yes" + if test "x$GCC" = "xyes" ; then + JE_CFLAGS_APPEND([-std=gnu99]) + JE_CFLAGS_APPEND([-Wall]) + JE_CFLAGS_APPEND([-pipe]) + JE_CFLAGS_APPEND([-g3]) + fi +fi +dnl Append EXTRA_CFLAGS to CFLAGS, if defined. +if test "x$EXTRA_CFLAGS" != "x" ; then + JE_CFLAGS_APPEND([$EXTRA_CFLAGS]) +fi +AC_PROG_CPP + +AC_CHECK_SIZEOF([void *]) +if test "x${ac_cv_sizeof_void_p}" = "x8" ; then + LG_SIZEOF_PTR=3 +elif test "x${ac_cv_sizeof_void_p}" = "x4" ; then + LG_SIZEOF_PTR=2 +else + AC_MSG_ERROR([Unsupported pointer size: ${ac_cv_sizeof_void_p}]) +fi +AC_DEFINE_UNQUOTED([LG_SIZEOF_PTR], [$LG_SIZEOF_PTR]) + +AC_CHECK_SIZEOF([int]) +if test "x${ac_cv_sizeof_int}" = "x8" ; then + LG_SIZEOF_INT=3 +elif test "x${ac_cv_sizeof_int}" = "x4" ; then + LG_SIZEOF_INT=2 +else + AC_MSG_ERROR([Unsupported int size: ${ac_cv_sizeof_int}]) +fi +AC_DEFINE_UNQUOTED([LG_SIZEOF_INT], [$LG_SIZEOF_INT]) + +AC_CHECK_SIZEOF([long]) +if test "x${ac_cv_sizeof_long}" = "x8" ; then + LG_SIZEOF_LONG=3 +elif test "x${ac_cv_sizeof_long}" = "x4" ; then + LG_SIZEOF_LONG=2 +else + AC_MSG_ERROR([Unsupported long size: ${ac_cv_sizeof_long}]) +fi +AC_DEFINE_UNQUOTED([LG_SIZEOF_LONG], [$LG_SIZEOF_LONG]) + +AC_CANONICAL_HOST +dnl CPU-specific settings. +CPU_SPINWAIT="" +case "${host_cpu}" in + i[[345]]86) + ;; + i686) + JE_COMPILABLE([__asm__], [], [[__asm__ volatile("pause"); return 0;]], + [asm]) + if test "x${asm}" = "xyes" ; then + CPU_SPINWAIT='__asm__ volatile("pause")' + fi + ;; + x86_64) + JE_COMPILABLE([__asm__ syntax], [], + [[__asm__ volatile("pause"); return 0;]], [asm]) + if test "x${asm}" = "xyes" ; then + CPU_SPINWAIT='__asm__ volatile("pause")' + fi + ;; + *) + ;; +esac +AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT]) + +dnl Platform-specific settings. abi and RPATH can probably be determined +dnl programmatically, but doing so is error-prone, which makes it generally +dnl not worth the trouble. +dnl +dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the +dnl definitions need to be seen before any headers are included, which is a pain +dnl to make happen otherwise. +case "${host}" in + *-*-darwin*) + CFLAGS="$CFLAGS -fno-common -no-cpp-precomp" + abi="macho" + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) + RPATH="" + ;; + *-*-freebsd*) + CFLAGS="$CFLAGS" + abi="elf" + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) + RPATH="-Wl,-rpath," + ;; + *-*-linux*) + CFLAGS="$CFLAGS" + CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" + abi="elf" + AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED]) + RPATH="-Wl,-rpath," + ;; + *-*-netbsd*) + AC_MSG_CHECKING([ABI]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM( +[[#ifdef __ELF__ +/* ELF */ +#else +#error aout +#endif +]])], + [CFLAGS="$CFLAGS"; abi="elf"], + [abi="aout"]) + AC_MSG_RESULT([$abi]) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) + RPATH="-Wl,-rpath," + ;; + *-*-solaris2*) + CFLAGS="$CFLAGS" + abi="elf" + RPATH="-Wl,-R," + dnl Solaris needs this for sigwait(). + CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" + LIBS="$LIBS -lposix4 -lsocket -lnsl" + ;; + *) + AC_MSG_RESULT([Unsupported operating system: ${host}]) + abi="elf" + RPATH="-Wl,-rpath," + ;; +esac +AC_SUBST([abi]) +AC_SUBST([RPATH]) + +JE_COMPILABLE([__attribute__ syntax], + [static __attribute__((unused)) void foo(void){}], + [], + [attribute]) +if test "x${attribute}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_ATTR], [ ]) + if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then + JE_CFLAGS_APPEND([-fvisibility=hidden]) + fi +fi + +JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ +#define _GNU_SOURCE +#include +], [ +void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); +], [mremap_fixed]) +if test "x${mremap_fixed}" = "xyes" ; then + AC_DEFINE([JEMALLOC_MREMAP_FIXED]) +fi + +dnl Support optional additions to rpath. +AC_ARG_WITH([rpath], + [AS_HELP_STRING([--with-rpath=], [Colon-separated rpath (ELF systems only)])], +if test "x$with_rpath" = "xno" ; then + RPATH_EXTRA= +else + RPATH_EXTRA="`echo $with_rpath | tr \":\" \" \"`" +fi, + RPATH_EXTRA= +) +AC_SUBST([RPATH_EXTRA]) + +dnl Disable rules that do automatic regeneration of configure output by default. +AC_ARG_ENABLE([autogen], + [AS_HELP_STRING([--enable-autogen], [Automatically regenerate configure output])], +if test "x$enable_autogen" = "xno" ; then + enable_autogen="0" +else + enable_autogen="1" +fi +, +enable_autogen="0" +) +AC_SUBST([enable_autogen]) + +AC_PROG_INSTALL +AC_PROG_RANLIB +AC_PATH_PROG([AR], [ar], , [$PATH]) +AC_PATH_PROG([LD], [ld], , [$PATH]) +AC_PATH_PROG([AUTOCONF], [autoconf], , [$PATH]) + +dnl Do not prefix public APIs by default. +AC_ARG_WITH([jemalloc_prefix], + [AS_HELP_STRING([--with-jemalloc-prefix=], [Prefix to prepend to all public APIs])], + [JEMALLOC_PREFIX="$with_jemalloc_prefix"], + [if test "x$abi" != "xmacho" ; then + JEMALLOC_PREFIX="" +else + JEMALLOC_PREFIX="je_" +fi] +) +if test "x$JEMALLOC_PREFIX" != "x" ; then + JEMALLOC_CPREFIX=`echo ${JEMALLOC_PREFIX} | tr "a-z" "A-Z"` + AC_DEFINE_UNQUOTED([JEMALLOC_PREFIX], ["$JEMALLOC_PREFIX"]) + AC_DEFINE_UNQUOTED([JEMALLOC_CPREFIX], ["$JEMALLOC_CPREFIX"]) + AC_DEFINE_UNQUOTED([JEMALLOC_P(string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix)], [${JEMALLOC_PREFIX}##string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix]) +fi + +dnl Do not mangle library-private APIs by default. +AC_ARG_WITH([private_namespace], + [AS_HELP_STRING([--with-private-namespace=], [Prefix to prepend to all library-private APIs])], + [JEMALLOC_PRIVATE_NAMESPACE="$with_private_namespace"], + [JEMALLOC_PRIVATE_NAMESPACE=""] +) +AC_DEFINE_UNQUOTED([JEMALLOC_PRIVATE_NAMESPACE], ["$JEMALLOC_PRIVATE_NAMESPACE"]) +if test "x$JEMALLOC_PRIVATE_NAMESPACE" != "x" ; then + AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [${JEMALLOC_PRIVATE_NAMESPACE}##string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix]) +else + AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix]) +fi + +dnl Do not add suffix to installed files by default. +AC_ARG_WITH([install_suffix], + [AS_HELP_STRING([--with-install-suffix=], [Suffix to append to all installed files])], + [INSTALL_SUFFIX="$with_install_suffix"], + [INSTALL_SUFFIX=] +) +install_suffix="$INSTALL_SUFFIX" +AC_SUBST([install_suffix]) + +cfgoutputs_in="${srcroot}Makefile.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/html.xsl.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/manpages.xsl.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/jemalloc.xml.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in" +cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/jemalloc_test.h.in" + +cfgoutputs_out="Makefile" +cfgoutputs_out="${cfgoutputs_out} doc/html.xsl" +cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl" +cfgoutputs_out="${cfgoutputs_out} doc/jemalloc${install_suffix}.xml" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc${install_suffix}.h" +cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h" +cfgoutputs_out="${cfgoutputs_out} test/jemalloc_test.h" + +cfgoutputs_tup="Makefile" +cfgoutputs_tup="${cfgoutputs_tup} doc/html.xsl:doc/html.xsl.in" +cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in" +cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc${install_suffix}.xml:doc/jemalloc.xml.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc${install_suffix}.h:include/jemalloc/jemalloc.h.in" +cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" +cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in" + +cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" + +cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h" + +cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in" + +dnl Do not silence irrelevant compiler warnings by default, since enabling this +dnl option incurs a performance penalty. +AC_ARG_ENABLE([cc-silence], + [AS_HELP_STRING([--enable-cc-silence], + [Silence irrelevant compiler warnings])], +[if test "x$enable_cc_silence" = "xno" ; then + enable_cc_silence="0" +else + enable_cc_silence="1" +fi +], +[enable_cc_silence="0"] +) +if test "x$enable_cc_silence" = "x1" ; then + AC_DEFINE([JEMALLOC_CC_SILENCE]) +fi + +dnl Do not compile with debugging by default. +AC_ARG_ENABLE([debug], + [AS_HELP_STRING([--enable-debug], [Build debugging code])], +[if test "x$enable_debug" = "xno" ; then + enable_debug="0" +else + enable_debug="1" +fi +], +[enable_debug="0"] +) +if test "x$enable_debug" = "x1" ; then + AC_DEFINE([JEMALLOC_DEBUG], [ ]) + AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) +fi +AC_SUBST([enable_debug]) + +dnl Only optimize if not debugging. +if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then + dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS. + optimize="no" + echo "$EXTRA_CFLAGS" | grep "\-O" >/dev/null || optimize="yes" + if test "x${optimize}" = "xyes" ; then + if test "x$GCC" = "xyes" ; then + JE_CFLAGS_APPEND([-O3]) + JE_CFLAGS_APPEND([-funroll-loops]) + else + JE_CFLAGS_APPEND([-O]) + fi + fi +fi + +dnl Do not enable statistics calculation by default. +AC_ARG_ENABLE([stats], + [AS_HELP_STRING([--enable-stats], [Enable statistics calculation/reporting])], +[if test "x$enable_stats" = "xno" ; then + enable_stats="0" +else + enable_stats="1" +fi +], +[enable_stats="0"] +) +if test "x$enable_stats" = "x1" ; then + AC_DEFINE([JEMALLOC_STATS], [ ]) +fi +AC_SUBST([enable_stats]) + +dnl Do not enable profiling by default. +AC_ARG_ENABLE([prof], + [AS_HELP_STRING([--enable-prof], [Enable allocation profiling])], +[if test "x$enable_prof" = "xno" ; then + enable_prof="0" +else + enable_prof="1" +fi +], +[enable_prof="0"] +) +if test "x$enable_prof" = "x1" ; then + backtrace_method="" +else + backtrace_method="N/A" +fi + +AC_ARG_ENABLE([prof-libunwind], + [AS_HELP_STRING([--enable-prof-libunwind], [Use libunwind for backtracing])], +[if test "x$enable_prof_libunwind" = "xno" ; then + enable_prof_libunwind="0" +else + enable_prof_libunwind="1" +fi +], +[enable_prof_libunwind="0"] +) +AC_ARG_WITH([static_libunwind], + [AS_HELP_STRING([--with-static-libunwind=], + [Path to static libunwind library; use rather than dynamically linking])], +if test "x$with_static_libunwind" = "xno" ; then + LUNWIND="-lunwind" +else + if test ! -f "$with_static_libunwind" ; then + AC_MSG_ERROR([Static libunwind not found: $with_static_libunwind]) + fi + LUNWIND="$with_static_libunwind" +fi, + LUNWIND="-lunwind" +) +if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then + AC_CHECK_HEADERS([libunwind.h], , [enable_prof_libunwind="0"]) + if test "x$LUNWIND" = "x-lunwind" ; then + AC_CHECK_LIB([unwind], [backtrace], [LIBS="$LIBS $LUNWIND"], + [enable_prof_libunwind="0"]) + else + LIBS="$LIBS $LUNWIND" + fi + if test "x${enable_prof_libunwind}" = "x1" ; then + backtrace_method="libunwind" + AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ]) + fi +fi + +AC_ARG_ENABLE([prof-libgcc], + [AS_HELP_STRING([--disable-prof-libgcc], + [Do not use libgcc for backtracing])], +[if test "x$enable_prof_libgcc" = "xno" ; then + enable_prof_libgcc="0" +else + enable_prof_libgcc="1" +fi +], +[enable_prof_libgcc="1"] +) +if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \ + -a "x$GCC" = "xyes" ; then + AC_CHECK_HEADERS([unwind.h], , [enable_prof_libgcc="0"]) + AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [LIBS="$LIBS -lgcc"], [enable_prof_libgcc="0"]) + dnl The following is conservative, in that it only has entries for CPUs on + dnl which jemalloc has been tested. + AC_MSG_CHECKING([libgcc-based backtracing reliability on ${host_cpu}]) + case "${host_cpu}" in + i[[3456]]86) + AC_MSG_RESULT([unreliable]) + enable_prof_libgcc="0"; + ;; + x86_64) + AC_MSG_RESULT([reliable]) + ;; + *) + AC_MSG_RESULT([unreliable]) + enable_prof_libgcc="0"; + ;; + esac + if test "x${enable_prof_libgcc}" = "x1" ; then + backtrace_method="libgcc" + AC_DEFINE([JEMALLOC_PROF_LIBGCC], [ ]) + fi +else + enable_prof_libgcc="0" +fi + +AC_ARG_ENABLE([prof-gcc], + [AS_HELP_STRING([--disable-prof-gcc], + [Do not use gcc intrinsics for backtracing])], +[if test "x$enable_prof_gcc" = "xno" ; then + enable_prof_gcc="0" +else + enable_prof_gcc="1" +fi +], +[enable_prof_gcc="1"] +) +if test "x$backtrace_method" = "x" -a "x$enable_prof_gcc" = "x1" \ + -a "x$GCC" = "xyes" ; then + backtrace_method="gcc intrinsics" + AC_DEFINE([JEMALLOC_PROF_GCC], [ ]) +else + enable_prof_gcc="0" +fi + +if test "x$backtrace_method" = "x" ; then + backtrace_method="none (disabling profiling)" + enable_prof="0" +fi +AC_MSG_CHECKING([configured backtracing method]) +AC_MSG_RESULT([$backtrace_method]) +if test "x$enable_prof" = "x1" ; then + LIBS="$LIBS -lm" + AC_DEFINE([JEMALLOC_PROF], [ ]) +fi +AC_SUBST([enable_prof]) + +dnl Enable tiny allocations by default. +AC_ARG_ENABLE([tiny], + [AS_HELP_STRING([--disable-tiny], [Disable tiny (sub-quantum) allocations])], +[if test "x$enable_tiny" = "xno" ; then + enable_tiny="0" +else + enable_tiny="1" +fi +], +[enable_tiny="1"] +) +if test "x$enable_tiny" = "x1" ; then + AC_DEFINE([JEMALLOC_TINY], [ ]) +fi +AC_SUBST([enable_tiny]) + +dnl Enable thread-specific caching by default. +AC_ARG_ENABLE([tcache], + [AS_HELP_STRING([--disable-tcache], [Disable per thread caches])], +[if test "x$enable_tcache" = "xno" ; then + enable_tcache="0" +else + enable_tcache="1" +fi +], +[enable_tcache="1"] +) +if test "x$enable_tcache" = "x1" ; then + AC_DEFINE([JEMALLOC_TCACHE], [ ]) +fi +AC_SUBST([enable_tcache]) + +dnl Do not enable mmap()ped swap files by default. +AC_ARG_ENABLE([swap], + [AS_HELP_STRING([--enable-swap], [Enable mmap()ped swap files])], +[if test "x$enable_swap" = "xno" ; then + enable_swap="0" +else + enable_swap="1" +fi +], +[enable_swap="0"] +) +if test "x$enable_swap" = "x1" ; then + AC_DEFINE([JEMALLOC_SWAP], [ ]) +fi +AC_SUBST([enable_swap]) + +dnl Do not enable allocation from DSS by default. +AC_ARG_ENABLE([dss], + [AS_HELP_STRING([--enable-dss], [Enable allocation from DSS])], +[if test "x$enable_dss" = "xno" ; then + enable_dss="0" +else + enable_dss="1" +fi +], +[enable_dss="0"] +) +if test "x$enable_dss" = "x1" ; then + AC_DEFINE([JEMALLOC_DSS], [ ]) +fi +AC_SUBST([enable_dss]) + +dnl Do not support the junk/zero filling option by default. +AC_ARG_ENABLE([fill], + [AS_HELP_STRING([--enable-fill], [Support junk/zero filling option])], +[if test "x$enable_fill" = "xno" ; then + enable_fill="0" +else + enable_fill="1" +fi +], +[enable_fill="0"] +) +if test "x$enable_fill" = "x1" ; then + AC_DEFINE([JEMALLOC_FILL], [ ]) +fi +AC_SUBST([enable_fill]) + +dnl Do not support the xmalloc option by default. +AC_ARG_ENABLE([xmalloc], + [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], +[if test "x$enable_xmalloc" = "xno" ; then + enable_xmalloc="0" +else + enable_xmalloc="1" +fi +], +[enable_xmalloc="0"] +) +if test "x$enable_xmalloc" = "x1" ; then + AC_DEFINE([JEMALLOC_XMALLOC], [ ]) +fi +AC_SUBST([enable_xmalloc]) + +dnl Do not support the SYSV option by default. +AC_ARG_ENABLE([sysv], + [AS_HELP_STRING([--enable-sysv], [Support SYSV semantics option])], +[if test "x$enable_sysv" = "xno" ; then + enable_sysv="0" +else + enable_sysv="1" +fi +], +[enable_sysv="0"] +) +if test "x$enable_sysv" = "x1" ; then + AC_DEFINE([JEMALLOC_SYSV], [ ]) +fi +AC_SUBST([enable_sysv]) + +dnl Do not determine page shift at run time by default. +AC_ARG_ENABLE([dynamic_page_shift], + [AS_HELP_STRING([--enable-dynamic-page-shift], + [Determine page size at run time (don't trust configure result)])], +[if test "x$enable_dynamic_page_shift" = "xno" ; then + enable_dynamic_page_shift="0" +else + enable_dynamic_page_shift="1" +fi +], +[enable_dynamic_page_shift="0"] +) +if test "x$enable_dynamic_page_shift" = "x1" ; then + AC_DEFINE([DYNAMIC_PAGE_SHIFT], [ ]) +fi +AC_SUBST([enable_dynamic_page_shift]) + +AC_MSG_CHECKING([STATIC_PAGE_SHIFT]) +AC_RUN_IFELSE([AC_LANG_PROGRAM( +[[#include +#include +#include +]], [[ + long result; + FILE *f; + + result = sysconf(_SC_PAGESIZE); + if (result == -1) { + return 1; + } + f = fopen("conftest.out", "w"); + if (f == NULL) { + return 1; + } + fprintf(f, "%u\n", ffs((int)result) - 1); + close(f); + + return 0; +]])], + [STATIC_PAGE_SHIFT=`cat conftest.out`] + AC_MSG_RESULT([$STATIC_PAGE_SHIFT]) + AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$STATIC_PAGE_SHIFT]), + AC_MSG_RESULT([error])) + +dnl ============================================================================ +dnl jemalloc configuration. +dnl + +dnl Set VERSION if source directory has an embedded git repository. +if test -d "${srcroot}.git" ; then + git describe --long --abbrev=40 > ${srcroot}VERSION +fi +jemalloc_version=`cat ${srcroot}VERSION` +jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` +jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` +jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` +jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` +jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` +AC_SUBST([jemalloc_version]) +AC_SUBST([jemalloc_version_major]) +AC_SUBST([jemalloc_version_minor]) +AC_SUBST([jemalloc_version_bugfix]) +AC_SUBST([jemalloc_version_nrev]) +AC_SUBST([jemalloc_version_gid]) + +dnl ============================================================================ +dnl Configure pthreads. + +AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) +AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], + [AC_MSG_ERROR([libpthread is missing])]) + +CPPFLAGS="$CPPFLAGS -D_REENTRANT" + +dnl Enable lazy locking by default. +AC_ARG_ENABLE([lazy_lock], + [AS_HELP_STRING([--disable-lazy-lock], + [Disable lazy locking (always lock, even when single-threaded)])], +[if test "x$enable_lazy_lock" = "xno" ; then + enable_lazy_lock="0" +else + enable_lazy_lock="1" +fi +], +[enable_lazy_lock="1"] +) +if test "x$enable_lazy_lock" = "x1" ; then + AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) + AC_CHECK_LIB([dl], [dlopen], [LIBS="$LIBS -ldl"], + [AC_MSG_ERROR([libdl is missing])]) + AC_DEFINE([JEMALLOC_LAZY_LOCK], [ ]) +fi +AC_SUBST([enable_lazy_lock]) + +AC_ARG_ENABLE([tls], + [AS_HELP_STRING([--disable-tls], [Disable thread-local storage (__thread keyword)])], +if test "x$enable_tls" = "xno" ; then + enable_tls="0" +else + enable_tls="1" +fi +, +enable_tls="1" +) +if test "x${enable_tls}" = "x1" ; then +AC_MSG_CHECKING([for TLS]) +AC_RUN_IFELSE([AC_LANG_PROGRAM( +[[ + __thread int x; +]], [[ + x = 42; + + return 0; +]])], + AC_MSG_RESULT([yes]), + AC_MSG_RESULT([no]) + enable_tls="0") +fi +AC_SUBST([enable_tls]) +if test "x${enable_tls}" = "x0" ; then + AC_DEFINE_UNQUOTED([NO_TLS], [ ]) +fi + +dnl ============================================================================ +dnl Check for ffsl(3), and fail if not found. This function exists on all +dnl platforms that jemalloc currently has a chance of functioning on without +dnl modification. + +AC_CHECK_FUNC([ffsl], [], + [AC_MSG_ERROR([Cannot build without ffsl(3)])]) + +dnl ============================================================================ +dnl Check for atomic(3) operations as provided on Darwin. + +JE_COMPILABLE([Darwin OSAtomic*()], [ +#include +#include +], [ + { + int32_t x32 = 0; + volatile int32_t *x32p = &x32; + OSAtomicAdd32(1, x32p); + } + { + int64_t x64 = 0; + volatile int64_t *x64p = &x64; + OSAtomicAdd64(1, x64p); + } +], [osatomic]) +if test "x${osatomic}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSATOMIC]) +fi + +dnl ============================================================================ +dnl Check for spinlock(3) operations as provided on Darwin. + +JE_COMPILABLE([Darwin OSSpin*()], [ +#include +#include +], [ + OSSpinLock lock = 0; + OSSpinLockLock(&lock); + OSSpinLockUnlock(&lock); +], [osspin]) +if test "x${osspin}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSSPIN]) +fi + +dnl ============================================================================ +dnl Check for allocator-related functions that should be wrapped. + +AC_CHECK_FUNC([memalign], + [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN])]) +AC_CHECK_FUNC([valloc], + [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC])]) + +dnl ============================================================================ +dnl Darwin-related configuration. + +if test "x${abi}" = "xmacho" ; then + AC_DEFINE([JEMALLOC_IVSALLOC]) + AC_DEFINE([JEMALLOC_ZONE]) + + dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6 + dnl releases. malloc_zone_t and malloc_introspection_t have new fields in + dnl 10.6, which is the only source-level indication of the change. + AC_MSG_CHECKING([malloc zone version]) + AC_TRY_COMPILE([#include +#include ], [ + static malloc_zone_t zone; + static struct malloc_introspection_t zone_introspect; + + zone.size = NULL; + zone.malloc = NULL; + zone.calloc = NULL; + zone.valloc = NULL; + zone.free = NULL; + zone.realloc = NULL; + zone.destroy = NULL; + zone.zone_name = "jemalloc_zone"; + zone.batch_malloc = NULL; + zone.batch_free = NULL; + zone.introspect = &zone_introspect; + zone.version = 6; + zone.memalign = NULL; + zone.free_definite_size = NULL; + + zone_introspect.enumerator = NULL; + zone_introspect.good_size = NULL; + zone_introspect.check = NULL; + zone_introspect.print = NULL; + zone_introspect.log = NULL; + zone_introspect.force_lock = NULL; + zone_introspect.force_unlock = NULL; + zone_introspect.statistics = NULL; + zone_introspect.zone_locked = NULL; +], [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [6]) + AC_MSG_RESULT([6])], + [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [3]) + AC_MSG_RESULT([3])]) +fi + +dnl ============================================================================ +dnl Check for typedefs, structures, and compiler characteristics. +AC_HEADER_STDBOOL + +dnl Process .in files. +AC_SUBST([cfghdrs_in]) +AC_SUBST([cfghdrs_out]) +AC_CONFIG_HEADERS([$cfghdrs_tup]) + +dnl ============================================================================ +dnl Generate outputs. +AC_CONFIG_FILES([$cfgoutputs_tup config.stamp]) +AC_SUBST([cfgoutputs_in]) +AC_SUBST([cfgoutputs_out]) +AC_OUTPUT + +dnl ============================================================================ +dnl Print out the results of configuration. +AC_MSG_RESULT([===============================================================================]) +AC_MSG_RESULT([jemalloc version : $jemalloc_version]) +AC_MSG_RESULT([]) +AC_MSG_RESULT([CC : ${CC}]) +AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) +AC_MSG_RESULT([CFLAGS : ${CFLAGS}]) +AC_MSG_RESULT([LDFLAGS : ${LDFLAGS}]) +AC_MSG_RESULT([LIBS : ${LIBS}]) +AC_MSG_RESULT([RPATH_EXTRA : ${RPATH_EXTRA}]) +AC_MSG_RESULT([]) +AC_MSG_RESULT([XSLTPROC : ${XSLTPROC}]) +AC_MSG_RESULT([XSLROOT : ${XSLROOT}]) +AC_MSG_RESULT([]) +AC_MSG_RESULT([PREFIX : ${PREFIX}]) +AC_MSG_RESULT([BINDIR : ${BINDIR}]) +AC_MSG_RESULT([INCLUDEDIR : ${INCLUDEDIR}]) +AC_MSG_RESULT([LIBDIR : ${LIBDIR}]) +AC_MSG_RESULT([DATADIR : ${DATADIR}]) +AC_MSG_RESULT([MANDIR : ${MANDIR}]) +AC_MSG_RESULT([]) +AC_MSG_RESULT([srcroot : ${srcroot}]) +AC_MSG_RESULT([abs_srcroot : ${abs_srcroot}]) +AC_MSG_RESULT([objroot : ${objroot}]) +AC_MSG_RESULT([abs_objroot : ${abs_objroot}]) +AC_MSG_RESULT([]) +AC_MSG_RESULT([JEMALLOC_PREFIX : ${JEMALLOC_PREFIX}]) +AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) +AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) +AC_MSG_RESULT([install_suffix : ${install_suffix}]) +AC_MSG_RESULT([autogen : ${enable_autogen}]) +AC_MSG_RESULT([cc-silence : ${enable_cc_silence}]) +AC_MSG_RESULT([debug : ${enable_debug}]) +AC_MSG_RESULT([stats : ${enable_stats}]) +AC_MSG_RESULT([prof : ${enable_prof}]) +AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) +AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) +AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) +AC_MSG_RESULT([tiny : ${enable_tiny}]) +AC_MSG_RESULT([tcache : ${enable_tcache}]) +AC_MSG_RESULT([fill : ${enable_fill}]) +AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) +AC_MSG_RESULT([sysv : ${enable_sysv}]) +AC_MSG_RESULT([swap : ${enable_swap}]) +AC_MSG_RESULT([dss : ${enable_dss}]) +AC_MSG_RESULT([dynamic_page_shift : ${enable_dynamic_page_shift}]) +AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) +AC_MSG_RESULT([tls : ${enable_tls}]) +AC_MSG_RESULT([===============================================================================]) diff --git a/deps/jemalloc.orig/doc/html.xsl.in b/deps/jemalloc.orig/doc/html.xsl.in new file mode 100644 index 00000000000..a91d9746f62 --- /dev/null +++ b/deps/jemalloc.orig/doc/html.xsl.in @@ -0,0 +1,4 @@ + + + + diff --git a/deps/jemalloc.orig/doc/jemalloc.xml.in b/deps/jemalloc.orig/doc/jemalloc.xml.in new file mode 100644 index 00000000000..7a32879abee --- /dev/null +++ b/deps/jemalloc.orig/doc/jemalloc.xml.in @@ -0,0 +1,2280 @@ + + + + + + + User Manual + jemalloc + @jemalloc_version@ + + + Jason + Evans + Author + + + + + JEMALLOC + 3 + + + jemalloc + jemalloc + + general purpose memory allocation functions + + + LIBRARY + This manual describes jemalloc @jemalloc_version@. More information + can be found at the jemalloc website. + + + SYNOPSIS + + #include <stdlib.h> +#include <jemalloc/jemalloc.h> + + Standard API + + void *malloc + size_t size + + + void *calloc + size_t number + size_t size + + + int posix_memalign + void **ptr + size_t alignment + size_t size + + + void *realloc + void *ptr + size_t size + + + void free + void *ptr + + + + Non-standard API + + size_t malloc_usable_size + const void *ptr + + + void malloc_stats_print + void (*write_cb) + void *, const char * + + void *cbopaque + const char *opts + + + int mallctl + const char *name + void *oldp + size_t *oldlenp + void *newp + size_t newlen + + + int mallctlnametomib + const char *name + size_t *mibp + size_t *miblenp + + + int mallctlbymib + const size_t *mib + size_t miblen + void *oldp + size_t *oldlenp + void *newp + size_t newlen + + + void (*malloc_message) + void *cbopaque + const char *s + + const char *malloc_conf; + + + Experimental API + + int allocm + void **ptr + size_t *rsize + size_t size + int flags + + + int rallocm + void **ptr + size_t *rsize + size_t size + size_t extra + int flags + + + int sallocm + const void *ptr + size_t *rsize + int flags + + + int dallocm + void *ptr + int flags + + + + + + DESCRIPTION + + Standard API + + The malloc function allocates + size bytes of uninitialized memory. The allocated + space is suitably aligned (after possible pointer coercion) for storage + of any type of object. + + The calloc function allocates + space for number objects, each + size bytes in length. The result is identical to + calling malloc with an argument of + number * size, with the + exception that the allocated memory is explicitly initialized to zero + bytes. + + The posix_memalign function + allocates size bytes of memory such that the + allocation's base address is an even multiple of + alignment, and returns the allocation in the value + pointed to by ptr. The requested + alignment must be a power of 2 at least as large + as sizeof(void *). + + The realloc function changes the + size of the previously allocated memory referenced by + ptr to size bytes. The + contents of the memory are unchanged up to the lesser of the new and old + sizes. If the new size is larger, the contents of the newly allocated + portion of the memory are undefined. Upon success, the memory referenced + by ptr is freed and a pointer to the newly + allocated memory is returned. Note that + realloc may move the memory allocation, + resulting in a different return value than ptr. + If ptr is NULL, the + realloc function behaves identically to + malloc for the specified size. + + The free function causes the + allocated memory referenced by ptr to be made + available for future allocations. If ptr is + NULL, no action occurs. + + + Non-standard API + + The malloc_usable_size function + returns the usable size of the allocation pointed to by + ptr. The return value may be larger than the size + that was requested during allocation. The + malloc_usable_size function is not a + mechanism for in-place realloc; rather + it is provided solely as a tool for introspection purposes. Any + discrepancy between the requested allocation size and the size reported + by malloc_usable_size should not be + depended on, since such behavior is entirely implementation-dependent. + + + The malloc_stats_print function + writes human-readable summary statistics via the + write_cb callback function pointer and + cbopaque data passed to + write_cb, or + malloc_message if + write_cb is NULL. This + function can be called repeatedly. General information that never + changes during execution can be omitted by specifying "g" as a character + within the opts string. Note that + malloc_message uses the + mallctl* functions internally, so + inconsistent statistics can be reported if multiple threads use these + functions simultaneously. If is + specified during configuration, “m” and “a” can + be specified to omit merged arena and per arena statistics, respectively; + “b” and “l” can be specified to omit per size + class statistics for bins and large objects, respectively. Unrecognized + characters are silently ignored. Note that thread caching may prevent + some statistics from being completely up to date, since extra locking + would be required to merge counters that track thread cache operations. + + + The mallctl function provides a + general interface for introspecting the memory allocator, as well as + setting modifiable parameters and triggering actions. The + period-separated name argument specifies a + location in a tree-structured namespace; see the section for + documentation on the tree contents. To read a value, pass a pointer via + oldp to adequate space to contain the value, and a + pointer to its length via oldlenp; otherwise pass + NULL and NULL. Similarly, to + write a value, pass a pointer to the value via + newp, and its length via + newlen; otherwise pass NULL + and 0. + + The mallctlnametomib function + provides a way to avoid repeated name lookups for applications that + repeatedly query the same portion of the namespace, by translating a name + to a “Management Information Base” (MIB) that can be passed + repeatedly to mallctlbymib. Upon + successful return from mallctlnametomib, + mibp contains an array of + *miblenp integers, where + *miblenp is the lesser of the number of components + in name and the input value of + *miblenp. Thus it is possible to pass a + *miblenp that is smaller than the number of + period-separated name components, which results in a partial MIB that can + be used as the basis for constructing a complete MIB. For name + components that are integers (e.g. the 2 in + arenas.bin.2.size), + the corresponding MIB component will always be that integer. Therefore, + it is legitimate to construct code like the following: + + + Experimental API + The experimental API is subject to change or removal without regard + for backward compatibility. + + The allocm, + rallocm, + sallocm, and + dallocm functions all have a + flags argument that can be used to specify + options. The functions only check the options that are contextually + relevant. Use bitwise or (|) operations to + specify one or more of the following: + + + ALLOCM_LG_ALIGN(la) + + + Align the memory allocation to start at an address + that is a multiple of (1 << + la). This macro does not validate + that la is within the valid + range. + + + ALLOCM_ALIGN(a) + + + Align the memory allocation to start at an address + that is a multiple of a, where + a is a power of two. This macro does not + validate that a is a power of 2. + + + + ALLOCM_ZERO + + Initialize newly allocated memory to contain zero + bytes. In the growing reallocation case, the real size prior to + reallocation defines the boundary between untouched bytes and those + that are initialized to contain zero bytes. If this option is + absent, newly allocated memory is uninitialized. + + + ALLOCM_NO_MOVE + + For reallocation, fail rather than moving the + object. This constraint can apply to both growth and + shrinkage. + + + + + The allocm function allocates at + least size bytes of memory, sets + *ptr to the base address of the allocation, and + sets *rsize to the real size of the allocation if + rsize is not NULL. + + The rallocm function resizes the + allocation at *ptr to be at least + size bytes, sets *ptr to + the base address of the allocation if it moved, and sets + *rsize to the real size of the allocation if + rsize is not NULL. If + extra is non-zero, an attempt is made to resize + the allocation to be at least size + + extra) bytes, though inability to allocate + the extra byte(s) will not by itself result in failure. Behavior is + undefined if (size + + extra > + SIZE_T_MAX). + + The sallocm function sets + *rsize to the real size of the allocation. + + The dallocm function causes the + memory referenced by ptr to be made available for + future allocations. + + + + TUNING + Once, when the first call is made to one of the memory allocation + routines, the allocator initializes its internals based in part on various + options that can be specified at compile- or run-time. + + The string pointed to by the global variable + malloc_conf, the “name” of the file + referenced by the symbolic link named /etc/malloc.conf, and the value of the + environment variable MALLOC_CONF, will be interpreted, in + that order, from left to right as options. + + An options string is a comma-separated list of option:value pairs. + There is one key corresponding to each opt.* mallctl (see the section for options + documentation). For example, abort:true,narenas:1 sets + the opt.abort and opt.narenas options. Some + options have boolean values (true/false), others have integer values (base + 8, 10, or 16, depending on prefix), and yet others have raw string + values. + + + IMPLEMENTATION NOTES + Traditionally, allocators have used + sbrk + 2 to obtain memory, which is + suboptimal for several reasons, including race conditions, increased + fragmentation, and artificial limitations on maximum usable memory. If + is specified during configuration, this + allocator uses both sbrk + 2 and + mmap + 2, in that order of preference; + otherwise only mmap + 2 is used. + + This allocator uses multiple arenas in order to reduce lock + contention for threaded programs on multi-processor systems. This works + well with regard to threading scalability, but incurs some costs. There is + a small fixed per-arena overhead, and additionally, arenas manage memory + completely independently of each other, which means a small fixed increase + in overall memory fragmentation. These overheads are not generally an + issue, given the number of arenas normally used. Note that using + substantially more arenas than the default is not likely to improve + performance, mainly due to reduced cache performance. However, it may make + sense to reduce the number of arenas if an application does not make much + use of the allocation functions. + + In addition to multiple arenas, unless + is specified during configuration, this + allocator supports thread-specific caching for small and large objects, in + order to make it possible to completely avoid synchronization for most + allocation requests. Such caching allows very fast allocation in the + common case, but it increases memory usage and fragmentation, since a + bounded number of objects can remain allocated in each thread cache. + + Memory is conceptually broken into equal-sized chunks, where the + chunk size is a power of two that is greater than the page size. Chunks + are always aligned to multiples of the chunk size. This alignment makes it + possible to find metadata for user objects very quickly. + + User objects are broken into three categories according to size: + small, large, and huge. Small objects are smaller than one page. Large + objects are smaller than the chunk size. Huge objects are a multiple of + the chunk size. Small and large objects are managed by arenas; huge + objects are managed separately in a single data structure that is shared by + all threads. Huge objects are used by applications infrequently enough + that this single data structure is not a scalability issue. + + Each chunk that is managed by an arena tracks its contents as runs of + contiguous pages (unused, backing a set of small objects, or backing one + large object). The combination of chunk alignment and chunk page maps + makes it possible to determine all metadata regarding small and large + allocations in constant time. + + Small objects are managed in groups by page runs. Each run maintains + a frontier and free list to track which regions are in use. Unless + is specified during configuration, + allocation requests that are no more than half the quantum (8 or 16, + depending on architecture) are rounded up to the nearest power of two that + is at least sizeof(void *). + Allocation requests that are more than half the quantum, but no more than + the minimum cacheline-multiple size class (see the opt.lg_qspace_max + option) are rounded up to the nearest multiple of the quantum. Allocation + requests that are more than the minimum cacheline-multiple size class, but + no more than the minimum subpage-multiple size class (see the opt.lg_cspace_max + option) are rounded up to the nearest multiple of the cacheline size (64). + Allocation requests that are more than the minimum subpage-multiple size + class, but no more than the maximum subpage-multiple size class are rounded + up to the nearest multiple of the subpage size (256). Allocation requests + that are more than the maximum subpage-multiple size class, but small + enough to fit in an arena-managed chunk (see the opt.lg_chunk option), are + rounded up to the nearest run size. Allocation requests that are too large + to fit in an arena-managed chunk are rounded up to the nearest multiple of + the chunk size. + + Allocations are packed tightly together, which can be an issue for + multi-threaded applications. If you need to assure that allocations do not + suffer from cacheline sharing, round your allocation requests up to the + nearest multiple of the cacheline size, or specify cacheline alignment when + allocating. + + Assuming 4 MiB chunks, 4 KiB pages, and a 16-byte quantum on a 64-bit + system, the size classes in each category are as shown in . + + + Size classes + + + + + + + Category + Subcategory + Size + + + + + Small + Tiny + [8] + + + Quantum-spaced + [16, 32, 48, ..., 128] + + + Cacheline-spaced + [192, 256, 320, ..., 512] + + + Subpage-spaced + [768, 1024, 1280, ..., 3840] + + + Large + [4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB] + + + Huge + [4 MiB, 8 MiB, 12 MiB, ...] + + + +
+ + + MALLCTL NAMESPACE + The following names are defined in the namespace accessible via the + mallctl* functions. Value types are + specified in parentheses, their readable/writable statuses are encoded as + rw, r-, -w, or + --, and required build configuration flags follow, if + any. A name element encoded as <i> or + <j> indicates an integer component, where the + integer varies from 0 to some upper value that must be determined via + introspection. In the case of stats.arenas.<i>.*, + <i> equal to arenas.narenas can be + used to access the summation of statistics from all arenas. Take special + note of the epoch mallctl, + which controls refreshing of cached dynamic statistics. + + + + + version + (const char *) + r- + + Return the jemalloc version string. + + + + + epoch + (uint64_t) + rw + + If a value is passed in, refresh the data from which + the mallctl* functions report values, + and increment the epoch. Return the current epoch. This is useful for + detecting whether another thread caused a refresh. + + + + + config.debug + (bool) + r- + + was specified during + build configuration. + + + + + config.dss + (bool) + r- + + was specified during + build configuration. + + + + + config.dynamic_page_shift + (bool) + r- + + was + specified during build configuration. + + + + + config.fill + (bool) + r- + + was specified during + build configuration. + + + + + config.lazy_lock + (bool) + r- + + was specified + during build configuration. + + + + + config.prof + (bool) + r- + + was specified during + build configuration. + + + + + config.prof_libgcc + (bool) + r- + + was not + specified during build configuration. + + + + + config.prof_libunwind + (bool) + r- + + was specified + during build configuration. + + + + + config.stats + (bool) + r- + + was specified during + build configuration. + + + + + config.swap + (bool) + r- + + was specified during + build configuration. + + + + + config.sysv + (bool) + r- + + was specified during + build configuration. + + + + + config.tcache + (bool) + r- + + was not specified + during build configuration. + + + + + config.tiny + (bool) + r- + + was not specified + during build configuration. + + + + + config.tls + (bool) + r- + + was not specified during + build configuration. + + + + + config.xmalloc + (bool) + r- + + was specified during + build configuration. + + + + + opt.abort + (bool) + r- + + Abort-on-warning enabled/disabled. If true, most + warnings are fatal. The process will call + abort + 3 in these cases. This option is + disabled by default unless is + specified during configuration, in which case it is enabled by default. + + + + + + opt.lg_qspace_max + (size_t) + r- + + Size (log base 2) of the maximum size class that is a + multiple of the quantum (8 or 16 bytes, depending on architecture). + Above this size, cacheline spacing is used for size classes. The + default value is 128 bytes (2^7). + + + + + opt.lg_cspace_max + (size_t) + r- + + Size (log base 2) of the maximum size class that is a + multiple of the cacheline size (64). Above this size, subpage spacing + (256 bytes) is used for size classes. The default value is 512 bytes + (2^9). + + + + + opt.lg_chunk + (size_t) + r- + + Virtual memory chunk size (log base 2). The default + chunk size is 4 MiB (2^22). + + + + + opt.narenas + (size_t) + r- + + Maximum number of arenas to use. The default maximum + number of arenas is four times the number of CPUs, or one if there is a + single CPU. + + + + + opt.lg_dirty_mult + (ssize_t) + r- + + Per-arena minimum ratio (log base 2) of active to dirty + pages. Some dirty unused pages may be allowed to accumulate, within + the limit set by the ratio (or one chunk worth of dirty pages, + whichever is greater), before informing the kernel about some of those + pages via madvise + 2 or a similar system call. This + provides the kernel with sufficient information to recycle dirty pages + if physical memory becomes scarce and the pages remain unused. The + default minimum ratio is 32:1 (2^5:1); an option value of -1 will + disable dirty page purging. + + + + + opt.stats_print + (bool) + r- + + Enable/disable statistics printing at exit. If + enabled, the malloc_stats_print + function is called at program exit via an + atexit + 3 function. If + is specified during configuration, this + has the potential to cause deadlock for a multi-threaded process that + exits while one or more threads are executing in the memory allocation + functions. Therefore, this option should only be used with care; it is + primarily intended as a performance tuning aid during application + development. This option is disabled by default. + + + + + opt.junk + (bool) + r- + [] + + Junk filling enabled/disabled. If enabled, each byte + of uninitialized allocated memory will be initialized to + 0xa5. All deallocated memory will be initialized to + 0x5a. This is intended for debugging and will + impact performance negatively. This option is disabled by default + unless is specified during + configuration, in which case it is enabled by default. + + + + + opt.zero + (bool) + r- + [] + + Zero filling enabled/disabled. If enabled, each byte + of uninitialized allocated memory will be initialized to 0. Note that + this initialization only happens once for each byte, so + realloc and + rallocm calls do not zero memory that + was previously allocated. This is intended for debugging and will + impact performance negatively. This option is disabled by default. + + + + + + opt.sysv + (bool) + r- + [] + + If enabled, attempting to allocate zero bytes will + return a NULL pointer instead of a valid pointer. + (The default behavior is to make a minimal allocation and return a + pointer to it.) This option is provided for System V compatibility. + This option is incompatible with the opt.xmalloc option. + This option is disabled by default. + + + + + opt.xmalloc + (bool) + r- + [] + + Abort-on-out-of-memory enabled/disabled. If enabled, + rather than returning failure for any allocation function, display a + diagnostic message on STDERR_FILENO and cause the + program to drop core (using + abort + 3). If an application is + designed to depend on this behavior, set the option at compile time by + including the following in the source code: + + This option is disabled by default. + + + + + opt.tcache + (bool) + r- + [] + + Thread-specific caching enabled/disabled. When there + are multiple threads, each thread uses a thread-specific cache for + objects up to a certain size. Thread-specific caching allows many + allocations to be satisfied without performing any thread + synchronization, at the cost of increased memory use. See the + opt.lg_tcache_gc_sweep + and opt.lg_tcache_max + options for related tuning information. This option is enabled by + default. + + + + + opt.lg_tcache_gc_sweep + (ssize_t) + r- + [] + + Approximate interval (log base 2) between full + thread-specific cache garbage collection sweeps, counted in terms of + thread-specific cache allocation/deallocation events. Garbage + collection is actually performed incrementally, one size class at a + time, in order to avoid large collection pauses. The default sweep + interval is 8192 (2^13); setting this option to -1 will disable garbage + collection. + + + + + opt.lg_tcache_max + (size_t) + r- + [] + + Maximum size class (log base 2) to cache in the + thread-specific cache. At a minimum, all small size classes are + cached, and at a maximum all large size classes are cached. The + default maximum is 32 KiB (2^15). + + + + + opt.prof + (bool) + r- + [] + + Memory profiling enabled/disabled. If enabled, profile + memory allocation activity, and use an + atexit + 3 function to dump final memory + usage to a file named according to the pattern + <prefix>.<pid>.<seq>.f.heap, + where <prefix> is controlled by the opt.prof_prefix + option. See the opt.lg_prof_bt_max + option for backtrace depth control. See the opt.prof_active + option for on-the-fly activation/deactivation. See the opt.lg_prof_sample + option for probabilistic sampling control. See the opt.prof_accum + option for control of cumulative sample reporting. See the opt.lg_prof_tcmax + option for control of per thread backtrace caching. See the opt.lg_prof_interval + option for information on interval-triggered profile dumping, and the + opt.prof_gdump + option for information on high-water-triggered profile dumping. + Profile output is compatible with the included pprof + Perl script, which originates from the google-perftools + package. + + + + + opt.prof_prefix + (const char *) + r- + [] + + Filename prefix for profile dumps. If the prefix is + set to the empty string, no automatic dumps will occur; this is + primarily useful for disabling the automatic final heap dump (which + also disables leak reporting, if enabled). The default prefix is + jeprof. + + + + + opt.lg_prof_bt_max + (size_t) + r- + [] + + Maximum backtrace depth (log base 2) when profiling + memory allocation activity. The default is 128 (2^7). + + + + + opt.prof_active + (bool) + r- + [] + + Profiling activated/deactivated. This is a secondary + control mechanism that makes it possible to start the application with + profiling enabled (see the opt.prof option) but + inactive, then toggle profiling at any time during program execution + with the prof.active mallctl. + This option is enabled by default. + + + + + opt.lg_prof_sample + (ssize_t) + r- + [] + + Average interval (log base 2) between allocation + samples, as measured in bytes of allocation activity. Increasing the + sampling interval decreases profile fidelity, but also decreases the + computational overhead. The default sample interval is 1 (2^0) (i.e. + all allocations are sampled). + + + + + opt.prof_accum + (bool) + r- + [] + + Reporting of cumulative object/byte counts in profile + dumps enabled/disabled. If this option is enabled, every unique + backtrace must be stored for the duration of execution. Depending on + the application, this can impose a large memory overhead, and the + cumulative counts are not always of interest. See the + opt.lg_prof_tcmax + option for control of per thread backtrace caching, which has important + interactions. This option is enabled by default. + + + + + opt.lg_prof_tcmax + (ssize_t) + r- + [] + + Maximum per thread backtrace cache (log base 2) used + for heap profiling. A backtrace can only be discarded if the + opt.prof_accum + option is disabled, and no thread caches currently refer to the + backtrace. Therefore, a backtrace cache limit should be imposed if the + intention is to limit how much memory is used by backtraces. By + default, no limit is imposed (encoded as -1). + + + + + + opt.lg_prof_interval + (ssize_t) + r- + [] + + Average interval (log base 2) between memory profile + dumps, as measured in bytes of allocation activity. The actual + interval between dumps may be sporadic because decentralized allocation + counters are used to avoid synchronization bottlenecks. Profiles are + dumped to files named according to the pattern + <prefix>.<pid>.<seq>.i<iseq>.heap, + where <prefix> is controlled by the + opt.prof_prefix + option. By default, interval-triggered profile dumping is disabled + (encoded as -1). + + + + + + opt.prof_gdump + (bool) + r- + [] + + Trigger a memory profile dump every time the total + virtual memory exceeds the previous maximum. Profiles are dumped to + files named according to the pattern + <prefix>.<pid>.<seq>.u<useq>.heap, + where <prefix> is controlled by the opt.prof_prefix + option. This option is disabled by default. + + + + + opt.prof_leak + (bool) + r- + [] + + Leak reporting enabled/disabled. If enabled, use an + atexit + 3 function to report memory leaks + detected by allocation sampling. See the + opt.lg_prof_bt_max + option for backtrace depth control. See the + opt.prof option for + information on analyzing heap profile output. This option is disabled + by default. + + + + + opt.overcommit + (bool) + r- + [] + + Over-commit enabled/disabled. If enabled, over-commit + memory as a side effect of using anonymous + mmap + 2 or + sbrk + 2 for virtual memory allocation. + In order for overcommit to be disabled, the swap.fds mallctl must have + been successfully written to. This option is enabled by + default. + + + + + tcache.flush + (void) + -- + [] + + Flush calling thread's tcache. This interface releases + all cached objects and internal data structures associated with the + calling thread's thread-specific cache. Ordinarily, this interface + need not be called, since automatic periodic incremental garbage + collection occurs, and the thread cache is automatically discarded when + a thread exits. However, garbage collection is triggered by allocation + activity, so it is possible for a thread that stops + allocating/deallocating to retain its cache indefinitely, in which case + the developer may find manual flushing useful. + + + + + thread.arena + (unsigned) + rw + + Get or set the arena associated with the calling + thread. The arena index must be less than the maximum number of arenas + (see the arenas.narenas + mallctl). If the specified arena was not initialized beforehand (see + the arenas.initialized + mallctl), it will be automatically initialized as a side effect of + calling this interface. + + + + + thread.allocated + (uint64_t) + r- + [] + + Get the total number of bytes ever allocated by the + calling thread. This counter has the potential to wrap around; it is + up to the application to appropriately interpret the counter in such + cases. + + + + + thread.allocatedp + (uint64_t *) + r- + [] + + Get a pointer to the the value that is returned by the + thread.allocated + mallctl. This is useful for avoiding the overhead of repeated + mallctl* calls. + + + + + thread.deallocated + (uint64_t) + r- + [] + + Get the total number of bytes ever deallocated by the + calling thread. This counter has the potential to wrap around; it is + up to the application to appropriately interpret the counter in such + cases. + + + + + thread.deallocatedp + (uint64_t *) + r- + [] + + Get a pointer to the the value that is returned by the + thread.deallocated + mallctl. This is useful for avoiding the overhead of repeated + mallctl* calls. + + + + + arenas.narenas + (unsigned) + r- + + Maximum number of arenas. + + + + + arenas.initialized + (bool *) + r- + + An array of arenas.narenas + booleans. Each boolean indicates whether the corresponding arena is + initialized. + + + + + arenas.quantum + (size_t) + r- + + Quantum size. + + + + + arenas.cacheline + (size_t) + r- + + Assumed cacheline size. + + + + + arenas.subpage + (size_t) + r- + + Subpage size class interval. + + + + + arenas.pagesize + (size_t) + r- + + Page size. + + + + + arenas.chunksize + (size_t) + r- + + Chunk size. + + + + + arenas.tspace_min + (size_t) + r- + + Minimum tiny size class. Tiny size classes are powers + of two. + + + + + arenas.tspace_max + (size_t) + r- + + Maximum tiny size class. Tiny size classes are powers + of two. + + + + + arenas.qspace_min + (size_t) + r- + + Minimum quantum-spaced size class. + + + + + arenas.qspace_max + (size_t) + r- + + Maximum quantum-spaced size class. + + + + + arenas.cspace_min + (size_t) + r- + + Minimum cacheline-spaced size class. + + + + + arenas.cspace_max + (size_t) + r- + + Maximum cacheline-spaced size class. + + + + + arenas.sspace_min + (size_t) + r- + + Minimum subpage-spaced size class. + + + + + arenas.sspace_max + (size_t) + r- + + Maximum subpage-spaced size class. + + + + + arenas.tcache_max + (size_t) + r- + [] + + Maximum thread-cached size class. + + + + + arenas.ntbins + (unsigned) + r- + + Number of tiny bin size classes. + + + + + arenas.nqbins + (unsigned) + r- + + Number of quantum-spaced bin size + classes. + + + + + arenas.ncbins + (unsigned) + r- + + Number of cacheline-spaced bin size + classes. + + + + + arenas.nsbins + (unsigned) + r- + + Number of subpage-spaced bin size + classes. + + + + + arenas.nbins + (unsigned) + r- + + Total number of bin size classes. + + + + + arenas.nhbins + (unsigned) + r- + [] + + Total number of thread cache bin size + classes. + + + + + arenas.bin.<i>.size + (size_t) + r- + + Maximum size supported by size class. + + + + + arenas.bin.<i>.nregs + (uint32_t) + r- + + Number of regions per page run. + + + + + arenas.bin.<i>.run_size + (size_t) + r- + + Number of bytes per page run. + + + + + arenas.nlruns + (size_t) + r- + + Total number of large size classes. + + + + + arenas.lrun.<i>.size + (size_t) + r- + + Maximum size supported by this large size + class. + + + + + arenas.purge + (unsigned) + -w + + Purge unused dirty pages for the specified arena, or + for all arenas if none is specified. + + + + + prof.active + (bool) + rw + [] + + Control whether sampling is currently active. See the + opt.prof_active + option for additional information. + + + + + + prof.dump + (const char *) + -w + [] + + Dump a memory profile to the specified file, or if NULL + is specified, to a file according to the pattern + <prefix>.<pid>.<seq>.m<mseq>.heap, + where <prefix> is controlled by the + opt.prof_prefix + option. + + + + + prof.interval + (uint64_t) + r- + [] + + Average number of bytes allocated between + inverval-based profile dumps. See the + opt.lg_prof_interval + option for additional information. + + + + + stats.cactive + (size_t *) + r- + [] + + Pointer to a counter that contains an approximate count + of the current number of bytes in active pages. The estimate may be + high, but never low, because each arena rounds up to the nearest + multiple of the chunk size when computing its contribution to the + counter. Note that the epoch mallctl has no bearing + on this counter. Furthermore, counter consistency is maintained via + atomic operations, so it is necessary to use an atomic operation in + order to guarantee a consistent read when dereferencing the pointer. + + + + + + stats.allocated + (size_t) + r- + [] + + Total number of bytes allocated by the + application. + + + + + stats.active + (size_t) + r- + [] + + Total number of bytes in active pages allocated by the + application. This is a multiple of the page size, and greater than or + equal to stats.allocated. + + + + + + stats.mapped + (size_t) + r- + [] + + Total number of bytes in chunks mapped on behalf of the + application. This is a multiple of the chunk size, and is at least as + large as stats.active. This + does not include inactive chunks backed by swap files. his does not + include inactive chunks embedded in the DSS. + + + + + stats.chunks.current + (size_t) + r- + [] + + Total number of chunks actively mapped on behalf of the + application. This does not include inactive chunks backed by swap + files. This does not include inactive chunks embedded in the DSS. + + + + + + stats.chunks.total + (uint64_t) + r- + [] + + Cumulative number of chunks allocated. + + + + + stats.chunks.high + (size_t) + r- + [] + + Maximum number of active chunks at any time thus far. + + + + + + stats.huge.allocated + (size_t) + r- + [] + + Number of bytes currently allocated by huge objects. + + + + + + stats.huge.nmalloc + (uint64_t) + r- + [] + + Cumulative number of huge allocation requests. + + + + + + stats.huge.ndalloc + (uint64_t) + r- + [] + + Cumulative number of huge deallocation requests. + + + + + + stats.arenas.<i>.nthreads + (unsigned) + r- + + Number of threads currently assigned to + arena. + + + + + stats.arenas.<i>.pactive + (size_t) + r- + + Number of pages in active runs. + + + + + stats.arenas.<i>.pdirty + (size_t) + r- + + Number of pages within unused runs that are potentially + dirty, and for which madvise... + MADV_DONTNEED or + similar has not been called. + + + + + stats.arenas.<i>.mapped + (size_t) + r- + [] + + Number of mapped bytes. + + + + + stats.arenas.<i>.npurge + (uint64_t) + r- + [] + + Number of dirty page purge sweeps performed. + + + + + + stats.arenas.<i>.nmadvise + (uint64_t) + r- + [] + + Number of madvise... + MADV_DONTNEED or + similar calls made to purge dirty pages. + + + + + stats.arenas.<i>.npurged + (uint64_t) + r- + [] + + Number of pages purged. + + + + + stats.arenas.<i>.small.allocated + (size_t) + r- + [] + + Number of bytes currently allocated by small objects. + + + + + + stats.arenas.<i>.small.nmalloc + (uint64_t) + r- + [] + + Cumulative number of allocation requests served by + small bins. + + + + + stats.arenas.<i>.small.ndalloc + (uint64_t) + r- + [] + + Cumulative number of small objects returned to bins. + + + + + + stats.arenas.<i>.small.nrequests + (uint64_t) + r- + [] + + Cumulative number of small allocation requests. + + + + + + stats.arenas.<i>.large.allocated + (size_t) + r- + [] + + Number of bytes currently allocated by large objects. + + + + + + stats.arenas.<i>.large.nmalloc + (uint64_t) + r- + [] + + Cumulative number of large allocation requests served + directly by the arena. + + + + + stats.arenas.<i>.large.ndalloc + (uint64_t) + r- + [] + + Cumulative number of large deallocation requests served + directly by the arena. + + + + + stats.arenas.<i>.large.nrequests + (uint64_t) + r- + [] + + Cumulative number of large allocation requests. + + + + + + stats.arenas.<i>.bins.<j>.allocated + (size_t) + r- + [] + + Current number of bytes allocated by + bin. + + + + + stats.arenas.<i>.bins.<j>.nmalloc + (uint64_t) + r- + [] + + Cumulative number of allocations served by bin. + + + + + + stats.arenas.<i>.bins.<j>.ndalloc + (uint64_t) + r- + [] + + Cumulative number of allocations returned to bin. + + + + + + stats.arenas.<i>.bins.<j>.nrequests + (uint64_t) + r- + [] + + Cumulative number of allocation + requests. + + + + + stats.arenas.<i>.bins.<j>.nfills + (uint64_t) + r- + [ ] + + Cumulative number of tcache fills. + + + + + stats.arenas.<i>.bins.<j>.nflushes + (uint64_t) + r- + [ ] + + Cumulative number of tcache flushes. + + + + + stats.arenas.<i>.bins.<j>.nruns + (uint64_t) + r- + [] + + Cumulative number of runs created. + + + + + stats.arenas.<i>.bins.<j>.nreruns + (uint64_t) + r- + [] + + Cumulative number of times the current run from which + to allocate changed. + + + + + stats.arenas.<i>.bins.<j>.highruns + (size_t) + r- + [] + + Maximum number of runs at any time thus far. + + + + + + stats.arenas.<i>.bins.<j>.curruns + (size_t) + r- + [] + + Current number of runs. + + + + + stats.arenas.<i>.lruns.<j>.nmalloc + (uint64_t) + r- + [] + + Cumulative number of allocation requests for this size + class served directly by the arena. + + + + + stats.arenas.<i>.lruns.<j>.ndalloc + (uint64_t) + r- + [] + + Cumulative number of deallocation requests for this + size class served directly by the arena. + + + + + stats.arenas.<i>.lruns.<j>.nrequests + (uint64_t) + r- + [] + + Cumulative number of allocation requests for this size + class. + + + + + stats.arenas.<i>.lruns.<j>.highruns + (size_t) + r- + [] + + Maximum number of runs at any time thus far for this + size class. + + + + + stats.arenas.<i>.lruns.<j>.curruns + (size_t) + r- + [] + + Current number of runs for this size class. + + + + + + swap.avail + (size_t) + r- + [] + + Number of swap file bytes that are currently not + associated with any chunk (i.e. mapped, but otherwise completely + unmanaged). + + + + + swap.prezeroed + (bool) + rw + [] + + If true, the allocator assumes that the swap file(s) + contain nothing but nil bytes. If this assumption is violated, + allocator behavior is undefined. This value becomes read-only after + swap.fds is + successfully written to. + + + + + swap.nfds + (size_t) + r- + [] + + Number of file descriptors in use for swap. + + + + + + swap.fds + (int *) + rw + [] + + When written to, the files associated with the + specified file descriptors are contiguously mapped via + mmap + 2. The resulting virtual memory + region is preferred over anonymous + mmap + 2 and + sbrk + 2 memory. Note that if a file's + size is not a multiple of the page size, it is automatically truncated + to the nearest page size multiple. See the + swap.prezeroed + mallctl for specifying that the files are pre-zeroed. + + + + + DEBUGGING MALLOC PROBLEMS + When debugging, it is a good idea to configure/build jemalloc with + the and + options, and recompile the program with suitable options and symbols for + debugger support. When so configured, jemalloc incorporates a wide variety + of run-time assertions that catch application errors such as double-free, + write-after-free, etc. + + Programs often accidentally depend on “uninitialized” + memory actually being filled with zero bytes. Junk filling + (see the opt.junk + option) tends to expose such bugs in the form of obviously incorrect + results and/or coredumps. Conversely, zero + filling (see the opt.zero option) eliminates + the symptoms of such bugs. Between these two options, it is usually + possible to quickly detect, diagnose, and eliminate such bugs. + + This implementation does not provide much detail about the problems + it detects, because the performance impact for storing such information + would be prohibitive. There are a number of allocator implementations + available on the Internet which focus on detecting and pinpointing problems + by trading performance for extra sanity checks and detailed + diagnostics. + + + DIAGNOSTIC MESSAGES + If any of the memory allocation/deallocation functions detect an + error or warning condition, a message will be printed to file descriptor + STDERR_FILENO. Errors will result in the process + dumping core. If the opt.abort option is set, most + warnings are treated as errors. + + The malloc_message variable allows the programmer + to override the function which emits the text strings forming the errors + and warnings if for some reason the STDERR_FILENO file + descriptor is not suitable for this. + malloc_message takes the + cbopaque pointer argument that is + NULL unless overridden by the arguments in a call to + malloc_stats_print, followed by a string + pointer. Please note that doing anything which tries to allocate memory in + this function is likely to result in a crash or deadlock. + + All messages are prefixed by + “<jemalloc>: ”. + + + RETURN VALUES + + Standard API + The malloc and + calloc functions return a pointer to the + allocated memory if successful; otherwise a NULL + pointer is returned and errno is set to + ENOMEM. + + The posix_memalign function + returns the value 0 if successful; otherwise it returns an error value. + The posix_memalign function will fail + if: + + + EINVAL + + The alignment parameter is + not a power of 2 at least as large as + sizeof(void *). + + + + ENOMEM + + Memory allocation error. + + + + + The realloc function returns a + pointer, possibly identical to ptr, to the + allocated memory if successful; otherwise a NULL + pointer is returned, and errno is set to + ENOMEM if the error was the result of an + allocation failure. The realloc + function always leaves the original buffer intact when an error occurs. + + + The free function returns no + value. + + + Non-standard API + The malloc_usable_size function + returns the usable size of the allocation pointed to by + ptr. + + The mallctl, + mallctlnametomib, and + mallctlbymib functions return 0 on + success; otherwise they return an error value. The functions will fail + if: + + + EINVAL + + newp is not + NULL, and newlen is too + large or too small. Alternatively, *oldlenp + is too large or too small; in this case as much data as possible + are read despite the error. + + + ENOMEM + + *oldlenp is too short to + hold the requested value. + + + ENOENT + + name or + mib specifies an unknown/invalid + value. + + + EPERM + + Attempt to read or write void value, or attempt to + write read-only value. + + + EAGAIN + + A memory allocation failure + occurred. + + + EFAULT + + An interface with side effects failed in some way + not directly related to mallctl* + read/write processing. + + + + + + Experimental API + The allocm, + rallocm, + sallocm, and + dallocm functions return + ALLOCM_SUCCESS on success; otherwise they return an + error value. The allocm and + rallocm functions will fail if: + + + ALLOCM_ERR_OOM + + Out of memory. Insufficient contiguous memory was + available to service the allocation request. The + allocm function additionally sets + *ptr to NULL, whereas + the rallocm function leaves + *ptr unmodified. + + + The rallocm function will also + fail if: + + + ALLOCM_ERR_NOT_MOVED + + ALLOCM_NO_MOVE was specified, + but the reallocation request could not be serviced without moving + the object. + + + + + + + ENVIRONMENT + The following environment variable affects the execution of the + allocation functions: + + + MALLOC_CONF + + If the environment variable + MALLOC_CONF is set, the characters it contains + will be interpreted as options. + + + + + + EXAMPLES + To dump core whenever a problem occurs: + ln -s 'abort:true' /etc/malloc.conf + + To specify in the source a chunk size that is 16 MiB: + + + + SEE ALSO + madvise + 2, + mmap + 2, + sbrk + 2, + alloca + 3, + atexit + 3, + getpagesize + 3 + + + STANDARDS + The malloc, + calloc, + realloc, and + free functions conform to ISO/IEC + 9899:1990 (“ISO C90”). + + The posix_memalign function conforms + to IEEE Std 1003.1-2001 (“POSIX.1”). + + diff --git a/deps/jemalloc.orig/doc/manpages.xsl.in b/deps/jemalloc.orig/doc/manpages.xsl.in new file mode 100644 index 00000000000..88b2626b958 --- /dev/null +++ b/deps/jemalloc.orig/doc/manpages.xsl.in @@ -0,0 +1,4 @@ + + + + diff --git a/deps/jemalloc.orig/doc/stylesheet.xsl b/deps/jemalloc.orig/doc/stylesheet.xsl new file mode 100644 index 00000000000..4e334a86f87 --- /dev/null +++ b/deps/jemalloc.orig/doc/stylesheet.xsl @@ -0,0 +1,7 @@ + + ansi + + + "" + + diff --git a/deps/jemalloc.orig/include/jemalloc/internal/arena.h b/deps/jemalloc.orig/include/jemalloc/internal/arena.h new file mode 100644 index 00000000000..b80c118d811 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/arena.h @@ -0,0 +1,743 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* + * Subpages are an artificially designated partitioning of pages. Their only + * purpose is to support subpage-spaced size classes. + * + * There must be at least 4 subpages per page, due to the way size classes are + * handled. + */ +#define LG_SUBPAGE 8 +#define SUBPAGE ((size_t)(1U << LG_SUBPAGE)) +#define SUBPAGE_MASK (SUBPAGE - 1) + +/* Return the smallest subpage multiple that is >= s. */ +#define SUBPAGE_CEILING(s) \ + (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK) + +#ifdef JEMALLOC_TINY + /* Smallest size class to support. */ +# define LG_TINY_MIN LG_SIZEOF_PTR +# define TINY_MIN (1U << LG_TINY_MIN) +#endif + +/* + * Maximum size class that is a multiple of the quantum, but not (necessarily) + * a power of 2. Above this size, allocations are rounded up to the nearest + * power of 2. + */ +#define LG_QSPACE_MAX_DEFAULT 7 + +/* + * Maximum size class that is a multiple of the cacheline, but not (necessarily) + * a power of 2. Above this size, allocations are rounded up to the nearest + * power of 2. + */ +#define LG_CSPACE_MAX_DEFAULT 9 + +/* + * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized + * as small as possible such that this setting is still honored, without + * violating other constraints. The goal is to make runs as small as possible + * without exceeding a per run external fragmentation threshold. + * + * We use binary fixed point math for overhead computations, where the binary + * point is implicitly RUN_BFP bits to the left. + * + * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be + * honored for some/all object sizes, since when heap profiling is enabled + * there is one pointer of header overhead per object (plus a constant). This + * constraint is relaxed (ignored) for runs that are so small that the + * per-region overhead is greater than: + * + * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) + */ +#define RUN_BFP 12 +/* \/ Implicit binary fixed point. */ +#define RUN_MAX_OVRHD 0x0000003dU +#define RUN_MAX_OVRHD_RELAX 0x00001800U + +/* Maximum number of regions in one run. */ +#define LG_RUN_MAXREGS 11 +#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) + +/* + * The minimum ratio of active:dirty pages per arena is computed as: + * + * (nactive >> opt_lg_dirty_mult) >= ndirty + * + * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 + * times as many active pages as dirty pages. + */ +#define LG_DIRTY_MULT_DEFAULT 5 + +typedef struct arena_chunk_map_s arena_chunk_map_t; +typedef struct arena_chunk_s arena_chunk_t; +typedef struct arena_run_s arena_run_t; +typedef struct arena_bin_info_s arena_bin_info_t; +typedef struct arena_bin_s arena_bin_t; +typedef struct arena_s arena_t; + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +/* Each element of the chunk map corresponds to one page within the chunk. */ +struct arena_chunk_map_s { + union { + /* + * Linkage for run trees. There are two disjoint uses: + * + * 1) arena_t's runs_avail_{clean,dirty} trees. + * 2) arena_run_t conceptually uses this linkage for in-use + * non-full runs, rather than directly embedding linkage. + */ + rb_node(arena_chunk_map_t) rb_link; + /* + * List of runs currently in purgatory. arena_chunk_purge() + * temporarily allocates runs that contain dirty pages while + * purging, so that other threads cannot use the runs while the + * purging thread is operating without the arena lock held. + */ + ql_elm(arena_chunk_map_t) ql_link; + } u; + +#ifdef JEMALLOC_PROF + /* Profile counters, used for large object runs. */ + prof_ctx_t *prof_ctx; +#endif + + /* + * Run address (or size) and various flags are stored together. The bit + * layout looks like (assuming 32-bit system): + * + * ???????? ???????? ????---- ----dula + * + * ? : Unallocated: Run address for first/last pages, unset for internal + * pages. + * Small: Run page offset. + * Large: Run size for first page, unset for trailing pages. + * - : Unused. + * d : dirty? + * u : unzeroed? + * l : large? + * a : allocated? + * + * Following are example bit patterns for the three types of runs. + * + * p : run page offset + * s : run size + * c : (binind+1) for size class (used only if prof_promote is true) + * x : don't care + * - : 0 + * + : 1 + * [DULA] : bit set + * [dula] : bit unset + * + * Unallocated (clean): + * ssssssss ssssssss ssss---- ----du-a + * xxxxxxxx xxxxxxxx xxxx---- -----Uxx + * ssssssss ssssssss ssss---- ----dU-a + * + * Unallocated (dirty): + * ssssssss ssssssss ssss---- ----D--a + * xxxxxxxx xxxxxxxx xxxx---- ----xxxx + * ssssssss ssssssss ssss---- ----D--a + * + * Small: + * pppppppp pppppppp pppp---- ----d--A + * pppppppp pppppppp pppp---- -------A + * pppppppp pppppppp pppp---- ----d--A + * + * Large: + * ssssssss ssssssss ssss---- ----D-LA + * xxxxxxxx xxxxxxxx xxxx---- ----xxxx + * -------- -------- -------- ----D-LA + * + * Large (sampled, size <= PAGE_SIZE): + * ssssssss ssssssss sssscccc ccccD-LA + * + * Large (not sampled, size == PAGE_SIZE): + * ssssssss ssssssss ssss---- ----D-LA + */ + size_t bits; +#ifdef JEMALLOC_PROF +#define CHUNK_MAP_CLASS_SHIFT 4 +#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U) +#endif +#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU) +#define CHUNK_MAP_DIRTY ((size_t)0x8U) +#define CHUNK_MAP_UNZEROED ((size_t)0x4U) +#define CHUNK_MAP_LARGE ((size_t)0x2U) +#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) +#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED +}; +typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; +typedef rb_tree(arena_chunk_map_t) arena_run_tree_t; + +/* Arena chunk header. */ +struct arena_chunk_s { + /* Arena that owns the chunk. */ + arena_t *arena; + + /* Linkage for the arena's chunks_dirty list. */ + ql_elm(arena_chunk_t) link_dirty; + + /* + * True if the chunk is currently in the chunks_dirty list, due to + * having at some point contained one or more dirty pages. Removal + * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. + */ + bool dirtied; + + /* Number of dirty pages. */ + size_t ndirty; + + /* + * Map of pages within chunk that keeps track of free/large/small. The + * first map_bias entries are omitted, since the chunk header does not + * need to be tracked in the map. This omission saves a header page + * for common chunk sizes (e.g. 4 MiB). + */ + arena_chunk_map_t map[1]; /* Dynamically sized. */ +}; +typedef rb_tree(arena_chunk_t) arena_chunk_tree_t; + +struct arena_run_s { +#ifdef JEMALLOC_DEBUG + uint32_t magic; +# define ARENA_RUN_MAGIC 0x384adf93 +#endif + + /* Bin this run is associated with. */ + arena_bin_t *bin; + + /* Index of next region that has never been allocated, or nregs. */ + uint32_t nextind; + + /* Number of free regions in run. */ + unsigned nfree; +}; + +/* + * Read-only information associated with each element of arena_t's bins array + * is stored separately, partly to reduce memory usage (only one copy, rather + * than one per arena), but mainly to avoid false cacheline sharing. + */ +struct arena_bin_info_s { + /* Size of regions in a run for this bin's size class. */ + size_t reg_size; + + /* Total size of a run for this bin's size class. */ + size_t run_size; + + /* Total number of regions in a run for this bin's size class. */ + uint32_t nregs; + + /* + * Offset of first bitmap_t element in a run header for this bin's size + * class. + */ + uint32_t bitmap_offset; + + /* + * Metadata used to manipulate bitmaps for runs associated with this + * bin. + */ + bitmap_info_t bitmap_info; + +#ifdef JEMALLOC_PROF + /* + * Offset of first (prof_ctx_t *) in a run header for this bin's size + * class, or 0 if (opt_prof == false). + */ + uint32_t ctx0_offset; +#endif + + /* Offset of first region in a run for this bin's size class. */ + uint32_t reg0_offset; +}; + +struct arena_bin_s { + /* + * All operations on runcur, runs, and stats require that lock be + * locked. Run allocation/deallocation are protected by the arena lock, + * which may be acquired while holding one or more bin locks, but not + * vise versa. + */ + malloc_mutex_t lock; + + /* + * Current run being used to service allocations of this bin's size + * class. + */ + arena_run_t *runcur; + + /* + * Tree of non-full runs. This tree is used when looking for an + * existing run when runcur is no longer usable. We choose the + * non-full run that is lowest in memory; this policy tends to keep + * objects packed well, and it can also help reduce the number of + * almost-empty chunks. + */ + arena_run_tree_t runs; + +#ifdef JEMALLOC_STATS + /* Bin statistics. */ + malloc_bin_stats_t stats; +#endif +}; + +struct arena_s { +#ifdef JEMALLOC_DEBUG + uint32_t magic; +# define ARENA_MAGIC 0x947d3d24 +#endif + + /* This arena's index within the arenas array. */ + unsigned ind; + + /* + * Number of threads currently assigned to this arena. This field is + * protected by arenas_lock. + */ + unsigned nthreads; + + /* + * There are three classes of arena operations from a locking + * perspective: + * 1) Thread asssignment (modifies nthreads) is protected by + * arenas_lock. + * 2) Bin-related operations are protected by bin locks. + * 3) Chunk- and run-related operations are protected by this mutex. + */ + malloc_mutex_t lock; + +#ifdef JEMALLOC_STATS + arena_stats_t stats; +# ifdef JEMALLOC_TCACHE + /* + * List of tcaches for extant threads associated with this arena. + * Stats from these are merged incrementally, and at exit. + */ + ql_head(tcache_t) tcache_ql; +# endif +#endif + +#ifdef JEMALLOC_PROF + uint64_t prof_accumbytes; +#endif + + /* List of dirty-page-containing chunks this arena manages. */ + ql_head(arena_chunk_t) chunks_dirty; + + /* + * In order to avoid rapid chunk allocation/deallocation when an arena + * oscillates right on the cusp of needing a new chunk, cache the most + * recently freed chunk. The spare is left in the arena's chunk trees + * until it is deleted. + * + * There is one spare chunk per arena, rather than one spare total, in + * order to avoid interactions between multiple threads that could make + * a single spare inadequate. + */ + arena_chunk_t *spare; + + /* Number of pages in active runs. */ + size_t nactive; + + /* + * Current count of pages within unused runs that are potentially + * dirty, and for which madvise(... MADV_DONTNEED) has not been called. + * By tracking this, we can institute a limit on how much dirty unused + * memory is mapped for each arena. + */ + size_t ndirty; + + /* + * Approximate number of pages being purged. It is possible for + * multiple threads to purge dirty pages concurrently, and they use + * npurgatory to indicate the total number of pages all threads are + * attempting to purge. + */ + size_t npurgatory; + + /* + * Size/address-ordered trees of this arena's available runs. The trees + * are used for first-best-fit run allocation. The dirty tree contains + * runs with dirty pages (i.e. very likely to have been touched and + * therefore have associated physical pages), whereas the clean tree + * contains runs with pages that either have no associated physical + * pages, or have pages that the kernel may recycle at any time due to + * previous madvise(2) calls. The dirty tree is used in preference to + * the clean tree for allocations, because using dirty pages reduces + * the amount of dirty purging necessary to keep the active:dirty page + * ratio below the purge threshold. + */ + arena_avail_tree_t runs_avail_clean; + arena_avail_tree_t runs_avail_dirty; + + /* + * bins is used to store trees of free regions of the following sizes, + * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and + * default MALLOC_CONF. + * + * bins[i] | size | + * --------+--------+ + * 0 | 8 | + * --------+--------+ + * 1 | 16 | + * 2 | 32 | + * 3 | 48 | + * : : + * 6 | 96 | + * 7 | 112 | + * 8 | 128 | + * --------+--------+ + * 9 | 192 | + * 10 | 256 | + * 11 | 320 | + * 12 | 384 | + * 13 | 448 | + * 14 | 512 | + * --------+--------+ + * 15 | 768 | + * 16 | 1024 | + * 17 | 1280 | + * : : + * 25 | 3328 | + * 26 | 3584 | + * 27 | 3840 | + * --------+--------+ + */ + arena_bin_t bins[1]; /* Dynamically sized. */ +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +extern size_t opt_lg_qspace_max; +extern size_t opt_lg_cspace_max; +extern ssize_t opt_lg_dirty_mult; +/* + * small_size2bin is a compact lookup table that rounds request sizes up to + * size classes. In order to reduce cache footprint, the table is compressed, + * and all accesses are via the SMALL_SIZE2BIN macro. + */ +extern uint8_t const *small_size2bin; +#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) + +extern arena_bin_info_t *arena_bin_info; + +/* Various bin-related settings. */ +#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ +# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN)) +#else +# define ntbins 0 +#endif +extern unsigned nqbins; /* Number of quantum-spaced bins. */ +extern unsigned ncbins; /* Number of cacheline-spaced bins. */ +extern unsigned nsbins; /* Number of subpage-spaced bins. */ +extern unsigned nbins; +#ifdef JEMALLOC_TINY +# define tspace_max ((size_t)(QUANTUM >> 1)) +#endif +#define qspace_min QUANTUM +extern size_t qspace_max; +extern size_t cspace_min; +extern size_t cspace_max; +extern size_t sspace_min; +extern size_t sspace_max; +#define small_maxclass sspace_max + +#define nlclasses (chunk_npages - map_bias) + +void arena_purge_all(arena_t *arena); +#ifdef JEMALLOC_PROF +void arena_prof_accum(arena_t *arena, uint64_t accumbytes); +#endif +#ifdef JEMALLOC_TCACHE +void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, + size_t binind +# ifdef JEMALLOC_PROF + , uint64_t prof_accumbytes +# endif + ); +#endif +void *arena_malloc_small(arena_t *arena, size_t size, bool zero); +void *arena_malloc_large(arena_t *arena, size_t size, bool zero); +void *arena_malloc(size_t size, bool zero); +void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size, + size_t alignment, bool zero); +size_t arena_salloc(const void *ptr); +#ifdef JEMALLOC_PROF +void arena_prof_promoted(const void *ptr, size_t size); +size_t arena_salloc_demote(const void *ptr); +#endif +void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, + arena_chunk_map_t *mapelm); +void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); +#ifdef JEMALLOC_STATS +void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, + arena_stats_t *astats, malloc_bin_stats_t *bstats, + malloc_large_stats_t *lstats); +#endif +void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, + size_t extra, bool zero); +void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero); +bool arena_new(arena_t *arena, unsigned ind); +bool arena_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); +unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, + const void *ptr); +# ifdef JEMALLOC_PROF +prof_ctx_t *arena_prof_ctx_get(const void *ptr); +void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +# endif +void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) +JEMALLOC_INLINE size_t +arena_bin_index(arena_t *arena, arena_bin_t *bin) +{ + size_t binind = bin - arena->bins; + assert(binind < nbins); + return (binind); +} + +JEMALLOC_INLINE unsigned +arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) +{ + unsigned shift, diff, regind; + size_t size; + + dassert(run->magic == ARENA_RUN_MAGIC); + /* + * Freeing a pointer lower than region zero can cause assertion + * failure. + */ + assert((uintptr_t)ptr >= (uintptr_t)run + + (uintptr_t)bin_info->reg0_offset); + + /* + * Avoid doing division with a variable divisor if possible. Using + * actual division here can reduce allocator throughput by over 20%! + */ + diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - + bin_info->reg0_offset); + + /* Rescale (factor powers of 2 out of the numerator and denominator). */ + size = bin_info->reg_size; + shift = ffs(size) - 1; + diff >>= shift; + size >>= shift; + + if (size == 1) { + /* The divisor was a power of 2. */ + regind = diff; + } else { + /* + * To divide by a number D that is not a power of two we + * multiply by (2^21 / D) and then right shift by 21 positions. + * + * X / D + * + * becomes + * + * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT + * + * We can omit the first three elements, because we never + * divide by 0, and 1 and 2 are both powers of two, which are + * handled above. + */ +#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) +#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) + static const unsigned size_invs[] = { + SIZE_INV(3), + SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), + SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), + SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), + SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), + SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), + SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), + SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) + }; + + if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) + regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; + else + regind = diff / size; +#undef SIZE_INV +#undef SIZE_INV_SHIFT + } + assert(diff == regind * size); + assert(regind < bin_info->nregs); + + return (regind); +} + +#ifdef JEMALLOC_PROF +JEMALLOC_INLINE prof_ctx_t * +arena_prof_ctx_get(const void *ptr) +{ + prof_ctx_t *ret; + arena_chunk_t *chunk; + size_t pageind, mapbits; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + mapbits = chunk->map[pageind-map_bias].bits; + assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); + if ((mapbits & CHUNK_MAP_LARGE) == 0) { + if (prof_promote) + ret = (prof_ctx_t *)(uintptr_t)1U; + else { + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << + PAGE_SHIFT)); + size_t binind = arena_bin_index(chunk->arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + unsigned regind; + + dassert(run->magic == ARENA_RUN_MAGIC); + regind = arena_run_regind(run, bin_info, ptr); + ret = *(prof_ctx_t **)((uintptr_t)run + + bin_info->ctx0_offset + (regind * + sizeof(prof_ctx_t *))); + } + } else + ret = chunk->map[pageind-map_bias].prof_ctx; + + return (ret); +} + +JEMALLOC_INLINE void +arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +{ + arena_chunk_t *chunk; + size_t pageind, mapbits; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + mapbits = chunk->map[pageind-map_bias].bits; + assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); + if ((mapbits & CHUNK_MAP_LARGE) == 0) { + if (prof_promote == false) { + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << + PAGE_SHIFT)); + arena_bin_t *bin = run->bin; + size_t binind; + arena_bin_info_t *bin_info; + unsigned regind; + + dassert(run->magic == ARENA_RUN_MAGIC); + binind = arena_bin_index(chunk->arena, bin); + bin_info = &arena_bin_info[binind]; + regind = arena_run_regind(run, bin_info, ptr); + + *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset + + (regind * sizeof(prof_ctx_t *)))) = ctx; + } else + assert((uintptr_t)ctx == (uintptr_t)1U); + } else + chunk->map[pageind-map_bias].prof_ctx = ctx; +} +#endif + +JEMALLOC_INLINE void +arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) +{ + size_t pageind; + arena_chunk_map_t *mapelm; + + assert(arena != NULL); + dassert(arena->magic == ARENA_MAGIC); + assert(chunk->arena == arena); + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + mapelm = &chunk->map[pageind-map_bias]; + assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); + if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { + /* Small allocation. */ +#ifdef JEMALLOC_TCACHE + tcache_t *tcache; + + if ((tcache = tcache_get()) != NULL) + tcache_dalloc_small(tcache, ptr); + else { +#endif + arena_run_t *run; + arena_bin_t *bin; + + run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - (mapelm->bits >> + PAGE_SHIFT)) << PAGE_SHIFT)); + dassert(run->magic == ARENA_RUN_MAGIC); + bin = run->bin; +#ifdef JEMALLOC_DEBUG + { + size_t binind = arena_bin_index(arena, bin); + arena_bin_info_t *bin_info = + &arena_bin_info[binind]; + assert(((uintptr_t)ptr - ((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset)) % + bin_info->reg_size == 0); + } +#endif + malloc_mutex_lock(&bin->lock); + arena_dalloc_bin(arena, chunk, ptr, mapelm); + malloc_mutex_unlock(&bin->lock); +#ifdef JEMALLOC_TCACHE + } +#endif + } else { +#ifdef JEMALLOC_TCACHE + size_t size = mapelm->bits & ~PAGE_MASK; + + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + if (size <= tcache_maxclass) { + tcache_t *tcache; + + if ((tcache = tcache_get()) != NULL) + tcache_dalloc_large(tcache, ptr, size); + else { + malloc_mutex_lock(&arena->lock); + arena_dalloc_large(arena, chunk, ptr); + malloc_mutex_unlock(&arena->lock); + } + } else { + malloc_mutex_lock(&arena->lock); + arena_dalloc_large(arena, chunk, ptr); + malloc_mutex_unlock(&arena->lock); + } +#else + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + malloc_mutex_lock(&arena->lock); + arena_dalloc_large(arena, chunk, ptr); + malloc_mutex_unlock(&arena->lock); +#endif + } +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/atomic.h b/deps/jemalloc.orig/include/jemalloc/internal/atomic.h new file mode 100644 index 00000000000..9a298623f8a --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/atomic.h @@ -0,0 +1,169 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#define atomic_read_uint64(p) atomic_add_uint64(p, 0) +#define atomic_read_uint32(p) atomic_add_uint32(p, 0) + +#if (LG_SIZEOF_PTR == 3) +# define atomic_read_z(p) \ + (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0) +# define atomic_add_z(p, x) \ + (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x) +# define atomic_sub_z(p, x) \ + (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x) +#elif (LG_SIZEOF_PTR == 2) +# define atomic_read_z(p) \ + (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0) +# define atomic_add_z(p, x) \ + (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x) +# define atomic_sub_z(p, x) \ + (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x) +#endif + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); +uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); +uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); +uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) +/******************************************************************************/ +/* 64-bit operations. */ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (__sync_add_and_fetch(p, x)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (__sync_sub_and_fetch(p, x)); +} +#elif (defined(JEMALLOC_OSATOMIC)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); +} +#elif (defined(__amd64_) || defined(__x86_64__)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + asm volatile ( + "lock; xaddq %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + x = (uint64_t)(-(int64_t)x); + asm volatile ( + "lock; xaddq %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} +#else +# if (LG_SIZEOF_PTR == 3) +# error "Missing implementation for 64-bit atomic operations" +# endif +#endif + +/******************************************************************************/ +/* 32-bit operations. */ +#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (__sync_add_and_fetch(p, x)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (__sync_sub_and_fetch(p, x)); +} +#elif (defined(JEMALLOC_OSATOMIC)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); +} +#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + asm volatile ( + "lock; xaddl %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + x = (uint32_t)(-(int32_t)x); + asm volatile ( + "lock; xaddl %0, %1;" + : "+r" (x), "=m" (*p) /* Outputs. */ + : "m" (*p) /* Inputs. */ + ); + + return (x); +} +#else +# error "Missing implementation for 32-bit atomic operations" +#endif +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/base.h b/deps/jemalloc.orig/include/jemalloc/internal/base.h new file mode 100644 index 00000000000..e353f309bd2 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/base.h @@ -0,0 +1,24 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +extern malloc_mutex_t base_mtx; + +void *base_alloc(size_t size); +extent_node_t *base_node_alloc(void); +void base_node_dealloc(extent_node_t *node); +bool base_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/bitmap.h b/deps/jemalloc.orig/include/jemalloc/internal/bitmap.h new file mode 100644 index 00000000000..605ebac58c1 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/bitmap.h @@ -0,0 +1,184 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ +#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS + +typedef struct bitmap_level_s bitmap_level_t; +typedef struct bitmap_info_s bitmap_info_t; +typedef unsigned long bitmap_t; +#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG + +/* Number of bits per group. */ +#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3) +#define BITMAP_GROUP_NBITS (ZU(1) << LG_BITMAP_GROUP_NBITS) +#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1) + +/* Maximum number of levels possible. */ +#define BITMAP_MAX_LEVELS \ + (LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \ + + !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +struct bitmap_level_s { + /* Offset of this level's groups within the array of groups. */ + size_t group_offset; +}; + +struct bitmap_info_s { + /* Logical number of bits in bitmap (stored at bottom level). */ + size_t nbits; + + /* Number of levels necessary for nbits. */ + unsigned nlevels; + + /* + * Only the first (nlevels+1) elements are used, and levels are ordered + * bottom to top (e.g. the bottom level is stored in levels[0]). + */ + bitmap_level_t levels[BITMAP_MAX_LEVELS+1]; +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void bitmap_info_init(bitmap_info_t *binfo, size_t nbits); +size_t bitmap_info_ngroups(const bitmap_info_t *binfo); +size_t bitmap_size(size_t nbits); +void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo); +bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); +void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); +size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo); +void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_)) +JEMALLOC_INLINE bool +bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) +{ + unsigned rgoff = binfo->levels[binfo->nlevels].group_offset - 1; + bitmap_t rg = bitmap[rgoff]; + /* The bitmap is full iff the root group is 0. */ + return (rg == 0); +} + +JEMALLOC_INLINE bool +bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) +{ + size_t goff; + bitmap_t g; + + assert(bit < binfo->nbits); + goff = bit >> LG_BITMAP_GROUP_NBITS; + g = bitmap[goff]; + return (!(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)))); +} + +JEMALLOC_INLINE void +bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) +{ + size_t goff; + bitmap_t *gp; + bitmap_t g; + + assert(bit < binfo->nbits); + assert(bitmap_get(bitmap, binfo, bit) == false); + goff = bit >> LG_BITMAP_GROUP_NBITS; + gp = &bitmap[goff]; + g = *gp; + assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))); + g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); + *gp = g; + assert(bitmap_get(bitmap, binfo, bit)); + /* Propagate group state transitions up the tree. */ + if (g == 0) { + unsigned i; + for (i = 1; i < binfo->nlevels; i++) { + bit = goff; + goff = bit >> LG_BITMAP_GROUP_NBITS; + gp = &bitmap[binfo->levels[i].group_offset + goff]; + g = *gp; + assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))); + g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); + *gp = g; + if (g != 0) + break; + } + } +} + +/* sfu: set first unset. */ +JEMALLOC_INLINE size_t +bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) +{ + size_t bit; + bitmap_t g; + unsigned i; + + assert(bitmap_full(bitmap, binfo) == false); + + i = binfo->nlevels - 1; + g = bitmap[binfo->levels[i].group_offset]; + bit = ffsl(g) - 1; + while (i > 0) { + i--; + g = bitmap[binfo->levels[i].group_offset + bit]; + bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffsl(g) - 1); + } + + bitmap_set(bitmap, binfo, bit); + return (bit); +} + +JEMALLOC_INLINE void +bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) +{ + size_t goff; + bitmap_t *gp; + bitmap_t g; + bool propagate; + + assert(bit < binfo->nbits); + assert(bitmap_get(bitmap, binfo, bit)); + goff = bit >> LG_BITMAP_GROUP_NBITS; + gp = &bitmap[goff]; + g = *gp; + propagate = (g == 0); + assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0); + g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); + *gp = g; + assert(bitmap_get(bitmap, binfo, bit) == false); + /* Propagate group state transitions up the tree. */ + if (propagate) { + unsigned i; + for (i = 1; i < binfo->nlevels; i++) { + bit = goff; + goff = bit >> LG_BITMAP_GROUP_NBITS; + gp = &bitmap[binfo->levels[i].group_offset + goff]; + g = *gp; + propagate = (g == 0); + assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) + == 0); + g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); + *gp = g; + if (propagate == false) + break; + } + } +} + +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk.h new file mode 100644 index 00000000000..54b6a3ec886 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/chunk.h @@ -0,0 +1,65 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* + * Size and alignment of memory chunks that are allocated by the OS's virtual + * memory system. + */ +#define LG_CHUNK_DEFAULT 22 + +/* Return the chunk address for allocation address a. */ +#define CHUNK_ADDR2BASE(a) \ + ((void *)((uintptr_t)(a) & ~chunksize_mask)) + +/* Return the chunk offset of address a. */ +#define CHUNK_ADDR2OFFSET(a) \ + ((size_t)((uintptr_t)(a) & chunksize_mask)) + +/* Return the smallest chunk multiple that is >= s. */ +#define CHUNK_CEILING(s) \ + (((s) + chunksize_mask) & ~chunksize_mask) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +extern size_t opt_lg_chunk; +#ifdef JEMALLOC_SWAP +extern bool opt_overcommit; +#endif + +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) +/* Protects stats_chunks; currently not used for any other purpose. */ +extern malloc_mutex_t chunks_mtx; +/* Chunk statistics. */ +extern chunk_stats_t stats_chunks; +#endif + +#ifdef JEMALLOC_IVSALLOC +extern rtree_t *chunks_rtree; +#endif + +extern size_t chunksize; +extern size_t chunksize_mask; /* (chunksize - 1). */ +extern size_t chunk_npages; +extern size_t map_bias; /* Number of arena chunk header pages. */ +extern size_t arena_maxclass; /* Max size class for arenas. */ + +void *chunk_alloc(size_t size, bool base, bool *zero); +void chunk_dealloc(void *chunk, size_t size, bool unmap); +bool chunk_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + +#include "jemalloc/internal/chunk_swap.h" +#include "jemalloc/internal/chunk_dss.h" +#include "jemalloc/internal/chunk_mmap.h" diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h new file mode 100644 index 00000000000..6f005222181 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h @@ -0,0 +1,30 @@ +#ifdef JEMALLOC_DSS +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +/* + * Protects sbrk() calls. This avoids malloc races among threads, though it + * does not protect against races with threads that call sbrk() directly. + */ +extern malloc_mutex_t dss_mtx; + +void *chunk_alloc_dss(size_t size, bool *zero); +bool chunk_in_dss(void *chunk); +bool chunk_dealloc_dss(void *chunk, size_t size); +bool chunk_dss_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ +#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h new file mode 100644 index 00000000000..07b50a4dc37 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h @@ -0,0 +1,23 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void *chunk_alloc_mmap(size_t size); +void *chunk_alloc_mmap_noreserve(size_t size); +void chunk_dealloc_mmap(void *chunk, size_t size); + +bool chunk_mmap_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_swap.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk_swap.h similarity index 100% rename from deps/jemalloc/include/jemalloc/internal/chunk_swap.h rename to deps/jemalloc.orig/include/jemalloc/internal/chunk_swap.h diff --git a/deps/jemalloc.orig/include/jemalloc/internal/ckh.h b/deps/jemalloc.orig/include/jemalloc/internal/ckh.h new file mode 100644 index 00000000000..3e4ad4c85f9 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/ckh.h @@ -0,0 +1,95 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct ckh_s ckh_t; +typedef struct ckhc_s ckhc_t; + +/* Typedefs to allow easy function pointer passing. */ +typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *); +typedef bool ckh_keycomp_t (const void *, const void *); + +/* Maintain counters used to get an idea of performance. */ +/* #define CKH_COUNT */ +/* Print counter values in ckh_delete() (requires CKH_COUNT). */ +/* #define CKH_VERBOSE */ + +/* + * There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket. Try to fit + * one bucket per L1 cache line. + */ +#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +/* Hash table cell. */ +struct ckhc_s { + const void *key; + const void *data; +}; + +struct ckh_s { +#ifdef JEMALLOC_DEBUG +#define CKH_MAGIC 0x3af2489d + uint32_t magic; +#endif + +#ifdef CKH_COUNT + /* Counters used to get an idea of performance. */ + uint64_t ngrows; + uint64_t nshrinks; + uint64_t nshrinkfails; + uint64_t ninserts; + uint64_t nrelocs; +#endif + + /* Used for pseudo-random number generation. */ +#define CKH_A 1103515241 +#define CKH_C 12347 + uint32_t prn_state; + + /* Total number of items. */ + size_t count; + + /* + * Minimum and current number of hash table buckets. There are + * 2^LG_CKH_BUCKET_CELLS cells per bucket. + */ + unsigned lg_minbuckets; + unsigned lg_curbuckets; + + /* Hash and comparison functions. */ + ckh_hash_t *hash; + ckh_keycomp_t *keycomp; + + /* Hash table with 2^lg_curbuckets buckets. */ + ckhc_t *tab; +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +bool ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, + ckh_keycomp_t *keycomp); +void ckh_delete(ckh_t *ckh); +size_t ckh_count(ckh_t *ckh); +bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data); +bool ckh_insert(ckh_t *ckh, const void *key, const void *data); +bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key, + void **data); +bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data); +void ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, + size_t *hash2); +bool ckh_string_keycomp(const void *k1, const void *k2); +void ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, + size_t *hash2); +bool ckh_pointer_keycomp(const void *k1, const void *k2); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/ctl.h b/deps/jemalloc.orig/include/jemalloc/internal/ctl.h new file mode 100644 index 00000000000..f1f5eb70a2a --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/ctl.h @@ -0,0 +1,118 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct ctl_node_s ctl_node_t; +typedef struct ctl_arena_stats_s ctl_arena_stats_t; +typedef struct ctl_stats_s ctl_stats_t; + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +struct ctl_node_s { + bool named; + union { + struct { + const char *name; + /* If (nchildren == 0), this is a terminal node. */ + unsigned nchildren; + const ctl_node_t *children; + } named; + struct { + const ctl_node_t *(*index)(const size_t *, size_t, + size_t); + } indexed; + } u; + int (*ctl)(const size_t *, size_t, void *, size_t *, void *, + size_t); +}; + +struct ctl_arena_stats_s { + bool initialized; + unsigned nthreads; + size_t pactive; + size_t pdirty; +#ifdef JEMALLOC_STATS + arena_stats_t astats; + + /* Aggregate stats for small size classes, based on bin stats. */ + size_t allocated_small; + uint64_t nmalloc_small; + uint64_t ndalloc_small; + uint64_t nrequests_small; + + malloc_bin_stats_t *bstats; /* nbins elements. */ + malloc_large_stats_t *lstats; /* nlclasses elements. */ +#endif +}; + +struct ctl_stats_s { +#ifdef JEMALLOC_STATS + size_t allocated; + size_t active; + size_t mapped; + struct { + size_t current; /* stats_chunks.curchunks */ + uint64_t total; /* stats_chunks.nchunks */ + size_t high; /* stats_chunks.highchunks */ + } chunks; + struct { + size_t allocated; /* huge_allocated */ + uint64_t nmalloc; /* huge_nmalloc */ + uint64_t ndalloc; /* huge_ndalloc */ + } huge; +#endif + ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ +#ifdef JEMALLOC_SWAP + size_t swap_avail; +#endif +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +int ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, + size_t newlen); +int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp); + +int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen); +bool ctl_boot(void); + +#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ + if (JEMALLOC_P(mallctl)(name, oldp, oldlenp, newp, newlen) \ + != 0) { \ + malloc_write(": Failure in xmallctl(\""); \ + malloc_write(name); \ + malloc_write("\", ...)\n"); \ + abort(); \ + } \ +} while (0) + +#define xmallctlnametomib(name, mibp, miblenp) do { \ + if (JEMALLOC_P(mallctlnametomib)(name, mibp, miblenp) != 0) { \ + malloc_write( \ + ": Failure in xmallctlnametomib(\""); \ + malloc_write(name); \ + malloc_write("\", ...)\n"); \ + abort(); \ + } \ +} while (0) + +#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \ + if (JEMALLOC_P(mallctlbymib)(mib, miblen, oldp, oldlenp, newp, \ + newlen) != 0) { \ + malloc_write( \ + ": Failure in xmallctlbymib()\n"); \ + abort(); \ + } \ +} while (0) + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/deps/jemalloc.orig/include/jemalloc/internal/extent.h b/deps/jemalloc.orig/include/jemalloc/internal/extent.h new file mode 100644 index 00000000000..6fe9702b5f6 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/extent.h @@ -0,0 +1,49 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct extent_node_s extent_node_t; + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +/* Tree of extents. */ +struct extent_node_s { +#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) + /* Linkage for the size/address-ordered tree. */ + rb_node(extent_node_t) link_szad; +#endif + + /* Linkage for the address-ordered tree. */ + rb_node(extent_node_t) link_ad; + +#ifdef JEMALLOC_PROF + /* Profile counters, used for huge objects. */ + prof_ctx_t *prof_ctx; +#endif + + /* Pointer to the extent that this tree node is responsible for. */ + void *addr; + + /* Total region size. */ + size_t size; +}; +typedef rb_tree(extent_node_t) extent_tree_t; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) +rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t) +#endif + +rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/deps/jemalloc.orig/include/jemalloc/internal/hash.h b/deps/jemalloc.orig/include/jemalloc/internal/hash.h new file mode 100644 index 00000000000..8a46ce30803 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/hash.h @@ -0,0 +1,70 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +uint64_t hash(const void *key, size_t len, uint64_t seed); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_)) +/* + * The following hash function is based on MurmurHash64A(), placed into the + * public domain by Austin Appleby. See http://murmurhash.googlepages.com/ for + * details. + */ +JEMALLOC_INLINE uint64_t +hash(const void *key, size_t len, uint64_t seed) +{ + const uint64_t m = 0xc6a4a7935bd1e995LLU; + const int r = 47; + uint64_t h = seed ^ (len * m); + const uint64_t *data = (const uint64_t *)key; + const uint64_t *end = data + (len/8); + const unsigned char *data2; + + assert(((uintptr_t)key & 0x7) == 0); + + while(data != end) { + uint64_t k = *data++; + + k *= m; + k ^= k >> r; + k *= m; + + h ^= k; + h *= m; + } + + data2 = (const unsigned char *)data; + switch(len & 7) { + case 7: h ^= ((uint64_t)(data2[6])) << 48; + case 6: h ^= ((uint64_t)(data2[5])) << 40; + case 5: h ^= ((uint64_t)(data2[4])) << 32; + case 4: h ^= ((uint64_t)(data2[3])) << 24; + case 3: h ^= ((uint64_t)(data2[2])) << 16; + case 2: h ^= ((uint64_t)(data2[1])) << 8; + case 1: h ^= ((uint64_t)(data2[0])); + h *= m; + } + + h ^= h >> r; + h *= m; + h ^= h >> r; + + return (h); +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/huge.h b/deps/jemalloc.orig/include/jemalloc/internal/huge.h new file mode 100644 index 00000000000..66544cf8d97 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/huge.h @@ -0,0 +1,41 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#ifdef JEMALLOC_STATS +/* Huge allocation statistics. */ +extern uint64_t huge_nmalloc; +extern uint64_t huge_ndalloc; +extern size_t huge_allocated; +#endif + +/* Protects chunk-related data structures. */ +extern malloc_mutex_t huge_mtx; + +void *huge_malloc(size_t size, bool zero); +void *huge_palloc(size_t size, size_t alignment, bool zero); +void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, + size_t extra); +void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero); +void huge_dalloc(void *ptr, bool unmap); +size_t huge_salloc(const void *ptr); +#ifdef JEMALLOC_PROF +prof_ctx_t *huge_prof_ctx_get(const void *ptr); +void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +#endif +bool huge_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in b/deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in new file mode 100644 index 00000000000..a44f0978ae4 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in @@ -0,0 +1,788 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#ifndef SIZE_T_MAX +# define SIZE_T_MAX SIZE_MAX +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef offsetof +# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "../jemalloc@install_suffix@.h" + +#include "jemalloc/internal/private_namespace.h" + +#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#include +#endif + +#ifdef JEMALLOC_ZONE +#include +#include +#include +#include +#endif + +#ifdef JEMALLOC_LAZY_LOCK +#include +#endif + +#define RB_COMPACT +#include "jemalloc/internal/rb.h" +#include "jemalloc/internal/qr.h" +#include "jemalloc/internal/ql.h" + +extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); + +/* + * Define a custom assert() in order to reduce the chances of deadlock during + * assertion failure. + */ +#ifndef assert +# ifdef JEMALLOC_DEBUG +# define assert(e) do { \ + if (!(e)) { \ + char line_buf[UMAX2S_BUFSIZE]; \ + malloc_write(": "); \ + malloc_write(__FILE__); \ + malloc_write(":"); \ + malloc_write(u2s(__LINE__, 10, line_buf)); \ + malloc_write(": Failed assertion: "); \ + malloc_write("\""); \ + malloc_write(#e); \ + malloc_write("\"\n"); \ + abort(); \ + } \ +} while (0) +# else +# define assert(e) +# endif +#endif + +#ifdef JEMALLOC_DEBUG +# define dassert(e) assert(e) +#else +# define dassert(e) +#endif + +/* + * jemalloc can conceptually be broken into components (arena, tcache, etc.), + * but there are circular dependencies that cannot be broken without + * substantial performance degradation. In order to reduce the effect on + * visual code flow, read the header files in multiple passes, with one of the + * following cpp variables defined during each pass: + * + * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data + * types. + * JEMALLOC_H_STRUCTS : Data structures. + * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. + * JEMALLOC_H_INLINES : Inline functions. + */ +/******************************************************************************/ +#define JEMALLOC_H_TYPES + +#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) + +#define ZU(z) ((size_t)z) + +#ifndef __DECONST +# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) +#endif + +#ifdef JEMALLOC_DEBUG + /* Disable inlining to make debugging easier. */ +# define JEMALLOC_INLINE +# define inline +#else +# define JEMALLOC_ENABLE_INLINE +# define JEMALLOC_INLINE static inline +#endif + +/* Size of stack-allocated buffer passed to buferror(). */ +#define BUFERROR_BUF 64 + +/* Minimum alignment of allocations is 2^LG_QUANTUM bytes. */ +#ifdef __i386__ +# define LG_QUANTUM 4 +#endif +#ifdef __ia64__ +# define LG_QUANTUM 4 +#endif +#ifdef __alpha__ +# define LG_QUANTUM 4 +#endif +#ifdef __sparc64__ +# define LG_QUANTUM 4 +#endif +#if (defined(__amd64__) || defined(__x86_64__)) +# define LG_QUANTUM 4 +#endif +#ifdef __arm__ +# define LG_QUANTUM 3 +#endif +#ifdef __mips__ +# define LG_QUANTUM 3 +#endif +#ifdef __powerpc__ +# define LG_QUANTUM 4 +#endif +#ifdef __s390x__ +# define LG_QUANTUM 4 +#endif + +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) + +/* Return the smallest quantum multiple that is >= a. */ +#define QUANTUM_CEILING(a) \ + (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) + +#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) +#define LONG_MASK (LONG - 1) + +/* Return the smallest long multiple that is >= a. */ +#define LONG_CEILING(a) \ + (((a) + LONG_MASK) & ~LONG_MASK) + +#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) +#define PTR_MASK (SIZEOF_PTR - 1) + +/* Return the smallest (void *) multiple that is >= a. */ +#define PTR_CEILING(a) \ + (((a) + PTR_MASK) & ~PTR_MASK) + +/* + * Maximum size of L1 cache line. This is used to avoid cache line aliasing. + * In addition, this controls the spacing of cacheline-spaced size classes. + */ +#define LG_CACHELINE 6 +#define CACHELINE ((size_t)(1U << LG_CACHELINE)) +#define CACHELINE_MASK (CACHELINE - 1) + +/* Return the smallest cacheline multiple that is >= s. */ +#define CACHELINE_CEILING(s) \ + (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) + +/* + * Page size. STATIC_PAGE_SHIFT is determined by the configure script. If + * DYNAMIC_PAGE_SHIFT is enabled, only use the STATIC_PAGE_* macros where + * compile-time values are required for the purposes of defining data + * structures. + */ +#define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT)) +#define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1)) + +#ifdef PAGE_SHIFT +# undef PAGE_SHIFT +#endif +#ifdef PAGE_SIZE +# undef PAGE_SIZE +#endif +#ifdef PAGE_MASK +# undef PAGE_MASK +#endif + +#ifdef DYNAMIC_PAGE_SHIFT +# define PAGE_SHIFT lg_pagesize +# define PAGE_SIZE pagesize +# define PAGE_MASK pagesize_mask +#else +# define PAGE_SHIFT STATIC_PAGE_SHIFT +# define PAGE_SIZE STATIC_PAGE_SIZE +# define PAGE_MASK STATIC_PAGE_MASK +#endif + +/* Return the smallest pagesize multiple that is >= s. */ +#define PAGE_CEILING(s) \ + (((s) + PAGE_MASK) & ~PAGE_MASK) + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/stats.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/mb.h" +#include "jemalloc/internal/extent.h" +#include "jemalloc/internal/arena.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/chunk.h" +#include "jemalloc/internal/huge.h" +#include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/tcache.h" +#include "jemalloc/internal/hash.h" +#ifdef JEMALLOC_ZONE +#include "jemalloc/internal/zone.h" +#endif +#include "jemalloc/internal/prof.h" + +#undef JEMALLOC_H_TYPES +/******************************************************************************/ +#define JEMALLOC_H_STRUCTS + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/stats.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/mb.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/extent.h" +#include "jemalloc/internal/arena.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/chunk.h" +#include "jemalloc/internal/huge.h" +#include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/tcache.h" +#include "jemalloc/internal/hash.h" +#ifdef JEMALLOC_ZONE +#include "jemalloc/internal/zone.h" +#endif +#include "jemalloc/internal/prof.h" + +#ifdef JEMALLOC_STATS +typedef struct { + uint64_t allocated; + uint64_t deallocated; +} thread_allocated_t; +#endif + +#undef JEMALLOC_H_STRUCTS +/******************************************************************************/ +#define JEMALLOC_H_EXTERNS + +extern bool opt_abort; +#ifdef JEMALLOC_FILL +extern bool opt_junk; +#endif +#ifdef JEMALLOC_SYSV +extern bool opt_sysv; +#endif +#ifdef JEMALLOC_XMALLOC +extern bool opt_xmalloc; +#endif +#ifdef JEMALLOC_FILL +extern bool opt_zero; +#endif +extern size_t opt_narenas; + +#ifdef DYNAMIC_PAGE_SHIFT +extern size_t pagesize; +extern size_t pagesize_mask; +extern size_t lg_pagesize; +#endif + +/* Number of CPUs. */ +extern unsigned ncpus; + +extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ +extern pthread_key_t arenas_tsd; +#ifndef NO_TLS +/* + * Map of pthread_self() --> arenas[???], used for selecting an arena to use + * for allocations. + */ +extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); +# define ARENA_GET() arenas_tls +# define ARENA_SET(v) do { \ + arenas_tls = (v); \ + pthread_setspecific(arenas_tsd, (void *)(v)); \ +} while (0) +#else +# define ARENA_GET() ((arena_t *)pthread_getspecific(arenas_tsd)) +# define ARENA_SET(v) do { \ + pthread_setspecific(arenas_tsd, (void *)(v)); \ +} while (0) +#endif + +/* + * Arenas that are used to service external requests. Not all elements of the + * arenas array are necessarily used; arenas are created lazily as needed. + */ +extern arena_t **arenas; +extern unsigned narenas; + +#ifdef JEMALLOC_STATS +# ifndef NO_TLS +extern __thread thread_allocated_t thread_allocated_tls; +# define ALLOCATED_GET() (thread_allocated_tls.allocated) +# define ALLOCATEDP_GET() (&thread_allocated_tls.allocated) +# define DEALLOCATED_GET() (thread_allocated_tls.deallocated) +# define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated) +# define ALLOCATED_ADD(a, d) do { \ + thread_allocated_tls.allocated += a; \ + thread_allocated_tls.deallocated += d; \ +} while (0) +# else +extern pthread_key_t thread_allocated_tsd; +thread_allocated_t *thread_allocated_get_hard(void); + +# define ALLOCATED_GET() (thread_allocated_get()->allocated) +# define ALLOCATEDP_GET() (&thread_allocated_get()->allocated) +# define DEALLOCATED_GET() (thread_allocated_get()->deallocated) +# define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated) +# define ALLOCATED_ADD(a, d) do { \ + thread_allocated_t *thread_allocated = thread_allocated_get(); \ + thread_allocated->allocated += (a); \ + thread_allocated->deallocated += (d); \ +} while (0) +# endif +#endif + +arena_t *arenas_extend(unsigned ind); +arena_t *choose_arena_hard(void); +int buferror(int errnum, char *buf, size_t buflen); +void jemalloc_prefork(void); +void jemalloc_postfork(void); + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/stats.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/mb.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/extent.h" +#include "jemalloc/internal/arena.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/chunk.h" +#include "jemalloc/internal/huge.h" +#include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/tcache.h" +#include "jemalloc/internal/hash.h" +#ifdef JEMALLOC_ZONE +#include "jemalloc/internal/zone.h" +#endif +#include "jemalloc/internal/prof.h" + +#undef JEMALLOC_H_EXTERNS +/******************************************************************************/ +#define JEMALLOC_H_INLINES + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/stats.h" +#include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/mb.h" +#include "jemalloc/internal/extent.h" +#include "jemalloc/internal/base.h" +#include "jemalloc/internal/chunk.h" +#include "jemalloc/internal/huge.h" + +#ifndef JEMALLOC_ENABLE_INLINE +size_t pow2_ceil(size_t x); +size_t s2u(size_t size); +size_t sa2u(size_t size, size_t alignment, size_t *run_size_p); +void malloc_write(const char *s); +arena_t *choose_arena(void); +# if (defined(JEMALLOC_STATS) && defined(NO_TLS)) +thread_allocated_t *thread_allocated_get(void); +# endif +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +/* Compute the smallest power of 2 that is >= x. */ +JEMALLOC_INLINE size_t +pow2_ceil(size_t x) +{ + + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; +#if (LG_SIZEOF_PTR == 3) + x |= x >> 32; +#endif + x++; + return (x); +} + +/* + * Compute usable size that would result from allocating an object with the + * specified size. + */ +JEMALLOC_INLINE size_t +s2u(size_t size) +{ + + if (size <= small_maxclass) + return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); + if (size <= arena_maxclass) + return (PAGE_CEILING(size)); + return (CHUNK_CEILING(size)); +} + +/* + * Compute usable size that would result from allocating an object with the + * specified size and alignment. + */ +JEMALLOC_INLINE size_t +sa2u(size_t size, size_t alignment, size_t *run_size_p) +{ + size_t usize; + + /* + * Round size up to the nearest multiple of alignment. + * + * This done, we can take advantage of the fact that for each small + * size class, every object is aligned at the smallest power of two + * that is non-zero in the base two representation of the size. For + * example: + * + * Size | Base 2 | Minimum alignment + * -----+----------+------------------ + * 96 | 1100000 | 32 + * 144 | 10100000 | 32 + * 192 | 11000000 | 64 + * + * Depending on runtime settings, it is possible that arena_malloc() + * will further round up to a power of two, but that never causes + * correctness issues. + */ + usize = (size + (alignment - 1)) & (-alignment); + /* + * (usize < size) protects against the combination of maximal + * alignment and size greater than maximal alignment. + */ + if (usize < size) { + /* size_t overflow. */ + return (0); + } + + if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { + if (usize <= small_maxclass) + return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); + return (PAGE_CEILING(usize)); + } else { + size_t run_size; + + /* + * We can't achieve subpage alignment, so round up alignment + * permanently; it makes later calculations simpler. + */ + alignment = PAGE_CEILING(alignment); + usize = PAGE_CEILING(size); + /* + * (usize < size) protects against very large sizes within + * PAGE_SIZE of SIZE_T_MAX. + * + * (usize + alignment < usize) protects against the + * combination of maximal alignment and usize large enough + * to cause overflow. This is similar to the first overflow + * check above, but it needs to be repeated due to the new + * usize value, which may now be *equal* to maximal + * alignment, whereas before we only detected overflow if the + * original size was *greater* than maximal alignment. + */ + if (usize < size || usize + alignment < usize) { + /* size_t overflow. */ + return (0); + } + + /* + * Calculate the size of the over-size run that arena_palloc() + * would need to allocate in order to guarantee the alignment. + */ + if (usize >= alignment) + run_size = usize + alignment - PAGE_SIZE; + else { + /* + * It is possible that (alignment << 1) will cause + * overflow, but it doesn't matter because we also + * subtract PAGE_SIZE, which in the case of overflow + * leaves us with a very large run_size. That causes + * the first conditional below to fail, which means + * that the bogus run_size value never gets used for + * anything important. + */ + run_size = (alignment << 1) - PAGE_SIZE; + } + if (run_size_p != NULL) + *run_size_p = run_size; + + if (run_size <= arena_maxclass) + return (PAGE_CEILING(usize)); + return (CHUNK_CEILING(usize)); + } +} + +/* + * Wrapper around malloc_message() that avoids the need for + * JEMALLOC_P(malloc_message)(...) throughout the code. + */ +JEMALLOC_INLINE void +malloc_write(const char *s) +{ + + JEMALLOC_P(malloc_message)(NULL, s); +} + +/* + * Choose an arena based on a per-thread value (fast-path code, calls slow-path + * code if necessary). + */ +JEMALLOC_INLINE arena_t * +choose_arena(void) +{ + arena_t *ret; + + ret = ARENA_GET(); + if (ret == NULL) { + ret = choose_arena_hard(); + assert(ret != NULL); + } + + return (ret); +} + +#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) +JEMALLOC_INLINE thread_allocated_t * +thread_allocated_get(void) +{ + thread_allocated_t *thread_allocated = (thread_allocated_t *) + pthread_getspecific(thread_allocated_tsd); + + if (thread_allocated == NULL) + return (thread_allocated_get_hard()); + return (thread_allocated); +} +#endif +#endif + +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/rtree.h" +#include "jemalloc/internal/tcache.h" +#include "jemalloc/internal/arena.h" +#include "jemalloc/internal/hash.h" +#ifdef JEMALLOC_ZONE +#include "jemalloc/internal/zone.h" +#endif + +#ifndef JEMALLOC_ENABLE_INLINE +void *imalloc(size_t size); +void *icalloc(size_t size); +void *ipalloc(size_t usize, size_t alignment, bool zero); +size_t isalloc(const void *ptr); +# ifdef JEMALLOC_IVSALLOC +size_t ivsalloc(const void *ptr); +# endif +void idalloc(void *ptr); +void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, + bool zero, bool no_move); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) +JEMALLOC_INLINE void * +imalloc(size_t size) +{ + + assert(size != 0); + + if (size <= arena_maxclass) + return (arena_malloc(size, false)); + else + return (huge_malloc(size, false)); +} + +JEMALLOC_INLINE void * +icalloc(size_t size) +{ + + if (size <= arena_maxclass) + return (arena_malloc(size, true)); + else + return (huge_malloc(size, true)); +} + +JEMALLOC_INLINE void * +ipalloc(size_t usize, size_t alignment, bool zero) +{ + void *ret; + + assert(usize != 0); + assert(usize == sa2u(usize, alignment, NULL)); + + if (usize <= arena_maxclass && alignment <= PAGE_SIZE) + ret = arena_malloc(usize, zero); + else { + size_t run_size +#ifdef JEMALLOC_CC_SILENCE + = 0 +#endif + ; + + /* + * Ideally we would only ever call sa2u() once per aligned + * allocation request, and the caller of this function has + * already done so once. However, it's rather burdensome to + * require every caller to pass in run_size, especially given + * that it's only relevant to large allocations. Therefore, + * just call it again here in order to get run_size. + */ + sa2u(usize, alignment, &run_size); + if (run_size <= arena_maxclass) { + ret = arena_palloc(choose_arena(), usize, run_size, + alignment, zero); + } else if (alignment <= chunksize) + ret = huge_malloc(usize, zero); + else + ret = huge_palloc(usize, alignment, zero); + } + + assert(((uintptr_t)ret & (alignment - 1)) == 0); + return (ret); +} + +JEMALLOC_INLINE size_t +isalloc(const void *ptr) +{ + size_t ret; + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) { + /* Region. */ + dassert(chunk->arena->magic == ARENA_MAGIC); + +#ifdef JEMALLOC_PROF + ret = arena_salloc_demote(ptr); +#else + ret = arena_salloc(ptr); +#endif + } else + ret = huge_salloc(ptr); + + return (ret); +} + +#ifdef JEMALLOC_IVSALLOC +JEMALLOC_INLINE size_t +ivsalloc(const void *ptr) +{ + + /* Return 0 if ptr is not within a chunk managed by jemalloc. */ + if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) + return (0); + + return (isalloc(ptr)); +} +#endif + +JEMALLOC_INLINE void +idalloc(void *ptr) +{ + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) + arena_dalloc(chunk->arena, chunk, ptr); + else + huge_dalloc(ptr, true); +} + +JEMALLOC_INLINE void * +iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, + bool no_move) +{ + void *ret; + size_t oldsize; + + assert(ptr != NULL); + assert(size != 0); + + oldsize = isalloc(ptr); + + if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) + != 0) { + size_t usize, copysize; + + /* + * Existing object alignment is inadquate; allocate new space + * and copy. + */ + if (no_move) + return (NULL); + usize = sa2u(size + extra, alignment, NULL); + if (usize == 0) + return (NULL); + ret = ipalloc(usize, alignment, zero); + if (ret == NULL) { + if (extra == 0) + return (NULL); + /* Try again, without extra this time. */ + usize = sa2u(size, alignment, NULL); + if (usize == 0) + return (NULL); + ret = ipalloc(usize, alignment, zero); + if (ret == NULL) + return (NULL); + } + /* + * Copy at most size bytes (not size+extra), since the caller + * has no expectation that the extra bytes will be reliably + * preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + memcpy(ret, ptr, copysize); + idalloc(ptr); + return (ret); + } + + if (no_move) { + if (size <= arena_maxclass) { + return (arena_ralloc_no_move(ptr, oldsize, size, + extra, zero)); + } else { + return (huge_ralloc_no_move(ptr, oldsize, size, + extra)); + } + } else { + if (size + extra <= arena_maxclass) { + return (arena_ralloc(ptr, oldsize, size, extra, + alignment, zero)); + } else { + return (huge_ralloc(ptr, oldsize, size, extra, + alignment, zero)); + } + } +} +#endif + +#include "jemalloc/internal/prof.h" + +#undef JEMALLOC_H_INLINES +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/mb.h b/deps/jemalloc.orig/include/jemalloc/internal/mb.h new file mode 100644 index 00000000000..dc9f2a54262 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/mb.h @@ -0,0 +1,108 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +void mb_write(void); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MB_C_)) +#ifdef __i386__ +/* + * According to the Intel Architecture Software Developer's Manual, current + * processors execute instructions in order from the perspective of other + * processors in a multiprocessor system, but 1) Intel reserves the right to + * change that, and 2) the compiler's optimizer could re-order instructions if + * there weren't some form of barrier. Therefore, even if running on an + * architecture that does not need memory barriers (everything through at least + * i686), an "optimizer barrier" is necessary. + */ +JEMALLOC_INLINE void +mb_write(void) +{ + +# if 0 + /* This is a true memory barrier. */ + asm volatile ("pusha;" + "xor %%eax,%%eax;" + "cpuid;" + "popa;" + : /* Outputs. */ + : /* Inputs. */ + : "memory" /* Clobbers. */ + ); +#else + /* + * This is hopefully enough to keep the compiler from reordering + * instructions around this one. + */ + asm volatile ("nop;" + : /* Outputs. */ + : /* Inputs. */ + : "memory" /* Clobbers. */ + ); +#endif +} +#elif (defined(__amd64_) || defined(__x86_64__)) +JEMALLOC_INLINE void +mb_write(void) +{ + + asm volatile ("sfence" + : /* Outputs. */ + : /* Inputs. */ + : "memory" /* Clobbers. */ + ); +} +#elif defined(__powerpc__) +JEMALLOC_INLINE void +mb_write(void) +{ + + asm volatile ("eieio" + : /* Outputs. */ + : /* Inputs. */ + : "memory" /* Clobbers. */ + ); +} +#elif defined(__sparc64__) +JEMALLOC_INLINE void +mb_write(void) +{ + + asm volatile ("membar #StoreStore" + : /* Outputs. */ + : /* Inputs. */ + : "memory" /* Clobbers. */ + ); +} +#else +/* + * This is much slower than a simple memory barrier, but the semantics of mutex + * unlock make this work. + */ +JEMALLOC_INLINE void +mb_write(void) +{ + malloc_mutex_t mtx; + + malloc_mutex_init(&mtx); + malloc_mutex_lock(&mtx); + malloc_mutex_unlock(&mtx); +} +#endif +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/mutex.h b/deps/jemalloc.orig/include/jemalloc/internal/mutex.h new file mode 100644 index 00000000000..62947ced55e --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/mutex.h @@ -0,0 +1,86 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#ifdef JEMALLOC_OSSPIN +typedef OSSpinLock malloc_mutex_t; +#else +typedef pthread_mutex_t malloc_mutex_t; +#endif + +#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +# define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +#else +# define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#ifdef JEMALLOC_LAZY_LOCK +extern bool isthreaded; +#else +# define isthreaded true +#endif + +bool malloc_mutex_init(malloc_mutex_t *mutex); +void malloc_mutex_destroy(malloc_mutex_t *mutex); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +void malloc_mutex_lock(malloc_mutex_t *mutex); +bool malloc_mutex_trylock(malloc_mutex_t *mutex); +void malloc_mutex_unlock(malloc_mutex_t *mutex); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) +JEMALLOC_INLINE void +malloc_mutex_lock(malloc_mutex_t *mutex) +{ + + if (isthreaded) { +#ifdef JEMALLOC_OSSPIN + OSSpinLockLock(mutex); +#else + pthread_mutex_lock(mutex); +#endif + } +} + +JEMALLOC_INLINE bool +malloc_mutex_trylock(malloc_mutex_t *mutex) +{ + + if (isthreaded) { +#ifdef JEMALLOC_OSSPIN + return (OSSpinLockTry(mutex) == false); +#else + return (pthread_mutex_trylock(mutex) != 0); +#endif + } else + return (false); +} + +JEMALLOC_INLINE void +malloc_mutex_unlock(malloc_mutex_t *mutex) +{ + + if (isthreaded) { +#ifdef JEMALLOC_OSSPIN + OSSpinLockUnlock(mutex); +#else + pthread_mutex_unlock(mutex); +#endif + } +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h b/deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h new file mode 100644 index 00000000000..d4f5f96d7b2 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h @@ -0,0 +1,195 @@ +#define arena_bin_index JEMALLOC_N(arena_bin_index) +#define arena_boot JEMALLOC_N(arena_boot) +#define arena_dalloc JEMALLOC_N(arena_dalloc) +#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) +#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) +#define arena_malloc JEMALLOC_N(arena_malloc) +#define arena_malloc_large JEMALLOC_N(arena_malloc_large) +#define arena_malloc_small JEMALLOC_N(arena_malloc_small) +#define arena_new JEMALLOC_N(arena_new) +#define arena_palloc JEMALLOC_N(arena_palloc) +#define arena_prof_accum JEMALLOC_N(arena_prof_accum) +#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) +#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) +#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) +#define arena_purge_all JEMALLOC_N(arena_purge_all) +#define arena_ralloc JEMALLOC_N(arena_ralloc) +#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) +#define arena_run_regind JEMALLOC_N(arena_run_regind) +#define arena_salloc JEMALLOC_N(arena_salloc) +#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote) +#define arena_stats_merge JEMALLOC_N(arena_stats_merge) +#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) +#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index) +#define arenas_extend JEMALLOC_N(arenas_extend) +#define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index) +#define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32) +#define atomic_add_uint64 JEMALLOC_N(atomic_add_uint64) +#define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) +#define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) +#define base_alloc JEMALLOC_N(base_alloc) +#define base_boot JEMALLOC_N(base_boot) +#define base_node_alloc JEMALLOC_N(base_node_alloc) +#define base_node_dealloc JEMALLOC_N(base_node_dealloc) +#define bitmap_full JEMALLOC_N(bitmap_full) +#define bitmap_get JEMALLOC_N(bitmap_get) +#define bitmap_info_init JEMALLOC_N(bitmap_info_init) +#define bitmap_info_ngroups JEMALLOC_N(bitmap_info_ngroups) +#define bitmap_init JEMALLOC_N(bitmap_init) +#define bitmap_set JEMALLOC_N(bitmap_set) +#define bitmap_sfu JEMALLOC_N(bitmap_sfu) +#define bitmap_size JEMALLOC_N(bitmap_size) +#define bitmap_unset JEMALLOC_N(bitmap_unset) +#define bt_init JEMALLOC_N(bt_init) +#define buferror JEMALLOC_N(buferror) +#define choose_arena JEMALLOC_N(choose_arena) +#define choose_arena_hard JEMALLOC_N(choose_arena_hard) +#define chunk_alloc JEMALLOC_N(chunk_alloc) +#define chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss) +#define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) +#define chunk_alloc_mmap_noreserve JEMALLOC_N(chunk_alloc_mmap_noreserve) +#define chunk_alloc_swap JEMALLOC_N(chunk_alloc_swap) +#define chunk_boot JEMALLOC_N(chunk_boot) +#define chunk_dealloc JEMALLOC_N(chunk_dealloc) +#define chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss) +#define chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap) +#define chunk_dealloc_swap JEMALLOC_N(chunk_dealloc_swap) +#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) +#define chunk_in_dss JEMALLOC_N(chunk_in_dss) +#define chunk_in_swap JEMALLOC_N(chunk_in_swap) +#define chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot) +#define chunk_swap_boot JEMALLOC_N(chunk_swap_boot) +#define chunk_swap_enable JEMALLOC_N(chunk_swap_enable) +#define ckh_bucket_search JEMALLOC_N(ckh_bucket_search) +#define ckh_count JEMALLOC_N(ckh_count) +#define ckh_delete JEMALLOC_N(ckh_delete) +#define ckh_evict_reloc_insert JEMALLOC_N(ckh_evict_reloc_insert) +#define ckh_insert JEMALLOC_N(ckh_insert) +#define ckh_isearch JEMALLOC_N(ckh_isearch) +#define ckh_iter JEMALLOC_N(ckh_iter) +#define ckh_new JEMALLOC_N(ckh_new) +#define ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash) +#define ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp) +#define ckh_rebuild JEMALLOC_N(ckh_rebuild) +#define ckh_remove JEMALLOC_N(ckh_remove) +#define ckh_search JEMALLOC_N(ckh_search) +#define ckh_string_hash JEMALLOC_N(ckh_string_hash) +#define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) +#define ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert) +#define ckh_try_insert JEMALLOC_N(ckh_try_insert) +#define create_zone JEMALLOC_N(create_zone) +#define ctl_boot JEMALLOC_N(ctl_boot) +#define ctl_bymib JEMALLOC_N(ctl_bymib) +#define ctl_byname JEMALLOC_N(ctl_byname) +#define ctl_nametomib JEMALLOC_N(ctl_nametomib) +#define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) +#define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) +#define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) +#define extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse) +#define extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start) +#define extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last) +#define extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new) +#define extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next) +#define extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch) +#define extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev) +#define extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch) +#define extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove) +#define extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter) +#define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) +#define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) +#define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) +#define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) +#define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) +#define extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter) +#define extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse) +#define extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start) +#define extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last) +#define extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new) +#define extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next) +#define extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch) +#define extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev) +#define extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch) +#define extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove) +#define extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter) +#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) +#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) +#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) +#define hash JEMALLOC_N(hash) +#define huge_boot JEMALLOC_N(huge_boot) +#define huge_dalloc JEMALLOC_N(huge_dalloc) +#define huge_malloc JEMALLOC_N(huge_malloc) +#define huge_palloc JEMALLOC_N(huge_palloc) +#define huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get) +#define huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set) +#define huge_ralloc JEMALLOC_N(huge_ralloc) +#define huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move) +#define huge_salloc JEMALLOC_N(huge_salloc) +#define iallocm JEMALLOC_N(iallocm) +#define icalloc JEMALLOC_N(icalloc) +#define idalloc JEMALLOC_N(idalloc) +#define imalloc JEMALLOC_N(imalloc) +#define ipalloc JEMALLOC_N(ipalloc) +#define iralloc JEMALLOC_N(iralloc) +#define isalloc JEMALLOC_N(isalloc) +#define ivsalloc JEMALLOC_N(ivsalloc) +#define jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init) +#define jemalloc_postfork JEMALLOC_N(jemalloc_postfork) +#define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) +#define malloc_cprintf JEMALLOC_N(malloc_cprintf) +#define malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy) +#define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) +#define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) +#define malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock) +#define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) +#define malloc_printf JEMALLOC_N(malloc_printf) +#define malloc_write JEMALLOC_N(malloc_write) +#define mb_write JEMALLOC_N(mb_write) +#define pow2_ceil JEMALLOC_N(pow2_ceil) +#define prof_backtrace JEMALLOC_N(prof_backtrace) +#define prof_boot0 JEMALLOC_N(prof_boot0) +#define prof_boot1 JEMALLOC_N(prof_boot1) +#define prof_boot2 JEMALLOC_N(prof_boot2) +#define prof_ctx_get JEMALLOC_N(prof_ctx_get) +#define prof_ctx_set JEMALLOC_N(prof_ctx_set) +#define prof_free JEMALLOC_N(prof_free) +#define prof_gdump JEMALLOC_N(prof_gdump) +#define prof_idump JEMALLOC_N(prof_idump) +#define prof_lookup JEMALLOC_N(prof_lookup) +#define prof_malloc JEMALLOC_N(prof_malloc) +#define prof_mdump JEMALLOC_N(prof_mdump) +#define prof_realloc JEMALLOC_N(prof_realloc) +#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) +#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) +#define prof_tdata_init JEMALLOC_N(prof_tdata_init) +#define pthread_create JEMALLOC_N(pthread_create) +#define rtree_get JEMALLOC_N(rtree_get) +#define rtree_get_locked JEMALLOC_N(rtree_get_locked) +#define rtree_new JEMALLOC_N(rtree_new) +#define rtree_set JEMALLOC_N(rtree_set) +#define s2u JEMALLOC_N(s2u) +#define sa2u JEMALLOC_N(sa2u) +#define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index) +#define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index) +#define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index) +#define stats_cactive_add JEMALLOC_N(stats_cactive_add) +#define stats_cactive_get JEMALLOC_N(stats_cactive_get) +#define stats_cactive_sub JEMALLOC_N(stats_cactive_sub) +#define stats_print JEMALLOC_N(stats_print) +#define szone2ozone JEMALLOC_N(szone2ozone) +#define tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy) +#define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) +#define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) +#define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) +#define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) +#define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) +#define tcache_boot JEMALLOC_N(tcache_boot) +#define tcache_create JEMALLOC_N(tcache_create) +#define tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large) +#define tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small) +#define tcache_destroy JEMALLOC_N(tcache_destroy) +#define tcache_event JEMALLOC_N(tcache_event) +#define tcache_get JEMALLOC_N(tcache_get) +#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) +#define thread_allocated_get JEMALLOC_N(thread_allocated_get) +#define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard) +#define u2s JEMALLOC_N(u2s) diff --git a/deps/jemalloc/include/jemalloc/internal/prn.h b/deps/jemalloc.orig/include/jemalloc/internal/prn.h similarity index 100% rename from deps/jemalloc/include/jemalloc/internal/prn.h rename to deps/jemalloc.orig/include/jemalloc/internal/prn.h diff --git a/deps/jemalloc.orig/include/jemalloc/internal/prof.h b/deps/jemalloc.orig/include/jemalloc/internal/prof.h new file mode 100644 index 00000000000..e9064ba6e73 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/prof.h @@ -0,0 +1,547 @@ +#ifdef JEMALLOC_PROF +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct prof_bt_s prof_bt_t; +typedef struct prof_cnt_s prof_cnt_t; +typedef struct prof_thr_cnt_s prof_thr_cnt_t; +typedef struct prof_ctx_s prof_ctx_t; +typedef struct prof_tdata_s prof_tdata_t; + +/* Option defaults. */ +#define PROF_PREFIX_DEFAULT "jeprof" +#define LG_PROF_BT_MAX_DEFAULT 7 +#define LG_PROF_SAMPLE_DEFAULT 0 +#define LG_PROF_INTERVAL_DEFAULT -1 +#define LG_PROF_TCMAX_DEFAULT -1 + +/* + * Hard limit on stack backtrace depth. Note that the version of + * prof_backtrace() that is based on __builtin_return_address() necessarily has + * a hard-coded number of backtrace frame handlers. + */ +#if (defined(JEMALLOC_PROF_LIBGCC) || defined(JEMALLOC_PROF_LIBUNWIND)) +# define LG_PROF_BT_MAX ((ZU(1) << (LG_SIZEOF_PTR+3)) - 1) +#else +# define LG_PROF_BT_MAX 7 /* >= LG_PROF_BT_MAX_DEFAULT */ +#endif +#define PROF_BT_MAX (1U << LG_PROF_BT_MAX) + +/* Initial hash table size. */ +#define PROF_CKH_MINITEMS 64 + +/* Size of memory buffer to use when writing dump files. */ +#define PROF_DUMP_BUF_SIZE 65536 + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +struct prof_bt_s { + /* Backtrace, stored as len program counters. */ + void **vec; + unsigned len; +}; + +#ifdef JEMALLOC_PROF_LIBGCC +/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ +typedef struct { + prof_bt_t *bt; + unsigned nignore; + unsigned max; +} prof_unwind_data_t; +#endif + +struct prof_cnt_s { + /* + * Profiling counters. An allocation/deallocation pair can operate on + * different prof_thr_cnt_t objects that are linked into the same + * prof_ctx_t cnts_ql, so it is possible for the cur* counters to go + * negative. In principle it is possible for the *bytes counters to + * overflow/underflow, but a general solution would require something + * like 128-bit counters; this implementation doesn't bother to solve + * that problem. + */ + int64_t curobjs; + int64_t curbytes; + uint64_t accumobjs; + uint64_t accumbytes; +}; + +struct prof_thr_cnt_s { + /* Linkage into prof_ctx_t's cnts_ql. */ + ql_elm(prof_thr_cnt_t) cnts_link; + + /* Linkage into thread's LRU. */ + ql_elm(prof_thr_cnt_t) lru_link; + + /* + * Associated context. If a thread frees an object that it did not + * allocate, it is possible that the context is not cached in the + * thread's hash table, in which case it must be able to look up the + * context, insert a new prof_thr_cnt_t into the thread's hash table, + * and link it into the prof_ctx_t's cnts_ql. + */ + prof_ctx_t *ctx; + + /* + * Threads use memory barriers to update the counters. Since there is + * only ever one writer, the only challenge is for the reader to get a + * consistent read of the counters. + * + * The writer uses this series of operations: + * + * 1) Increment epoch to an odd number. + * 2) Update counters. + * 3) Increment epoch to an even number. + * + * The reader must assure 1) that the epoch is even while it reads the + * counters, and 2) that the epoch doesn't change between the time it + * starts and finishes reading the counters. + */ + unsigned epoch; + + /* Profiling counters. */ + prof_cnt_t cnts; +}; + +struct prof_ctx_s { + /* Associated backtrace. */ + prof_bt_t *bt; + + /* Protects cnt_merged and cnts_ql. */ + malloc_mutex_t lock; + + /* Temporary storage for summation during dump. */ + prof_cnt_t cnt_summed; + + /* When threads exit, they merge their stats into cnt_merged. */ + prof_cnt_t cnt_merged; + + /* + * List of profile counters, one for each thread that has allocated in + * this context. + */ + ql_head(prof_thr_cnt_t) cnts_ql; +}; + +struct prof_tdata_s { + /* + * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a + * cache of backtraces, with associated thread-specific prof_thr_cnt_t + * objects. Other threads may read the prof_thr_cnt_t contents, but no + * others will ever write them. + * + * Upon thread exit, the thread must merge all the prof_thr_cnt_t + * counter data into the associated prof_ctx_t objects, and unlink/free + * the prof_thr_cnt_t objects. + */ + ckh_t bt2cnt; + + /* LRU for contents of bt2cnt. */ + ql_head(prof_thr_cnt_t) lru_ql; + + /* Backtrace vector, used for calls to prof_backtrace(). */ + void **vec; + + /* Sampling state. */ + uint64_t prn_state; + uint64_t threshold; + uint64_t accum; +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +extern bool opt_prof; +/* + * Even if opt_prof is true, sampling can be temporarily disabled by setting + * opt_prof_active to false. No locking is used when updating opt_prof_active, + * so there are no guarantees regarding how long it will take for all threads + * to notice state changes. + */ +extern bool opt_prof_active; +extern size_t opt_lg_prof_bt_max; /* Maximum backtrace depth. */ +extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ +extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ +extern bool opt_prof_gdump; /* High-water memory dumping. */ +extern bool opt_prof_leak; /* Dump leak summary at exit. */ +extern bool opt_prof_accum; /* Report cumulative bytes. */ +extern ssize_t opt_lg_prof_tcmax; /* lg(max per thread bactrace cache) */ +extern char opt_prof_prefix[PATH_MAX + 1]; + +/* + * Profile dump interval, measured in bytes allocated. Each arena triggers a + * profile dump when it reaches this threshold. The effect is that the + * interval between profile dumps averages prof_interval, though the actual + * interval between dumps will tend to be sporadic, and the interval will be a + * maximum of approximately (prof_interval * narenas). + */ +extern uint64_t prof_interval; + +/* + * If true, promote small sampled objects to large objects, since small run + * headers do not have embedded profile context pointers. + */ +extern bool prof_promote; + +/* (1U << opt_lg_prof_bt_max). */ +extern unsigned prof_bt_max; + +/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ +#ifndef NO_TLS +extern __thread prof_tdata_t *prof_tdata_tls + JEMALLOC_ATTR(tls_model("initial-exec")); +# define PROF_TCACHE_GET() prof_tdata_tls +# define PROF_TCACHE_SET(v) do { \ + prof_tdata_tls = (v); \ + pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ +} while (0) +#else +# define PROF_TCACHE_GET() \ + ((prof_tdata_t *)pthread_getspecific(prof_tdata_tsd)) +# define PROF_TCACHE_SET(v) do { \ + pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ +} while (0) +#endif +/* + * Same contents as b2cnt_tls, but initialized such that the TSD destructor is + * called when a thread exits, so that prof_tdata_tls contents can be merged, + * unlinked, and deallocated. + */ +extern pthread_key_t prof_tdata_tsd; + +void bt_init(prof_bt_t *bt, void **vec); +void prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max); +prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); +void prof_idump(void); +bool prof_mdump(const char *filename); +void prof_gdump(void); +prof_tdata_t *prof_tdata_init(void); +void prof_boot0(void); +void prof_boot1(void); +bool prof_boot2(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#define PROF_ALLOC_PREP(nignore, size, ret) do { \ + prof_tdata_t *prof_tdata; \ + prof_bt_t bt; \ + \ + assert(size == s2u(size)); \ + \ + prof_tdata = PROF_TCACHE_GET(); \ + if (prof_tdata == NULL) { \ + prof_tdata = prof_tdata_init(); \ + if (prof_tdata == NULL) { \ + ret = NULL; \ + break; \ + } \ + } \ + \ + if (opt_prof_active == false) { \ + /* Sampling is currently inactive, so avoid sampling. */\ + ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ + } else if (opt_lg_prof_sample == 0) { \ + /* Don't bother with sampling logic, since sampling */\ + /* interval is 1. */\ + bt_init(&bt, prof_tdata->vec); \ + prof_backtrace(&bt, nignore, prof_bt_max); \ + ret = prof_lookup(&bt); \ + } else { \ + if (prof_tdata->threshold == 0) { \ + /* Initialize. Seed the prng differently for */\ + /* each thread. */\ + prof_tdata->prn_state = \ + (uint64_t)(uintptr_t)&size; \ + prof_sample_threshold_update(prof_tdata); \ + } \ + \ + /* Determine whether to capture a backtrace based on */\ + /* whether size is enough for prof_accum to reach */\ + /* prof_tdata->threshold. However, delay updating */\ + /* these variables until prof_{m,re}alloc(), because */\ + /* we don't know for sure that the allocation will */\ + /* succeed. */\ + /* */\ + /* Use subtraction rather than addition to avoid */\ + /* potential integer overflow. */\ + if (size >= prof_tdata->threshold - \ + prof_tdata->accum) { \ + bt_init(&bt, prof_tdata->vec); \ + prof_backtrace(&bt, nignore, prof_bt_max); \ + ret = prof_lookup(&bt); \ + } else \ + ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ + } \ +} while (0) + +#ifndef JEMALLOC_ENABLE_INLINE +void prof_sample_threshold_update(prof_tdata_t *prof_tdata); +prof_ctx_t *prof_ctx_get(const void *ptr); +void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +bool prof_sample_accum_update(size_t size); +void prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt); +void prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, + size_t old_size, prof_ctx_t *old_ctx); +void prof_free(const void *ptr, size_t size); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) +JEMALLOC_INLINE void +prof_sample_threshold_update(prof_tdata_t *prof_tdata) +{ + uint64_t r; + double u; + + /* + * Compute sample threshold as a geometrically distributed random + * variable with mean (2^opt_lg_prof_sample). + * + * __ __ + * | log(u) | 1 + * prof_tdata->threshold = | -------- |, where p = ------------------- + * | log(1-p) | opt_lg_prof_sample + * 2 + * + * For more information on the math, see: + * + * Non-Uniform Random Variate Generation + * Luc Devroye + * Springer-Verlag, New York, 1986 + * pp 500 + * (http://cg.scs.carleton.ca/~luc/rnbookindex.html) + */ + prn64(r, 53, prof_tdata->prn_state, + (uint64_t)6364136223846793005LLU, (uint64_t)1442695040888963407LLU); + u = (double)r * (1.0/9007199254740992.0L); + prof_tdata->threshold = (uint64_t)(log(u) / + log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) + + (uint64_t)1U; +} + +JEMALLOC_INLINE prof_ctx_t * +prof_ctx_get(const void *ptr) +{ + prof_ctx_t *ret; + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) { + /* Region. */ + dassert(chunk->arena->magic == ARENA_MAGIC); + + ret = arena_prof_ctx_get(ptr); + } else + ret = huge_prof_ctx_get(ptr); + + return (ret); +} + +JEMALLOC_INLINE void +prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +{ + arena_chunk_t *chunk; + + assert(ptr != NULL); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) { + /* Region. */ + dassert(chunk->arena->magic == ARENA_MAGIC); + + arena_prof_ctx_set(ptr, ctx); + } else + huge_prof_ctx_set(ptr, ctx); +} + +JEMALLOC_INLINE bool +prof_sample_accum_update(size_t size) +{ + prof_tdata_t *prof_tdata; + + /* Sampling logic is unnecessary if the interval is 1. */ + assert(opt_lg_prof_sample != 0); + + prof_tdata = PROF_TCACHE_GET(); + assert(prof_tdata != NULL); + + /* Take care to avoid integer overflow. */ + if (size >= prof_tdata->threshold - prof_tdata->accum) { + prof_tdata->accum -= (prof_tdata->threshold - size); + /* Compute new sample threshold. */ + prof_sample_threshold_update(prof_tdata); + while (prof_tdata->accum >= prof_tdata->threshold) { + prof_tdata->accum -= prof_tdata->threshold; + prof_sample_threshold_update(prof_tdata); + } + return (false); + } else { + prof_tdata->accum += size; + return (true); + } +} + +JEMALLOC_INLINE void +prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) +{ + + assert(ptr != NULL); + assert(size == isalloc(ptr)); + + if (opt_lg_prof_sample != 0) { + if (prof_sample_accum_update(size)) { + /* + * Don't sample. For malloc()-like allocation, it is + * always possible to tell in advance how large an + * object's usable size will be, so there should never + * be a difference between the size passed to + * PROF_ALLOC_PREP() and prof_malloc(). + */ + assert((uintptr_t)cnt == (uintptr_t)1U); + } + } + + if ((uintptr_t)cnt > (uintptr_t)1U) { + prof_ctx_set(ptr, cnt->ctx); + + cnt->epoch++; + /*********/ + mb_write(); + /*********/ + cnt->cnts.curobjs++; + cnt->cnts.curbytes += size; + if (opt_prof_accum) { + cnt->cnts.accumobjs++; + cnt->cnts.accumbytes += size; + } + /*********/ + mb_write(); + /*********/ + cnt->epoch++; + /*********/ + mb_write(); + /*********/ + } else + prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); +} + +JEMALLOC_INLINE void +prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, + size_t old_size, prof_ctx_t *old_ctx) +{ + prof_thr_cnt_t *told_cnt; + + assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); + + if (ptr != NULL) { + assert(size == isalloc(ptr)); + if (opt_lg_prof_sample != 0) { + if (prof_sample_accum_update(size)) { + /* + * Don't sample. The size passed to + * PROF_ALLOC_PREP() was larger than what + * actually got allocated, so a backtrace was + * captured for this allocation, even though + * its actual size was insufficient to cross + * the sample threshold. + */ + cnt = (prof_thr_cnt_t *)(uintptr_t)1U; + } + } + } + + if ((uintptr_t)old_ctx > (uintptr_t)1U) { + told_cnt = prof_lookup(old_ctx->bt); + if (told_cnt == NULL) { + /* + * It's too late to propagate OOM for this realloc(), + * so operate directly on old_cnt->ctx->cnt_merged. + */ + malloc_mutex_lock(&old_ctx->lock); + old_ctx->cnt_merged.curobjs--; + old_ctx->cnt_merged.curbytes -= old_size; + malloc_mutex_unlock(&old_ctx->lock); + told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; + } + } else + told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; + + if ((uintptr_t)told_cnt > (uintptr_t)1U) + told_cnt->epoch++; + if ((uintptr_t)cnt > (uintptr_t)1U) { + prof_ctx_set(ptr, cnt->ctx); + cnt->epoch++; + } else + prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); + /*********/ + mb_write(); + /*********/ + if ((uintptr_t)told_cnt > (uintptr_t)1U) { + told_cnt->cnts.curobjs--; + told_cnt->cnts.curbytes -= old_size; + } + if ((uintptr_t)cnt > (uintptr_t)1U) { + cnt->cnts.curobjs++; + cnt->cnts.curbytes += size; + if (opt_prof_accum) { + cnt->cnts.accumobjs++; + cnt->cnts.accumbytes += size; + } + } + /*********/ + mb_write(); + /*********/ + if ((uintptr_t)told_cnt > (uintptr_t)1U) + told_cnt->epoch++; + if ((uintptr_t)cnt > (uintptr_t)1U) + cnt->epoch++; + /*********/ + mb_write(); /* Not strictly necessary. */ +} + +JEMALLOC_INLINE void +prof_free(const void *ptr, size_t size) +{ + prof_ctx_t *ctx = prof_ctx_get(ptr); + + if ((uintptr_t)ctx > (uintptr_t)1) { + assert(size == isalloc(ptr)); + prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); + + if (tcnt != NULL) { + tcnt->epoch++; + /*********/ + mb_write(); + /*********/ + tcnt->cnts.curobjs--; + tcnt->cnts.curbytes -= size; + /*********/ + mb_write(); + /*********/ + tcnt->epoch++; + /*********/ + mb_write(); + /*********/ + } else { + /* + * OOM during free() cannot be propagated, so operate + * directly on cnt->ctx->cnt_merged. + */ + malloc_mutex_lock(&ctx->lock); + ctx->cnt_merged.curobjs--; + ctx->cnt_merged.curbytes -= size; + malloc_mutex_unlock(&ctx->lock); + } + } +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ +#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/ql.h b/deps/jemalloc.orig/include/jemalloc/internal/ql.h new file mode 100644 index 00000000000..a9ed2393f0c --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/ql.h @@ -0,0 +1,83 @@ +/* + * List definitions. + */ +#define ql_head(a_type) \ +struct { \ + a_type *qlh_first; \ +} + +#define ql_head_initializer(a_head) {NULL} + +#define ql_elm(a_type) qr(a_type) + +/* List functions. */ +#define ql_new(a_head) do { \ + (a_head)->qlh_first = NULL; \ +} while (0) + +#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field) + +#define ql_first(a_head) ((a_head)->qlh_first) + +#define ql_last(a_head, a_field) \ + ((ql_first(a_head) != NULL) \ + ? qr_prev(ql_first(a_head), a_field) : NULL) + +#define ql_next(a_head, a_elm, a_field) \ + ((ql_last(a_head, a_field) != (a_elm)) \ + ? qr_next((a_elm), a_field) : NULL) + +#define ql_prev(a_head, a_elm, a_field) \ + ((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) \ + : NULL) + +#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ + qr_before_insert((a_qlelm), (a_elm), a_field); \ + if (ql_first(a_head) == (a_qlelm)) { \ + ql_first(a_head) = (a_elm); \ + } \ +} while (0) + +#define ql_after_insert(a_qlelm, a_elm, a_field) \ + qr_after_insert((a_qlelm), (a_elm), a_field) + +#define ql_head_insert(a_head, a_elm, a_field) do { \ + if (ql_first(a_head) != NULL) { \ + qr_before_insert(ql_first(a_head), (a_elm), a_field); \ + } \ + ql_first(a_head) = (a_elm); \ +} while (0) + +#define ql_tail_insert(a_head, a_elm, a_field) do { \ + if (ql_first(a_head) != NULL) { \ + qr_before_insert(ql_first(a_head), (a_elm), a_field); \ + } \ + ql_first(a_head) = qr_next((a_elm), a_field); \ +} while (0) + +#define ql_remove(a_head, a_elm, a_field) do { \ + if (ql_first(a_head) == (a_elm)) { \ + ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ + } \ + if (ql_first(a_head) != (a_elm)) { \ + qr_remove((a_elm), a_field); \ + } else { \ + ql_first(a_head) = NULL; \ + } \ +} while (0) + +#define ql_head_remove(a_head, a_type, a_field) do { \ + a_type *t = ql_first(a_head); \ + ql_remove((a_head), t, a_field); \ +} while (0) + +#define ql_tail_remove(a_head, a_type, a_field) do { \ + a_type *t = ql_last(a_head, a_field); \ + ql_remove((a_head), t, a_field); \ +} while (0) + +#define ql_foreach(a_var, a_head, a_field) \ + qr_foreach((a_var), ql_first(a_head), a_field) + +#define ql_reverse_foreach(a_var, a_head, a_field) \ + qr_reverse_foreach((a_var), ql_first(a_head), a_field) diff --git a/deps/jemalloc.orig/include/jemalloc/internal/qr.h b/deps/jemalloc.orig/include/jemalloc/internal/qr.h new file mode 100644 index 00000000000..fe22352fedd --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/qr.h @@ -0,0 +1,67 @@ +/* Ring definitions. */ +#define qr(a_type) \ +struct { \ + a_type *qre_next; \ + a_type *qre_prev; \ +} + +/* Ring functions. */ +#define qr_new(a_qr, a_field) do { \ + (a_qr)->a_field.qre_next = (a_qr); \ + (a_qr)->a_field.qre_prev = (a_qr); \ +} while (0) + +#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next) + +#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev) + +#define qr_before_insert(a_qrelm, a_qr, a_field) do { \ + (a_qr)->a_field.qre_prev = (a_qrelm)->a_field.qre_prev; \ + (a_qr)->a_field.qre_next = (a_qrelm); \ + (a_qr)->a_field.qre_prev->a_field.qre_next = (a_qr); \ + (a_qrelm)->a_field.qre_prev = (a_qr); \ +} while (0) + +#define qr_after_insert(a_qrelm, a_qr, a_field) \ + do \ + { \ + (a_qr)->a_field.qre_next = (a_qrelm)->a_field.qre_next; \ + (a_qr)->a_field.qre_prev = (a_qrelm); \ + (a_qr)->a_field.qre_next->a_field.qre_prev = (a_qr); \ + (a_qrelm)->a_field.qre_next = (a_qr); \ + } while (0) + +#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ + void *t; \ + (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ + (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ + t = (a_qr_a)->a_field.qre_prev; \ + (a_qr_a)->a_field.qre_prev = (a_qr_b)->a_field.qre_prev; \ + (a_qr_b)->a_field.qre_prev = t; \ +} while (0) + +/* qr_meld() and qr_split() are functionally equivalent, so there's no need to + * have two copies of the code. */ +#define qr_split(a_qr_a, a_qr_b, a_field) \ + qr_meld((a_qr_a), (a_qr_b), a_field) + +#define qr_remove(a_qr, a_field) do { \ + (a_qr)->a_field.qre_prev->a_field.qre_next \ + = (a_qr)->a_field.qre_next; \ + (a_qr)->a_field.qre_next->a_field.qre_prev \ + = (a_qr)->a_field.qre_prev; \ + (a_qr)->a_field.qre_next = (a_qr); \ + (a_qr)->a_field.qre_prev = (a_qr); \ +} while (0) + +#define qr_foreach(var, a_qr, a_field) \ + for ((var) = (a_qr); \ + (var) != NULL; \ + (var) = (((var)->a_field.qre_next != (a_qr)) \ + ? (var)->a_field.qre_next : NULL)) + +#define qr_reverse_foreach(var, a_qr, a_field) \ + for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ + (var) != NULL; \ + (var) = (((var) != (a_qr)) \ + ? (var)->a_field.qre_prev : NULL)) diff --git a/deps/jemalloc.orig/include/jemalloc/internal/rb.h b/deps/jemalloc.orig/include/jemalloc/internal/rb.h new file mode 100644 index 00000000000..ee9b009d235 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/rb.h @@ -0,0 +1,973 @@ +/*- + ******************************************************************************* + * + * cpp macro implementation of left-leaning 2-3 red-black trees. Parent + * pointers are not used, and color bits are stored in the least significant + * bit of right-child pointers (if RB_COMPACT is defined), thus making node + * linkage as compact as is possible for red-black trees. + * + * Usage: + * + * #include + * #include + * #define NDEBUG // (Optional, see assert(3).) + * #include + * #define RB_COMPACT // (Optional, embed color bits in right-child pointers.) + * #include + * ... + * + ******************************************************************************* + */ + +#ifndef RB_H_ +#define RB_H_ + +#if 0 +__FBSDID("$FreeBSD: head/lib/libc/stdlib/rb.h 204493 2010-02-28 22:57:13Z jasone $"); +#endif + +#ifdef RB_COMPACT +/* Node structure. */ +#define rb_node(a_type) \ +struct { \ + a_type *rbn_left; \ + a_type *rbn_right_red; \ +} +#else +#define rb_node(a_type) \ +struct { \ + a_type *rbn_left; \ + a_type *rbn_right; \ + bool rbn_red; \ +} +#endif + +/* Root structure. */ +#define rb_tree(a_type) \ +struct { \ + a_type *rbt_root; \ + a_type rbt_nil; \ +} + +/* Left accessors. */ +#define rbtn_left_get(a_type, a_field, a_node) \ + ((a_node)->a_field.rbn_left) +#define rbtn_left_set(a_type, a_field, a_node, a_left) do { \ + (a_node)->a_field.rbn_left = a_left; \ +} while (0) + +#ifdef RB_COMPACT +/* Right accessors. */ +#define rbtn_right_get(a_type, a_field, a_node) \ + ((a_type *) (((intptr_t) (a_node)->a_field.rbn_right_red) \ + & ((ssize_t)-2))) +#define rbtn_right_set(a_type, a_field, a_node, a_right) do { \ + (a_node)->a_field.rbn_right_red = (a_type *) (((uintptr_t) a_right) \ + | (((uintptr_t) (a_node)->a_field.rbn_right_red) & ((size_t)1))); \ +} while (0) + +/* Color accessors. */ +#define rbtn_red_get(a_type, a_field, a_node) \ + ((bool) (((uintptr_t) (a_node)->a_field.rbn_right_red) \ + & ((size_t)1))) +#define rbtn_color_set(a_type, a_field, a_node, a_red) do { \ + (a_node)->a_field.rbn_right_red = (a_type *) ((((intptr_t) \ + (a_node)->a_field.rbn_right_red) & ((ssize_t)-2)) \ + | ((ssize_t)a_red)); \ +} while (0) +#define rbtn_red_set(a_type, a_field, a_node) do { \ + (a_node)->a_field.rbn_right_red = (a_type *) (((uintptr_t) \ + (a_node)->a_field.rbn_right_red) | ((size_t)1)); \ +} while (0) +#define rbtn_black_set(a_type, a_field, a_node) do { \ + (a_node)->a_field.rbn_right_red = (a_type *) (((intptr_t) \ + (a_node)->a_field.rbn_right_red) & ((ssize_t)-2)); \ +} while (0) +#else +/* Right accessors. */ +#define rbtn_right_get(a_type, a_field, a_node) \ + ((a_node)->a_field.rbn_right) +#define rbtn_right_set(a_type, a_field, a_node, a_right) do { \ + (a_node)->a_field.rbn_right = a_right; \ +} while (0) + +/* Color accessors. */ +#define rbtn_red_get(a_type, a_field, a_node) \ + ((a_node)->a_field.rbn_red) +#define rbtn_color_set(a_type, a_field, a_node, a_red) do { \ + (a_node)->a_field.rbn_red = (a_red); \ +} while (0) +#define rbtn_red_set(a_type, a_field, a_node) do { \ + (a_node)->a_field.rbn_red = true; \ +} while (0) +#define rbtn_black_set(a_type, a_field, a_node) do { \ + (a_node)->a_field.rbn_red = false; \ +} while (0) +#endif + +/* Node initializer. */ +#define rbt_node_new(a_type, a_field, a_rbt, a_node) do { \ + rbtn_left_set(a_type, a_field, (a_node), &(a_rbt)->rbt_nil); \ + rbtn_right_set(a_type, a_field, (a_node), &(a_rbt)->rbt_nil); \ + rbtn_red_set(a_type, a_field, (a_node)); \ +} while (0) + +/* Tree initializer. */ +#define rb_new(a_type, a_field, a_rbt) do { \ + (a_rbt)->rbt_root = &(a_rbt)->rbt_nil; \ + rbt_node_new(a_type, a_field, a_rbt, &(a_rbt)->rbt_nil); \ + rbtn_black_set(a_type, a_field, &(a_rbt)->rbt_nil); \ +} while (0) + +/* Internal utility macros. */ +#define rbtn_first(a_type, a_field, a_rbt, a_root, r_node) do { \ + (r_node) = (a_root); \ + if ((r_node) != &(a_rbt)->rbt_nil) { \ + for (; \ + rbtn_left_get(a_type, a_field, (r_node)) != &(a_rbt)->rbt_nil;\ + (r_node) = rbtn_left_get(a_type, a_field, (r_node))) { \ + } \ + } \ +} while (0) + +#define rbtn_last(a_type, a_field, a_rbt, a_root, r_node) do { \ + (r_node) = (a_root); \ + if ((r_node) != &(a_rbt)->rbt_nil) { \ + for (; rbtn_right_get(a_type, a_field, (r_node)) != \ + &(a_rbt)->rbt_nil; (r_node) = rbtn_right_get(a_type, a_field, \ + (r_node))) { \ + } \ + } \ +} while (0) + +#define rbtn_rotate_left(a_type, a_field, a_node, r_node) do { \ + (r_node) = rbtn_right_get(a_type, a_field, (a_node)); \ + rbtn_right_set(a_type, a_field, (a_node), \ + rbtn_left_get(a_type, a_field, (r_node))); \ + rbtn_left_set(a_type, a_field, (r_node), (a_node)); \ +} while (0) + +#define rbtn_rotate_right(a_type, a_field, a_node, r_node) do { \ + (r_node) = rbtn_left_get(a_type, a_field, (a_node)); \ + rbtn_left_set(a_type, a_field, (a_node), \ + rbtn_right_get(a_type, a_field, (r_node))); \ + rbtn_right_set(a_type, a_field, (r_node), (a_node)); \ +} while (0) + +/* + * The rb_proto() macro generates function prototypes that correspond to the + * functions generated by an equivalently parameterized call to rb_gen(). + */ + +#define rb_proto(a_attr, a_prefix, a_rbt_type, a_type) \ +a_attr void \ +a_prefix##new(a_rbt_type *rbtree); \ +a_attr a_type * \ +a_prefix##first(a_rbt_type *rbtree); \ +a_attr a_type * \ +a_prefix##last(a_rbt_type *rbtree); \ +a_attr a_type * \ +a_prefix##next(a_rbt_type *rbtree, a_type *node); \ +a_attr a_type * \ +a_prefix##prev(a_rbt_type *rbtree, a_type *node); \ +a_attr a_type * \ +a_prefix##search(a_rbt_type *rbtree, a_type *key); \ +a_attr a_type * \ +a_prefix##nsearch(a_rbt_type *rbtree, a_type *key); \ +a_attr a_type * \ +a_prefix##psearch(a_rbt_type *rbtree, a_type *key); \ +a_attr void \ +a_prefix##insert(a_rbt_type *rbtree, a_type *node); \ +a_attr void \ +a_prefix##remove(a_rbt_type *rbtree, a_type *node); \ +a_attr a_type * \ +a_prefix##iter(a_rbt_type *rbtree, a_type *start, a_type *(*cb)( \ + a_rbt_type *, a_type *, void *), void *arg); \ +a_attr a_type * \ +a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ + a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg); + +/* + * The rb_gen() macro generates a type-specific red-black tree implementation, + * based on the above cpp macros. + * + * Arguments: + * + * a_attr : Function attribute for generated functions (ex: static). + * a_prefix : Prefix for generated functions (ex: ex_). + * a_rb_type : Type for red-black tree data structure (ex: ex_t). + * a_type : Type for red-black tree node data structure (ex: ex_node_t). + * a_field : Name of red-black tree node linkage (ex: ex_link). + * a_cmp : Node comparison function name, with the following prototype: + * int (a_cmp *)(a_type *a_node, a_type *a_other); + * ^^^^^^ + * or a_key + * Interpretation of comparision function return values: + * -1 : a_node < a_other + * 0 : a_node == a_other + * 1 : a_node > a_other + * In all cases, the a_node or a_key macro argument is the first + * argument to the comparison function, which makes it possible + * to write comparison functions that treat the first argument + * specially. + * + * Assuming the following setup: + * + * typedef struct ex_node_s ex_node_t; + * struct ex_node_s { + * rb_node(ex_node_t) ex_link; + * }; + * typedef rb_tree(ex_node_t) ex_t; + * rb_gen(static, ex_, ex_t, ex_node_t, ex_link, ex_cmp) + * + * The following API is generated: + * + * static void + * ex_new(ex_t *extree); + * Description: Initialize a red-black tree structure. + * Args: + * extree: Pointer to an uninitialized red-black tree object. + * + * static ex_node_t * + * ex_first(ex_t *extree); + * static ex_node_t * + * ex_last(ex_t *extree); + * Description: Get the first/last node in extree. + * Args: + * extree: Pointer to an initialized red-black tree object. + * Ret: First/last node in extree, or NULL if extree is empty. + * + * static ex_node_t * + * ex_next(ex_t *extree, ex_node_t *node); + * static ex_node_t * + * ex_prev(ex_t *extree, ex_node_t *node); + * Description: Get node's successor/predecessor. + * Args: + * extree: Pointer to an initialized red-black tree object. + * node : A node in extree. + * Ret: node's successor/predecessor in extree, or NULL if node is + * last/first. + * + * static ex_node_t * + * ex_search(ex_t *extree, ex_node_t *key); + * Description: Search for node that matches key. + * Args: + * extree: Pointer to an initialized red-black tree object. + * key : Search key. + * Ret: Node in extree that matches key, or NULL if no match. + * + * static ex_node_t * + * ex_nsearch(ex_t *extree, ex_node_t *key); + * static ex_node_t * + * ex_psearch(ex_t *extree, ex_node_t *key); + * Description: Search for node that matches key. If no match is found, + * return what would be key's successor/predecessor, were + * key in extree. + * Args: + * extree: Pointer to an initialized red-black tree object. + * key : Search key. + * Ret: Node in extree that matches key, or if no match, hypothetical + * node's successor/predecessor (NULL if no successor/predecessor). + * + * static void + * ex_insert(ex_t *extree, ex_node_t *node); + * Description: Insert node into extree. + * Args: + * extree: Pointer to an initialized red-black tree object. + * node : Node to be inserted into extree. + * + * static void + * ex_remove(ex_t *extree, ex_node_t *node); + * Description: Remove node from extree. + * Args: + * extree: Pointer to an initialized red-black tree object. + * node : Node in extree to be removed. + * + * static ex_node_t * + * ex_iter(ex_t *extree, ex_node_t *start, ex_node_t *(*cb)(ex_t *, + * ex_node_t *, void *), void *arg); + * static ex_node_t * + * ex_reverse_iter(ex_t *extree, ex_node_t *start, ex_node *(*cb)(ex_t *, + * ex_node_t *, void *), void *arg); + * Description: Iterate forward/backward over extree, starting at node. + * If extree is modified, iteration must be immediately + * terminated by the callback function that causes the + * modification. + * Args: + * extree: Pointer to an initialized red-black tree object. + * start : Node at which to start iteration, or NULL to start at + * first/last node. + * cb : Callback function, which is called for each node during + * iteration. Under normal circumstances the callback function + * should return NULL, which causes iteration to continue. If a + * callback function returns non-NULL, iteration is immediately + * terminated and the non-NULL return value is returned by the + * iterator. This is useful for re-starting iteration after + * modifying extree. + * arg : Opaque pointer passed to cb(). + * Ret: NULL if iteration completed, or the non-NULL callback return value + * that caused termination of the iteration. + */ +#define rb_gen(a_attr, a_prefix, a_rbt_type, a_type, a_field, a_cmp) \ +a_attr void \ +a_prefix##new(a_rbt_type *rbtree) { \ + rb_new(a_type, a_field, rbtree); \ +} \ +a_attr a_type * \ +a_prefix##first(a_rbt_type *rbtree) { \ + a_type *ret; \ + rbtn_first(a_type, a_field, rbtree, rbtree->rbt_root, ret); \ + if (ret == &rbtree->rbt_nil) { \ + ret = NULL; \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##last(a_rbt_type *rbtree) { \ + a_type *ret; \ + rbtn_last(a_type, a_field, rbtree, rbtree->rbt_root, ret); \ + if (ret == &rbtree->rbt_nil) { \ + ret = NULL; \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##next(a_rbt_type *rbtree, a_type *node) { \ + a_type *ret; \ + if (rbtn_right_get(a_type, a_field, node) != &rbtree->rbt_nil) { \ + rbtn_first(a_type, a_field, rbtree, rbtn_right_get(a_type, \ + a_field, node), ret); \ + } else { \ + a_type *tnode = rbtree->rbt_root; \ + assert(tnode != &rbtree->rbt_nil); \ + ret = &rbtree->rbt_nil; \ + while (true) { \ + int cmp = (a_cmp)(node, tnode); \ + if (cmp < 0) { \ + ret = tnode; \ + tnode = rbtn_left_get(a_type, a_field, tnode); \ + } else if (cmp > 0) { \ + tnode = rbtn_right_get(a_type, a_field, tnode); \ + } else { \ + break; \ + } \ + assert(tnode != &rbtree->rbt_nil); \ + } \ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = (NULL); \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##prev(a_rbt_type *rbtree, a_type *node) { \ + a_type *ret; \ + if (rbtn_left_get(a_type, a_field, node) != &rbtree->rbt_nil) { \ + rbtn_last(a_type, a_field, rbtree, rbtn_left_get(a_type, \ + a_field, node), ret); \ + } else { \ + a_type *tnode = rbtree->rbt_root; \ + assert(tnode != &rbtree->rbt_nil); \ + ret = &rbtree->rbt_nil; \ + while (true) { \ + int cmp = (a_cmp)(node, tnode); \ + if (cmp < 0) { \ + tnode = rbtn_left_get(a_type, a_field, tnode); \ + } else if (cmp > 0) { \ + ret = tnode; \ + tnode = rbtn_right_get(a_type, a_field, tnode); \ + } else { \ + break; \ + } \ + assert(tnode != &rbtree->rbt_nil); \ + } \ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = (NULL); \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##search(a_rbt_type *rbtree, a_type *key) { \ + a_type *ret; \ + int cmp; \ + ret = rbtree->rbt_root; \ + while (ret != &rbtree->rbt_nil \ + && (cmp = (a_cmp)(key, ret)) != 0) { \ + if (cmp < 0) { \ + ret = rbtn_left_get(a_type, a_field, ret); \ + } else { \ + ret = rbtn_right_get(a_type, a_field, ret); \ + } \ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = (NULL); \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##nsearch(a_rbt_type *rbtree, a_type *key) { \ + a_type *ret; \ + a_type *tnode = rbtree->rbt_root; \ + ret = &rbtree->rbt_nil; \ + while (tnode != &rbtree->rbt_nil) { \ + int cmp = (a_cmp)(key, tnode); \ + if (cmp < 0) { \ + ret = tnode; \ + tnode = rbtn_left_get(a_type, a_field, tnode); \ + } else if (cmp > 0) { \ + tnode = rbtn_right_get(a_type, a_field, tnode); \ + } else { \ + ret = tnode; \ + break; \ + } \ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = (NULL); \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##psearch(a_rbt_type *rbtree, a_type *key) { \ + a_type *ret; \ + a_type *tnode = rbtree->rbt_root; \ + ret = &rbtree->rbt_nil; \ + while (tnode != &rbtree->rbt_nil) { \ + int cmp = (a_cmp)(key, tnode); \ + if (cmp < 0) { \ + tnode = rbtn_left_get(a_type, a_field, tnode); \ + } else if (cmp > 0) { \ + ret = tnode; \ + tnode = rbtn_right_get(a_type, a_field, tnode); \ + } else { \ + ret = tnode; \ + break; \ + } \ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = (NULL); \ + } \ + return (ret); \ +} \ +a_attr void \ +a_prefix##insert(a_rbt_type *rbtree, a_type *node) { \ + struct { \ + a_type *node; \ + int cmp; \ + } path[sizeof(void *) << 4], *pathp; \ + rbt_node_new(a_type, a_field, rbtree, node); \ + /* Wind. */ \ + path->node = rbtree->rbt_root; \ + for (pathp = path; pathp->node != &rbtree->rbt_nil; pathp++) { \ + int cmp = pathp->cmp = a_cmp(node, pathp->node); \ + assert(cmp != 0); \ + if (cmp < 0) { \ + pathp[1].node = rbtn_left_get(a_type, a_field, \ + pathp->node); \ + } else { \ + pathp[1].node = rbtn_right_get(a_type, a_field, \ + pathp->node); \ + } \ + } \ + pathp->node = node; \ + /* Unwind. */ \ + for (pathp--; (uintptr_t)pathp >= (uintptr_t)path; pathp--) { \ + a_type *cnode = pathp->node; \ + if (pathp->cmp < 0) { \ + a_type *left = pathp[1].node; \ + rbtn_left_set(a_type, a_field, cnode, left); \ + if (rbtn_red_get(a_type, a_field, left)) { \ + a_type *leftleft = rbtn_left_get(a_type, a_field, left);\ + if (rbtn_red_get(a_type, a_field, leftleft)) { \ + /* Fix up 4-node. */ \ + a_type *tnode; \ + rbtn_black_set(a_type, a_field, leftleft); \ + rbtn_rotate_right(a_type, a_field, cnode, tnode); \ + cnode = tnode; \ + } \ + } else { \ + return; \ + } \ + } else { \ + a_type *right = pathp[1].node; \ + rbtn_right_set(a_type, a_field, cnode, right); \ + if (rbtn_red_get(a_type, a_field, right)) { \ + a_type *left = rbtn_left_get(a_type, a_field, cnode); \ + if (rbtn_red_get(a_type, a_field, left)) { \ + /* Split 4-node. */ \ + rbtn_black_set(a_type, a_field, left); \ + rbtn_black_set(a_type, a_field, right); \ + rbtn_red_set(a_type, a_field, cnode); \ + } else { \ + /* Lean left. */ \ + a_type *tnode; \ + bool tred = rbtn_red_get(a_type, a_field, cnode); \ + rbtn_rotate_left(a_type, a_field, cnode, tnode); \ + rbtn_color_set(a_type, a_field, tnode, tred); \ + rbtn_red_set(a_type, a_field, cnode); \ + cnode = tnode; \ + } \ + } else { \ + return; \ + } \ + } \ + pathp->node = cnode; \ + } \ + /* Set root, and make it black. */ \ + rbtree->rbt_root = path->node; \ + rbtn_black_set(a_type, a_field, rbtree->rbt_root); \ +} \ +a_attr void \ +a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ + struct { \ + a_type *node; \ + int cmp; \ + } *pathp, *nodep, path[sizeof(void *) << 4]; \ + /* Wind. */ \ + nodep = NULL; /* Silence compiler warning. */ \ + path->node = rbtree->rbt_root; \ + for (pathp = path; pathp->node != &rbtree->rbt_nil; pathp++) { \ + int cmp = pathp->cmp = a_cmp(node, pathp->node); \ + if (cmp < 0) { \ + pathp[1].node = rbtn_left_get(a_type, a_field, \ + pathp->node); \ + } else { \ + pathp[1].node = rbtn_right_get(a_type, a_field, \ + pathp->node); \ + if (cmp == 0) { \ + /* Find node's successor, in preparation for swap. */ \ + pathp->cmp = 1; \ + nodep = pathp; \ + for (pathp++; pathp->node != &rbtree->rbt_nil; \ + pathp++) { \ + pathp->cmp = -1; \ + pathp[1].node = rbtn_left_get(a_type, a_field, \ + pathp->node); \ + } \ + break; \ + } \ + } \ + } \ + assert(nodep->node == node); \ + pathp--; \ + if (pathp->node != node) { \ + /* Swap node with its successor. */ \ + bool tred = rbtn_red_get(a_type, a_field, pathp->node); \ + rbtn_color_set(a_type, a_field, pathp->node, \ + rbtn_red_get(a_type, a_field, node)); \ + rbtn_left_set(a_type, a_field, pathp->node, \ + rbtn_left_get(a_type, a_field, node)); \ + /* If node's successor is its right child, the following code */\ + /* will do the wrong thing for the right child pointer. */\ + /* However, it doesn't matter, because the pointer will be */\ + /* properly set when the successor is pruned. */\ + rbtn_right_set(a_type, a_field, pathp->node, \ + rbtn_right_get(a_type, a_field, node)); \ + rbtn_color_set(a_type, a_field, node, tred); \ + /* The pruned leaf node's child pointers are never accessed */\ + /* again, so don't bother setting them to nil. */\ + nodep->node = pathp->node; \ + pathp->node = node; \ + if (nodep == path) { \ + rbtree->rbt_root = nodep->node; \ + } else { \ + if (nodep[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, nodep[-1].node, \ + nodep->node); \ + } else { \ + rbtn_right_set(a_type, a_field, nodep[-1].node, \ + nodep->node); \ + } \ + } \ + } else { \ + a_type *left = rbtn_left_get(a_type, a_field, node); \ + if (left != &rbtree->rbt_nil) { \ + /* node has no successor, but it has a left child. */\ + /* Splice node out, without losing the left child. */\ + assert(rbtn_red_get(a_type, a_field, node) == false); \ + assert(rbtn_red_get(a_type, a_field, left)); \ + rbtn_black_set(a_type, a_field, left); \ + if (pathp == path) { \ + rbtree->rbt_root = left; \ + } else { \ + if (pathp[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, pathp[-1].node, \ + left); \ + } else { \ + rbtn_right_set(a_type, a_field, pathp[-1].node, \ + left); \ + } \ + } \ + return; \ + } else if (pathp == path) { \ + /* The tree only contained one node. */ \ + rbtree->rbt_root = &rbtree->rbt_nil; \ + return; \ + } \ + } \ + if (rbtn_red_get(a_type, a_field, pathp->node)) { \ + /* Prune red node, which requires no fixup. */ \ + assert(pathp[-1].cmp < 0); \ + rbtn_left_set(a_type, a_field, pathp[-1].node, \ + &rbtree->rbt_nil); \ + return; \ + } \ + /* The node to be pruned is black, so unwind until balance is */\ + /* restored. */\ + pathp->node = &rbtree->rbt_nil; \ + for (pathp--; (uintptr_t)pathp >= (uintptr_t)path; pathp--) { \ + assert(pathp->cmp != 0); \ + if (pathp->cmp < 0) { \ + rbtn_left_set(a_type, a_field, pathp->node, \ + pathp[1].node); \ + assert(rbtn_red_get(a_type, a_field, pathp[1].node) \ + == false); \ + if (rbtn_red_get(a_type, a_field, pathp->node)) { \ + a_type *right = rbtn_right_get(a_type, a_field, \ + pathp->node); \ + a_type *rightleft = rbtn_left_get(a_type, a_field, \ + right); \ + a_type *tnode; \ + if (rbtn_red_get(a_type, a_field, rightleft)) { \ + /* In the following diagrams, ||, //, and \\ */\ + /* indicate the path to the removed node. */\ + /* */\ + /* || */\ + /* pathp(r) */\ + /* // \ */\ + /* (b) (b) */\ + /* / */\ + /* (r) */\ + /* */\ + rbtn_black_set(a_type, a_field, pathp->node); \ + rbtn_rotate_right(a_type, a_field, right, tnode); \ + rbtn_right_set(a_type, a_field, pathp->node, tnode);\ + rbtn_rotate_left(a_type, a_field, pathp->node, \ + tnode); \ + } else { \ + /* || */\ + /* pathp(r) */\ + /* // \ */\ + /* (b) (b) */\ + /* / */\ + /* (b) */\ + /* */\ + rbtn_rotate_left(a_type, a_field, pathp->node, \ + tnode); \ + } \ + /* Balance restored, but rotation modified subtree */\ + /* root. */\ + assert((uintptr_t)pathp > (uintptr_t)path); \ + if (pathp[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, pathp[-1].node, \ + tnode); \ + } else { \ + rbtn_right_set(a_type, a_field, pathp[-1].node, \ + tnode); \ + } \ + return; \ + } else { \ + a_type *right = rbtn_right_get(a_type, a_field, \ + pathp->node); \ + a_type *rightleft = rbtn_left_get(a_type, a_field, \ + right); \ + if (rbtn_red_get(a_type, a_field, rightleft)) { \ + /* || */\ + /* pathp(b) */\ + /* // \ */\ + /* (b) (b) */\ + /* / */\ + /* (r) */\ + a_type *tnode; \ + rbtn_black_set(a_type, a_field, rightleft); \ + rbtn_rotate_right(a_type, a_field, right, tnode); \ + rbtn_right_set(a_type, a_field, pathp->node, tnode);\ + rbtn_rotate_left(a_type, a_field, pathp->node, \ + tnode); \ + /* Balance restored, but rotation modified */\ + /* subree root, which may actually be the tree */\ + /* root. */\ + if (pathp == path) { \ + /* Set root. */ \ + rbtree->rbt_root = tnode; \ + } else { \ + if (pathp[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, \ + pathp[-1].node, tnode); \ + } else { \ + rbtn_right_set(a_type, a_field, \ + pathp[-1].node, tnode); \ + } \ + } \ + return; \ + } else { \ + /* || */\ + /* pathp(b) */\ + /* // \ */\ + /* (b) (b) */\ + /* / */\ + /* (b) */\ + a_type *tnode; \ + rbtn_red_set(a_type, a_field, pathp->node); \ + rbtn_rotate_left(a_type, a_field, pathp->node, \ + tnode); \ + pathp->node = tnode; \ + } \ + } \ + } else { \ + a_type *left; \ + rbtn_right_set(a_type, a_field, pathp->node, \ + pathp[1].node); \ + left = rbtn_left_get(a_type, a_field, pathp->node); \ + if (rbtn_red_get(a_type, a_field, left)) { \ + a_type *tnode; \ + a_type *leftright = rbtn_right_get(a_type, a_field, \ + left); \ + a_type *leftrightleft = rbtn_left_get(a_type, a_field, \ + leftright); \ + if (rbtn_red_get(a_type, a_field, leftrightleft)) { \ + /* || */\ + /* pathp(b) */\ + /* / \\ */\ + /* (r) (b) */\ + /* \ */\ + /* (b) */\ + /* / */\ + /* (r) */\ + a_type *unode; \ + rbtn_black_set(a_type, a_field, leftrightleft); \ + rbtn_rotate_right(a_type, a_field, pathp->node, \ + unode); \ + rbtn_rotate_right(a_type, a_field, pathp->node, \ + tnode); \ + rbtn_right_set(a_type, a_field, unode, tnode); \ + rbtn_rotate_left(a_type, a_field, unode, tnode); \ + } else { \ + /* || */\ + /* pathp(b) */\ + /* / \\ */\ + /* (r) (b) */\ + /* \ */\ + /* (b) */\ + /* / */\ + /* (b) */\ + assert(leftright != &rbtree->rbt_nil); \ + rbtn_red_set(a_type, a_field, leftright); \ + rbtn_rotate_right(a_type, a_field, pathp->node, \ + tnode); \ + rbtn_black_set(a_type, a_field, tnode); \ + } \ + /* Balance restored, but rotation modified subtree */\ + /* root, which may actually be the tree root. */\ + if (pathp == path) { \ + /* Set root. */ \ + rbtree->rbt_root = tnode; \ + } else { \ + if (pathp[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, pathp[-1].node, \ + tnode); \ + } else { \ + rbtn_right_set(a_type, a_field, pathp[-1].node, \ + tnode); \ + } \ + } \ + return; \ + } else if (rbtn_red_get(a_type, a_field, pathp->node)) { \ + a_type *leftleft = rbtn_left_get(a_type, a_field, left);\ + if (rbtn_red_get(a_type, a_field, leftleft)) { \ + /* || */\ + /* pathp(r) */\ + /* / \\ */\ + /* (b) (b) */\ + /* / */\ + /* (r) */\ + a_type *tnode; \ + rbtn_black_set(a_type, a_field, pathp->node); \ + rbtn_red_set(a_type, a_field, left); \ + rbtn_black_set(a_type, a_field, leftleft); \ + rbtn_rotate_right(a_type, a_field, pathp->node, \ + tnode); \ + /* Balance restored, but rotation modified */\ + /* subtree root. */\ + assert((uintptr_t)pathp > (uintptr_t)path); \ + if (pathp[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, pathp[-1].node, \ + tnode); \ + } else { \ + rbtn_right_set(a_type, a_field, pathp[-1].node, \ + tnode); \ + } \ + return; \ + } else { \ + /* || */\ + /* pathp(r) */\ + /* / \\ */\ + /* (b) (b) */\ + /* / */\ + /* (b) */\ + rbtn_red_set(a_type, a_field, left); \ + rbtn_black_set(a_type, a_field, pathp->node); \ + /* Balance restored. */ \ + return; \ + } \ + } else { \ + a_type *leftleft = rbtn_left_get(a_type, a_field, left);\ + if (rbtn_red_get(a_type, a_field, leftleft)) { \ + /* || */\ + /* pathp(b) */\ + /* / \\ */\ + /* (b) (b) */\ + /* / */\ + /* (r) */\ + a_type *tnode; \ + rbtn_black_set(a_type, a_field, leftleft); \ + rbtn_rotate_right(a_type, a_field, pathp->node, \ + tnode); \ + /* Balance restored, but rotation modified */\ + /* subtree root, which may actually be the tree */\ + /* root. */\ + if (pathp == path) { \ + /* Set root. */ \ + rbtree->rbt_root = tnode; \ + } else { \ + if (pathp[-1].cmp < 0) { \ + rbtn_left_set(a_type, a_field, \ + pathp[-1].node, tnode); \ + } else { \ + rbtn_right_set(a_type, a_field, \ + pathp[-1].node, tnode); \ + } \ + } \ + return; \ + } else { \ + /* || */\ + /* pathp(b) */\ + /* / \\ */\ + /* (b) (b) */\ + /* / */\ + /* (b) */\ + rbtn_red_set(a_type, a_field, left); \ + } \ + } \ + } \ + } \ + /* Set root. */ \ + rbtree->rbt_root = path->node; \ + assert(rbtn_red_get(a_type, a_field, rbtree->rbt_root) == false); \ +} \ +a_attr a_type * \ +a_prefix##iter_recurse(a_rbt_type *rbtree, a_type *node, \ + a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ + if (node == &rbtree->rbt_nil) { \ + return (&rbtree->rbt_nil); \ + } else { \ + a_type *ret; \ + if ((ret = a_prefix##iter_recurse(rbtree, rbtn_left_get(a_type, \ + a_field, node), cb, arg)) != &rbtree->rbt_nil \ + || (ret = cb(rbtree, node, arg)) != NULL) { \ + return (ret); \ + } \ + return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \ + a_field, node), cb, arg)); \ + } \ +} \ +a_attr a_type * \ +a_prefix##iter_start(a_rbt_type *rbtree, a_type *start, a_type *node, \ + a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ + int cmp = a_cmp(start, node); \ + if (cmp < 0) { \ + a_type *ret; \ + if ((ret = a_prefix##iter_start(rbtree, start, \ + rbtn_left_get(a_type, a_field, node), cb, arg)) != \ + &rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \ + return (ret); \ + } \ + return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \ + a_field, node), cb, arg)); \ + } else if (cmp > 0) { \ + return (a_prefix##iter_start(rbtree, start, \ + rbtn_right_get(a_type, a_field, node), cb, arg)); \ + } else { \ + a_type *ret; \ + if ((ret = cb(rbtree, node, arg)) != NULL) { \ + return (ret); \ + } \ + return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \ + a_field, node), cb, arg)); \ + } \ +} \ +a_attr a_type * \ +a_prefix##iter(a_rbt_type *rbtree, a_type *start, a_type *(*cb)( \ + a_rbt_type *, a_type *, void *), void *arg) { \ + a_type *ret; \ + if (start != NULL) { \ + ret = a_prefix##iter_start(rbtree, start, rbtree->rbt_root, \ + cb, arg); \ + } else { \ + ret = a_prefix##iter_recurse(rbtree, rbtree->rbt_root, cb, arg);\ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = NULL; \ + } \ + return (ret); \ +} \ +a_attr a_type * \ +a_prefix##reverse_iter_recurse(a_rbt_type *rbtree, a_type *node, \ + a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ + if (node == &rbtree->rbt_nil) { \ + return (&rbtree->rbt_nil); \ + } else { \ + a_type *ret; \ + if ((ret = a_prefix##reverse_iter_recurse(rbtree, \ + rbtn_right_get(a_type, a_field, node), cb, arg)) != \ + &rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \ + return (ret); \ + } \ + return (a_prefix##reverse_iter_recurse(rbtree, \ + rbtn_left_get(a_type, a_field, node), cb, arg)); \ + } \ +} \ +a_attr a_type * \ +a_prefix##reverse_iter_start(a_rbt_type *rbtree, a_type *start, \ + a_type *node, a_type *(*cb)(a_rbt_type *, a_type *, void *), \ + void *arg) { \ + int cmp = a_cmp(start, node); \ + if (cmp > 0) { \ + a_type *ret; \ + if ((ret = a_prefix##reverse_iter_start(rbtree, start, \ + rbtn_right_get(a_type, a_field, node), cb, arg)) != \ + &rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \ + return (ret); \ + } \ + return (a_prefix##reverse_iter_recurse(rbtree, \ + rbtn_left_get(a_type, a_field, node), cb, arg)); \ + } else if (cmp < 0) { \ + return (a_prefix##reverse_iter_start(rbtree, start, \ + rbtn_left_get(a_type, a_field, node), cb, arg)); \ + } else { \ + a_type *ret; \ + if ((ret = cb(rbtree, node, arg)) != NULL) { \ + return (ret); \ + } \ + return (a_prefix##reverse_iter_recurse(rbtree, \ + rbtn_left_get(a_type, a_field, node), cb, arg)); \ + } \ +} \ +a_attr a_type * \ +a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ + a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ + a_type *ret; \ + if (start != NULL) { \ + ret = a_prefix##reverse_iter_start(rbtree, start, \ + rbtree->rbt_root, cb, arg); \ + } else { \ + ret = a_prefix##reverse_iter_recurse(rbtree, rbtree->rbt_root, \ + cb, arg); \ + } \ + if (ret == &rbtree->rbt_nil) { \ + ret = NULL; \ + } \ + return (ret); \ +} + +#endif /* RB_H_ */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/rtree.h b/deps/jemalloc.orig/include/jemalloc/internal/rtree.h new file mode 100644 index 00000000000..95d6355a5f4 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/rtree.h @@ -0,0 +1,161 @@ +/* + * This radix tree implementation is tailored to the singular purpose of + * tracking which chunks are currently owned by jemalloc. This functionality + * is mandatory for OS X, where jemalloc must be able to respond to object + * ownership queries. + * + ******************************************************************************* + */ +#ifdef JEMALLOC_H_TYPES + +typedef struct rtree_s rtree_t; + +/* + * Size of each radix tree node (must be a power of 2). This impacts tree + * depth. + */ +#if (LG_SIZEOF_PTR == 2) +# define RTREE_NODESIZE (1U << 14) +#else +# define RTREE_NODESIZE CACHELINE +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +struct rtree_s { + malloc_mutex_t mutex; + void **root; + unsigned height; + unsigned level2bits[1]; /* Dynamically sized. */ +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +rtree_t *rtree_new(unsigned bits); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +#ifndef JEMALLOC_DEBUG +void *rtree_get_locked(rtree_t *rtree, uintptr_t key); +#endif +void *rtree_get(rtree_t *rtree, uintptr_t key); +bool rtree_set(rtree_t *rtree, uintptr_t key, void *val); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) +#define RTREE_GET_GENERATE(f) \ +/* The least significant bits of the key are ignored. */ \ +JEMALLOC_INLINE void * \ +f(rtree_t *rtree, uintptr_t key) \ +{ \ + void *ret; \ + uintptr_t subkey; \ + unsigned i, lshift, height, bits; \ + void **node, **child; \ + \ + RTREE_LOCK(&rtree->mutex); \ + for (i = lshift = 0, height = rtree->height, node = rtree->root;\ + i < height - 1; \ + i++, lshift += bits, node = child) { \ + bits = rtree->level2bits[i]; \ + subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ + 3)) - bits); \ + child = (void**)node[subkey]; \ + if (child == NULL) { \ + RTREE_UNLOCK(&rtree->mutex); \ + return (NULL); \ + } \ + } \ + \ + /* \ + * node is a leaf, so it contains values rather than node \ + * pointers. \ + */ \ + bits = rtree->level2bits[i]; \ + subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - \ + bits); \ + ret = node[subkey]; \ + RTREE_UNLOCK(&rtree->mutex); \ + \ + RTREE_GET_VALIDATE \ + return (ret); \ +} + +#ifdef JEMALLOC_DEBUG +# define RTREE_LOCK(l) malloc_mutex_lock(l) +# define RTREE_UNLOCK(l) malloc_mutex_unlock(l) +# define RTREE_GET_VALIDATE +RTREE_GET_GENERATE(rtree_get_locked) +# undef RTREE_LOCK +# undef RTREE_UNLOCK +# undef RTREE_GET_VALIDATE +#endif + +#define RTREE_LOCK(l) +#define RTREE_UNLOCK(l) +#ifdef JEMALLOC_DEBUG + /* + * Suppose that it were possible for a jemalloc-allocated chunk to be + * munmap()ped, followed by a different allocator in another thread re-using + * overlapping virtual memory, all without invalidating the cached rtree + * value. The result would be a false positive (the rtree would claim that + * jemalloc owns memory that it had actually discarded). This scenario + * seems impossible, but the following assertion is a prudent sanity check. + */ +# define RTREE_GET_VALIDATE \ + assert(rtree_get_locked(rtree, key) == ret); +#else +# define RTREE_GET_VALIDATE +#endif +RTREE_GET_GENERATE(rtree_get) +#undef RTREE_LOCK +#undef RTREE_UNLOCK +#undef RTREE_GET_VALIDATE + +JEMALLOC_INLINE bool +rtree_set(rtree_t *rtree, uintptr_t key, void *val) +{ + uintptr_t subkey; + unsigned i, lshift, height, bits; + void **node, **child; + + malloc_mutex_lock(&rtree->mutex); + for (i = lshift = 0, height = rtree->height, node = rtree->root; + i < height - 1; + i++, lshift += bits, node = child) { + bits = rtree->level2bits[i]; + subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - + bits); + child = (void**)node[subkey]; + if (child == NULL) { + child = (void**)base_alloc(sizeof(void *) << + rtree->level2bits[i+1]); + if (child == NULL) { + malloc_mutex_unlock(&rtree->mutex); + return (true); + } + memset(child, 0, sizeof(void *) << + rtree->level2bits[i+1]); + node[subkey] = child; + } + } + + /* node is a leaf, so it contains values rather than node pointers. */ + bits = rtree->level2bits[i]; + subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits); + node[subkey] = val; + malloc_mutex_unlock(&rtree->mutex); + + return (false); +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/stats.h b/deps/jemalloc.orig/include/jemalloc/internal/stats.h new file mode 100644 index 00000000000..2a9b31d9ffc --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/stats.h @@ -0,0 +1,207 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +#define UMAX2S_BUFSIZE 65 + +#ifdef JEMALLOC_STATS +typedef struct tcache_bin_stats_s tcache_bin_stats_t; +typedef struct malloc_bin_stats_s malloc_bin_stats_t; +typedef struct malloc_large_stats_s malloc_large_stats_t; +typedef struct arena_stats_s arena_stats_t; +#endif +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) +typedef struct chunk_stats_s chunk_stats_t; +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#ifdef JEMALLOC_STATS + +#ifdef JEMALLOC_TCACHE +struct tcache_bin_stats_s { + /* + * Number of allocation requests that corresponded to the size of this + * bin. + */ + uint64_t nrequests; +}; +#endif + +struct malloc_bin_stats_s { + /* + * Current number of bytes allocated, including objects currently + * cached by tcache. + */ + size_t allocated; + + /* + * Total number of allocation/deallocation requests served directly by + * the bin. Note that tcache may allocate an object, then recycle it + * many times, resulting many increments to nrequests, but only one + * each to nmalloc and ndalloc. + */ + uint64_t nmalloc; + uint64_t ndalloc; + + /* + * Number of allocation requests that correspond to the size of this + * bin. This includes requests served by tcache, though tcache only + * periodically merges into this counter. + */ + uint64_t nrequests; + +#ifdef JEMALLOC_TCACHE + /* Number of tcache fills from this bin. */ + uint64_t nfills; + + /* Number of tcache flushes to this bin. */ + uint64_t nflushes; +#endif + + /* Total number of runs created for this bin's size class. */ + uint64_t nruns; + + /* + * Total number of runs reused by extracting them from the runs tree for + * this bin's size class. + */ + uint64_t reruns; + + /* High-water mark for this bin. */ + size_t highruns; + + /* Current number of runs in this bin. */ + size_t curruns; +}; + +struct malloc_large_stats_s { + /* + * Total number of allocation/deallocation requests served directly by + * the arena. Note that tcache may allocate an object, then recycle it + * many times, resulting many increments to nrequests, but only one + * each to nmalloc and ndalloc. + */ + uint64_t nmalloc; + uint64_t ndalloc; + + /* + * Number of allocation requests that correspond to this size class. + * This includes requests served by tcache, though tcache only + * periodically merges into this counter. + */ + uint64_t nrequests; + + /* High-water mark for this size class. */ + size_t highruns; + + /* Current number of runs of this size class. */ + size_t curruns; +}; + +struct arena_stats_s { + /* Number of bytes currently mapped. */ + size_t mapped; + + /* + * Total number of purge sweeps, total number of madvise calls made, + * and total pages purged in order to keep dirty unused memory under + * control. + */ + uint64_t npurge; + uint64_t nmadvise; + uint64_t purged; + + /* Per-size-category statistics. */ + size_t allocated_large; + uint64_t nmalloc_large; + uint64_t ndalloc_large; + uint64_t nrequests_large; + + /* + * One element for each possible size class, including sizes that + * overlap with bin size classes. This is necessary because ipalloc() + * sometimes has to use such large objects in order to assure proper + * alignment. + */ + malloc_large_stats_t *lstats; +}; +#endif /* JEMALLOC_STATS */ + +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) +struct chunk_stats_s { +# ifdef JEMALLOC_STATS + /* Number of chunks that were allocated. */ + uint64_t nchunks; +# endif + + /* High-water mark for number of chunks allocated. */ + size_t highchunks; + + /* + * Current number of chunks allocated. This value isn't maintained for + * any other purpose, so keep track of it in order to be able to set + * highchunks. + */ + size_t curchunks; +}; +#endif /* JEMALLOC_STATS */ + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +extern bool opt_stats_print; + +#ifdef JEMALLOC_STATS +extern size_t stats_cactive; +#endif + +char *u2s(uint64_t x, unsigned base, char *s); +#ifdef JEMALLOC_STATS +void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, + const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); +void malloc_printf(const char *format, ...) + JEMALLOC_ATTR(format(printf, 1, 2)); +#endif +void stats_print(void (*write)(void *, const char *), void *cbopaque, + const char *opts); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES +#ifdef JEMALLOC_STATS + +#ifndef JEMALLOC_ENABLE_INLINE +size_t stats_cactive_get(void); +void stats_cactive_add(size_t size); +void stats_cactive_sub(size_t size); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_STATS_C_)) +JEMALLOC_INLINE size_t +stats_cactive_get(void) +{ + + return (atomic_read_z(&stats_cactive)); +} + +JEMALLOC_INLINE void +stats_cactive_add(size_t size) +{ + + atomic_add_z(&stats_cactive, size); +} + +JEMALLOC_INLINE void +stats_cactive_sub(size_t size) +{ + + atomic_sub_z(&stats_cactive, size); +} +#endif + +#endif /* JEMALLOC_STATS */ +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/tcache.h b/deps/jemalloc.orig/include/jemalloc/internal/tcache.h new file mode 100644 index 00000000000..da3c68c5770 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/internal/tcache.h @@ -0,0 +1,431 @@ +#ifdef JEMALLOC_TCACHE +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +typedef struct tcache_bin_info_s tcache_bin_info_t; +typedef struct tcache_bin_s tcache_bin_t; +typedef struct tcache_s tcache_t; + +/* + * Absolute maximum number of cache slots for each small bin in the thread + * cache. This is an additional constraint beyond that imposed as: twice the + * number of regions per run for this size class. + * + * This constant must be an even number. + */ +#define TCACHE_NSLOTS_SMALL_MAX 200 + +/* Number of cache slots for large size classes. */ +#define TCACHE_NSLOTS_LARGE 20 + +/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */ +#define LG_TCACHE_MAXCLASS_DEFAULT 15 + +/* + * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation + * events between full GC sweeps (-1: disabled). Integer rounding may cause + * the actual number to be slightly higher, since GC is performed + * incrementally. + */ +#define LG_TCACHE_GC_SWEEP_DEFAULT 13 + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +/* + * Read-only information associated with each element of tcache_t's tbins array + * is stored separately, mainly to reduce memory usage. + */ +struct tcache_bin_info_s { + unsigned ncached_max; /* Upper limit on ncached. */ +}; + +struct tcache_bin_s { +# ifdef JEMALLOC_STATS + tcache_bin_stats_t tstats; +# endif + int low_water; /* Min # cached since last GC. */ + unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */ + unsigned ncached; /* # of cached objects. */ + void **avail; /* Stack of available objects. */ +}; + +struct tcache_s { +# ifdef JEMALLOC_STATS + ql_elm(tcache_t) link; /* Used for aggregating stats. */ +# endif +# ifdef JEMALLOC_PROF + uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */ +# endif + arena_t *arena; /* This thread's arena. */ + unsigned ev_cnt; /* Event count since incremental GC. */ + unsigned next_gc_bin; /* Next bin to GC. */ + tcache_bin_t tbins[1]; /* Dynamically sized. */ + /* + * The pointer stacks associated with tbins follow as a contiguous + * array. During tcache initialization, the avail pointer in each + * element of tbins is initialized to point to the proper offset within + * this array. + */ +}; + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +extern bool opt_tcache; +extern ssize_t opt_lg_tcache_max; +extern ssize_t opt_lg_tcache_gc_sweep; + +extern tcache_bin_info_t *tcache_bin_info; + +/* Map of thread-specific caches. */ +#ifndef NO_TLS +extern __thread tcache_t *tcache_tls + JEMALLOC_ATTR(tls_model("initial-exec")); +# define TCACHE_GET() tcache_tls +# define TCACHE_SET(v) do { \ + tcache_tls = (tcache_t *)(v); \ + pthread_setspecific(tcache_tsd, (void *)(v)); \ +} while (0) +#else +# define TCACHE_GET() ((tcache_t *)pthread_getspecific(tcache_tsd)) +# define TCACHE_SET(v) do { \ + pthread_setspecific(tcache_tsd, (void *)(v)); \ +} while (0) +#endif +extern pthread_key_t tcache_tsd; + +/* + * Number of tcache bins. There are nbins small-object bins, plus 0 or more + * large-object bins. + */ +extern size_t nhbins; + +/* Maximum cached size class. */ +extern size_t tcache_maxclass; + +/* Number of tcache allocation/deallocation events between incremental GCs. */ +extern unsigned tcache_gc_incr; + +void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache_t *tcache +#endif + ); +void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache_t *tcache +#endif + ); +tcache_t *tcache_create(arena_t *arena); +void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, + size_t binind); +void tcache_destroy(tcache_t *tcache); +#ifdef JEMALLOC_STATS +void tcache_stats_merge(tcache_t *tcache, arena_t *arena); +#endif +bool tcache_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +void tcache_event(tcache_t *tcache); +tcache_t *tcache_get(void); +void *tcache_alloc_easy(tcache_bin_t *tbin); +void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); +void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); +void tcache_dalloc_small(tcache_t *tcache, void *ptr); +void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) +JEMALLOC_INLINE tcache_t * +tcache_get(void) +{ + tcache_t *tcache; + + if ((isthreaded & opt_tcache) == false) + return (NULL); + + tcache = TCACHE_GET(); + if ((uintptr_t)tcache <= (uintptr_t)2) { + if (tcache == NULL) { + tcache = tcache_create(choose_arena()); + if (tcache == NULL) + return (NULL); + } else { + if (tcache == (void *)(uintptr_t)1) { + /* + * Make a note that an allocator function was + * called after the tcache_thread_cleanup() was + * called. + */ + TCACHE_SET((uintptr_t)2); + } + return (NULL); + } + } + + return (tcache); +} + +JEMALLOC_INLINE void +tcache_event(tcache_t *tcache) +{ + + if (tcache_gc_incr == 0) + return; + + tcache->ev_cnt++; + assert(tcache->ev_cnt <= tcache_gc_incr); + if (tcache->ev_cnt == tcache_gc_incr) { + size_t binind = tcache->next_gc_bin; + tcache_bin_t *tbin = &tcache->tbins[binind]; + tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; + + if (tbin->low_water > 0) { + /* + * Flush (ceiling) 3/4 of the objects below the low + * water mark. + */ + if (binind < nbins) { + tcache_bin_flush_small(tbin, binind, + tbin->ncached - tbin->low_water + + (tbin->low_water >> 2) +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache +#endif + ); + } else { + tcache_bin_flush_large(tbin, binind, + tbin->ncached - tbin->low_water + + (tbin->low_water >> 2) +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache +#endif + ); + } + /* + * Reduce fill count by 2X. Limit lg_fill_div such that + * the fill count is always at least 1. + */ + if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) + >= 1) + tbin->lg_fill_div++; + } else if (tbin->low_water < 0) { + /* + * Increase fill count by 2X. Make sure lg_fill_div + * stays greater than 0. + */ + if (tbin->lg_fill_div > 1) + tbin->lg_fill_div--; + } + tbin->low_water = tbin->ncached; + + tcache->next_gc_bin++; + if (tcache->next_gc_bin == nhbins) + tcache->next_gc_bin = 0; + tcache->ev_cnt = 0; + } +} + +JEMALLOC_INLINE void * +tcache_alloc_easy(tcache_bin_t *tbin) +{ + void *ret; + + if (tbin->ncached == 0) { + tbin->low_water = -1; + return (NULL); + } + tbin->ncached--; + if ((int)tbin->ncached < tbin->low_water) + tbin->low_water = tbin->ncached; + ret = tbin->avail[tbin->ncached]; + return (ret); +} + +JEMALLOC_INLINE void * +tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) +{ + void *ret; + size_t binind; + tcache_bin_t *tbin; + + binind = SMALL_SIZE2BIN(size); + assert(binind < nbins); + tbin = &tcache->tbins[binind]; + ret = tcache_alloc_easy(tbin); + if (ret == NULL) { + ret = tcache_alloc_small_hard(tcache, tbin, binind); + if (ret == NULL) + return (NULL); + } + assert(arena_salloc(ret) == arena_bin_info[binind].reg_size); + + if (zero == false) { +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); +#endif + } else + memset(ret, 0, size); + +#ifdef JEMALLOC_STATS + tbin->tstats.nrequests++; +#endif +#ifdef JEMALLOC_PROF + tcache->prof_accumbytes += arena_bin_info[binind].reg_size; +#endif + tcache_event(tcache); + return (ret); +} + +JEMALLOC_INLINE void * +tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) +{ + void *ret; + size_t binind; + tcache_bin_t *tbin; + + size = PAGE_CEILING(size); + assert(size <= tcache_maxclass); + binind = nbins + (size >> PAGE_SHIFT) - 1; + assert(binind < nhbins); + tbin = &tcache->tbins[binind]; + ret = tcache_alloc_easy(tbin); + if (ret == NULL) { + /* + * Only allocate one large object at a time, because it's quite + * expensive to create one and not use it. + */ + ret = arena_malloc_large(tcache->arena, size, zero); + if (ret == NULL) + return (NULL); + } else { +#ifdef JEMALLOC_PROF + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); + size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> + PAGE_SHIFT); + chunk->map[pageind-map_bias].bits &= ~CHUNK_MAP_CLASS_MASK; +#endif + if (zero == false) { +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); +#endif + } else + memset(ret, 0, size); + +#ifdef JEMALLOC_STATS + tbin->tstats.nrequests++; +#endif +#ifdef JEMALLOC_PROF + tcache->prof_accumbytes += size; +#endif + } + + tcache_event(tcache); + return (ret); +} + +JEMALLOC_INLINE void +tcache_dalloc_small(tcache_t *tcache, void *ptr) +{ + arena_t *arena; + arena_chunk_t *chunk; + arena_run_t *run; + arena_bin_t *bin; + tcache_bin_t *tbin; + tcache_bin_info_t *tbin_info; + size_t pageind, binind; + arena_chunk_map_t *mapelm; + + assert(arena_salloc(ptr) <= small_maxclass); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = chunk->arena; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + mapelm = &chunk->map[pageind-map_bias]; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); + dassert(run->magic == ARENA_RUN_MAGIC); + bin = run->bin; + binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) / + sizeof(arena_bin_t); + assert(binind < nbins); + +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ptr, 0x5a, arena_bin_info[binind].reg_size); +#endif + + tbin = &tcache->tbins[binind]; + tbin_info = &tcache_bin_info[binind]; + if (tbin->ncached == tbin_info->ncached_max) { + tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >> + 1) +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache +#endif + ); + } + assert(tbin->ncached < tbin_info->ncached_max); + tbin->avail[tbin->ncached] = ptr; + tbin->ncached++; + + tcache_event(tcache); +} + +JEMALLOC_INLINE void +tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) +{ + arena_t *arena; + arena_chunk_t *chunk; + size_t pageind, binind; + tcache_bin_t *tbin; + tcache_bin_info_t *tbin_info; + + assert((size & PAGE_MASK) == 0); + assert(arena_salloc(ptr) > small_maxclass); + assert(arena_salloc(ptr) <= tcache_maxclass); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = chunk->arena; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + binind = nbins + (size >> PAGE_SHIFT) - 1; + +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ptr, 0x5a, size); +#endif + + tbin = &tcache->tbins[binind]; + tbin_info = &tcache_bin_info[binind]; + if (tbin->ncached == tbin_info->ncached_max) { + tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >> + 1) +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache +#endif + ); + } + assert(tbin->ncached < tbin_info->ncached_max); + tbin->avail[tbin->ncached] = ptr; + tbin->ncached++; + + tcache_event(tcache); +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ +#endif /* JEMALLOC_TCACHE */ diff --git a/deps/jemalloc/include/jemalloc/internal/zone.h b/deps/jemalloc.orig/include/jemalloc/internal/zone.h similarity index 100% rename from deps/jemalloc/include/jemalloc/internal/zone.h rename to deps/jemalloc.orig/include/jemalloc/internal/zone.h diff --git a/deps/jemalloc.orig/include/jemalloc/jemalloc.h.in b/deps/jemalloc.orig/include/jemalloc/jemalloc.h.in new file mode 100644 index 00000000000..580a5ec5d5f --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/jemalloc.h.in @@ -0,0 +1,66 @@ +#ifndef JEMALLOC_H_ +#define JEMALLOC_H_ +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +#define JEMALLOC_VERSION "@jemalloc_version@" +#define JEMALLOC_VERSION_MAJOR @jemalloc_version_major@ +#define JEMALLOC_VERSION_MINOR @jemalloc_version_minor@ +#define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ +#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ +#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" + +#include "jemalloc_defs@install_suffix@.h" +#ifndef JEMALLOC_P +# define JEMALLOC_P(s) s +#endif + +#define ALLOCM_LG_ALIGN(la) (la) +#if LG_SIZEOF_PTR == 2 +#define ALLOCM_ALIGN(a) (ffs(a)-1) +#else +#define ALLOCM_ALIGN(a) ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) +#endif +#define ALLOCM_ZERO ((int)0x40) +#define ALLOCM_NO_MOVE ((int)0x80) + +#define ALLOCM_SUCCESS 0 +#define ALLOCM_ERR_OOM 1 +#define ALLOCM_ERR_NOT_MOVED 2 + +extern const char *JEMALLOC_P(malloc_conf); +extern void (*JEMALLOC_P(malloc_message))(void *, const char *); + +void *JEMALLOC_P(malloc)(size_t size) JEMALLOC_ATTR(malloc); +void *JEMALLOC_P(calloc)(size_t num, size_t size) JEMALLOC_ATTR(malloc); +int JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) + JEMALLOC_ATTR(nonnull(1)); +void *JEMALLOC_P(realloc)(void *ptr, size_t size); +void JEMALLOC_P(free)(void *ptr); + +size_t JEMALLOC_P(malloc_usable_size)(const void *ptr); +void JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), + void *cbopaque, const char *opts); +int JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, + void *newp, size_t newlen); +int JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, + size_t *miblenp); +int JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen); + +int JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) + JEMALLOC_ATTR(nonnull(1)); +int JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, + size_t extra, int flags) JEMALLOC_ATTR(nonnull(1)); +int JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) + JEMALLOC_ATTR(nonnull(1)); +int JEMALLOC_P(dallocm)(void *ptr, int flags) JEMALLOC_ATTR(nonnull(1)); + +#ifdef __cplusplus +}; +#endif +#endif /* JEMALLOC_H_ */ diff --git a/deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in b/deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in new file mode 100644 index 00000000000..9ac7e1c2076 --- /dev/null +++ b/deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in @@ -0,0 +1,167 @@ +#ifndef JEMALLOC_DEFS_H_ +#define JEMALLOC_DEFS_H_ + +/* + * If JEMALLOC_PREFIX is defined, it will cause all public APIs to be prefixed. + * This makes it possible, with some care, to use multiple allocators + * simultaneously. + * + * In many cases it is more convenient to manually prefix allocator function + * calls than to let macros do it automatically, particularly when using + * multiple allocators simultaneously. Define JEMALLOC_MANGLE before + * #include'ing jemalloc.h in order to cause name mangling that corresponds to + * the API prefixing. + */ +#undef JEMALLOC_PREFIX +#undef JEMALLOC_CPREFIX +#if (defined(JEMALLOC_PREFIX) && defined(JEMALLOC_MANGLE)) +#undef JEMALLOC_P +#endif + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#undef JEMALLOC_PRIVATE_NAMESPACE +#undef JEMALLOC_N + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#undef CPU_SPINWAIT + +/* + * Defined if OSAtomic*() functions are available, as provided by Darwin, and + * documented in the atomic(3) manual page. + */ +#undef JEMALLOC_OSATOMIC + +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +#undef JEMALLOC_OSSPIN + +/* Defined if __attribute__((...)) syntax is supported. */ +#undef JEMALLOC_HAVE_ATTR +#ifdef JEMALLOC_HAVE_ATTR +# define JEMALLOC_ATTR(s) __attribute__((s)) +#else +# define JEMALLOC_ATTR(s) +#endif + +/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */ +#undef JEMALLOC_CC_SILENCE + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +#undef JEMALLOC_DEBUG + +/* JEMALLOC_STATS enables statistics calculation. */ +#undef JEMALLOC_STATS + +/* JEMALLOC_PROF enables allocation profiling. */ +#undef JEMALLOC_PROF + +/* Use libunwind for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBUNWIND + +/* Use libgcc for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBGCC + +/* Use gcc intrinsics for profile backtracing if defined. */ +#undef JEMALLOC_PROF_GCC + +/* + * JEMALLOC_TINY enables support for tiny objects, which are smaller than one + * quantum. + */ +#undef JEMALLOC_TINY + +/* + * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects. + * This makes it possible to allocate/deallocate objects without any locking + * when the cache is in the steady state. + */ +#undef JEMALLOC_TCACHE + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage + * segment (DSS). + */ +#undef JEMALLOC_DSS + +/* JEMALLOC_SWAP enables mmap()ed swap file support. */ +#undef JEMALLOC_SWAP + +/* Support memory filling (junk/zero). */ +#undef JEMALLOC_FILL + +/* Support optional abort() on OOM. */ +#undef JEMALLOC_XMALLOC + +/* Support SYSV semantics. */ +#undef JEMALLOC_SYSV + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +#undef JEMALLOC_LAZY_LOCK + +/* Determine page size at run time if defined. */ +#undef DYNAMIC_PAGE_SHIFT + +/* One page is 2^STATIC_PAGE_SHIFT bytes. */ +#undef STATIC_PAGE_SHIFT + +/* TLS is used to map arenas and magazine caches to threads. */ +#undef NO_TLS + +/* + * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside + * within jemalloc-owned chunks before dereferencing them. + */ +#undef JEMALLOC_IVSALLOC + +/* + * Define overrides for non-standard allocator-related functions if they + * are present on the system. + */ +#undef JEMALLOC_OVERRIDE_MEMALIGN +#undef JEMALLOC_OVERRIDE_VALLOC + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +#undef JEMALLOC_ZONE +#undef JEMALLOC_ZONE_VERSION + +/* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). */ +#undef JEMALLOC_MREMAP_FIXED + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched. + * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being + * unused, such that they will be discarded rather + * than swapped out. + */ +#undef JEMALLOC_PURGE_MADVISE_DONTNEED +#undef JEMALLOC_PURGE_MADVISE_FREE + +/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ +#undef LG_SIZEOF_PTR + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#undef LG_SIZEOF_INT + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#undef LG_SIZEOF_LONG + +#endif /* JEMALLOC_DEFS_H_ */ diff --git a/deps/jemalloc.orig/install-sh b/deps/jemalloc.orig/install-sh new file mode 100755 index 00000000000..ebc66913e94 --- /dev/null +++ b/deps/jemalloc.orig/install-sh @@ -0,0 +1,250 @@ +#! /bin/sh +# +# install - install a program, script, or datafile +# This comes from X11R5 (mit/util/scripts/install.sh). +# +# Copyright 1991 by the Massachusetts Institute of Technology +# +# Permission to use, copy, modify, distribute, and sell this software and its +# documentation for any purpose is hereby granted without fee, provided that +# the above copyright notice appear in all copies and that both that +# copyright notice and this permission notice appear in supporting +# documentation, and that the name of M.I.T. not be used in advertising or +# publicity pertaining to distribution of the software without specific, +# written prior permission. M.I.T. makes no representations about the +# suitability of this software for any purpose. It is provided "as is" +# without express or implied warranty. +# +# Calling this script install-sh is preferred over install.sh, to prevent +# `make' implicit rules from creating a file called install from it +# when there is no Makefile. +# +# This script is compatible with the BSD install script, but was written +# from scratch. It can only install one file at a time, a restriction +# shared with many OS's install programs. + + +# set DOITPROG to echo to test this script + +# Don't use :- since 4.3BSD and earlier shells don't like it. +doit="${DOITPROG-}" + + +# put in absolute paths if you don't have them in your path; or use env. vars. + +mvprog="${MVPROG-mv}" +cpprog="${CPPROG-cp}" +chmodprog="${CHMODPROG-chmod}" +chownprog="${CHOWNPROG-chown}" +chgrpprog="${CHGRPPROG-chgrp}" +stripprog="${STRIPPROG-strip}" +rmprog="${RMPROG-rm}" +mkdirprog="${MKDIRPROG-mkdir}" + +transformbasename="" +transform_arg="" +instcmd="$mvprog" +chmodcmd="$chmodprog 0755" +chowncmd="" +chgrpcmd="" +stripcmd="" +rmcmd="$rmprog -f" +mvcmd="$mvprog" +src="" +dst="" +dir_arg="" + +while [ x"$1" != x ]; do + case $1 in + -c) instcmd="$cpprog" + shift + continue;; + + -d) dir_arg=true + shift + continue;; + + -m) chmodcmd="$chmodprog $2" + shift + shift + continue;; + + -o) chowncmd="$chownprog $2" + shift + shift + continue;; + + -g) chgrpcmd="$chgrpprog $2" + shift + shift + continue;; + + -s) stripcmd="$stripprog" + shift + continue;; + + -t=*) transformarg=`echo $1 | sed 's/-t=//'` + shift + continue;; + + -b=*) transformbasename=`echo $1 | sed 's/-b=//'` + shift + continue;; + + *) if [ x"$src" = x ] + then + src=$1 + else + # this colon is to work around a 386BSD /bin/sh bug + : + dst=$1 + fi + shift + continue;; + esac +done + +if [ x"$src" = x ] +then + echo "install: no input file specified" + exit 1 +else + true +fi + +if [ x"$dir_arg" != x ]; then + dst=$src + src="" + + if [ -d $dst ]; then + instcmd=: + else + instcmd=mkdir + fi +else + +# Waiting for this to be detected by the "$instcmd $src $dsttmp" command +# might cause directories to be created, which would be especially bad +# if $src (and thus $dsttmp) contains '*'. + + if [ -f $src -o -d $src ] + then + true + else + echo "install: $src does not exist" + exit 1 + fi + + if [ x"$dst" = x ] + then + echo "install: no destination specified" + exit 1 + else + true + fi + +# If destination is a directory, append the input filename; if your system +# does not like double slashes in filenames, you may need to add some logic + + if [ -d $dst ] + then + dst="$dst"/`basename $src` + else + true + fi +fi + +## this sed command emulates the dirname command +dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'` + +# Make sure that the destination directory exists. +# this part is taken from Noah Friedman's mkinstalldirs script + +# Skip lots of stat calls in the usual case. +if [ ! -d "$dstdir" ]; then +defaultIFS=' +' +IFS="${IFS-${defaultIFS}}" + +oIFS="${IFS}" +# Some sh's can't handle IFS=/ for some reason. +IFS='%' +set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'` +IFS="${oIFS}" + +pathcomp='' + +while [ $# -ne 0 ] ; do + pathcomp="${pathcomp}${1}" + shift + + if [ ! -d "${pathcomp}" ] ; + then + $mkdirprog "${pathcomp}" + else + true + fi + + pathcomp="${pathcomp}/" +done +fi + +if [ x"$dir_arg" != x ] +then + $doit $instcmd $dst && + + if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi && + if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi && + if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi && + if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi +else + +# If we're going to rename the final executable, determine the name now. + + if [ x"$transformarg" = x ] + then + dstfile=`basename $dst` + else + dstfile=`basename $dst $transformbasename | + sed $transformarg`$transformbasename + fi + +# don't allow the sed command to completely eliminate the filename + + if [ x"$dstfile" = x ] + then + dstfile=`basename $dst` + else + true + fi + +# Make a temp file name in the proper directory. + + dsttmp=$dstdir/#inst.$$# + +# Move or copy the file name to the temp name + + $doit $instcmd $src $dsttmp && + + trap "rm -f ${dsttmp}" 0 && + +# and set any options; do chmod last to preserve setuid bits + +# If any of these fail, we abort the whole thing. If we want to +# ignore errors from any of these, just make sure not to ignore +# errors from the above "$doit $instcmd $src $dsttmp" command. + + if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi && + if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi && + if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi && + if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi && + +# Now rename the file to the real destination. + + $doit $rmcmd -f $dstdir/$dstfile && + $doit $mvcmd $dsttmp $dstdir/$dstfile + +fi && + + +exit 0 diff --git a/deps/jemalloc.orig/src/arena.c b/deps/jemalloc.orig/src/arena.c new file mode 100644 index 00000000000..d166ca1ec4d --- /dev/null +++ b/deps/jemalloc.orig/src/arena.c @@ -0,0 +1,2704 @@ +#define JEMALLOC_ARENA_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +size_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; +size_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; +ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; +uint8_t const *small_size2bin; +arena_bin_info_t *arena_bin_info; + +/* Various bin-related settings. */ +unsigned nqbins; +unsigned ncbins; +unsigned nsbins; +unsigned nbins; +size_t qspace_max; +size_t cspace_min; +size_t cspace_max; +size_t sspace_min; +size_t sspace_max; + +size_t lg_mspace; +size_t mspace_mask; + +/* + * const_small_size2bin is a static constant lookup table that in the common + * case can be used as-is for small_size2bin. + */ +#if (LG_TINY_MIN == 2) +#define S2B_4(i) i, +#define S2B_8(i) S2B_4(i) S2B_4(i) +#elif (LG_TINY_MIN == 3) +#define S2B_8(i) i, +#else +# error "Unsupported LG_TINY_MIN" +#endif +#define S2B_16(i) S2B_8(i) S2B_8(i) +#define S2B_32(i) S2B_16(i) S2B_16(i) +#define S2B_64(i) S2B_32(i) S2B_32(i) +#define S2B_128(i) S2B_64(i) S2B_64(i) +#define S2B_256(i) S2B_128(i) S2B_128(i) +/* + * The number of elements in const_small_size2bin is dependent on the + * definition for SUBPAGE. + */ +static JEMALLOC_ATTR(aligned(CACHELINE)) + const uint8_t const_small_size2bin[] = { +#if (LG_QUANTUM == 4) +/* 16-byte quantum **********************/ +# ifdef JEMALLOC_TINY +# if (LG_TINY_MIN == 2) + S2B_4(0) /* 4 */ + S2B_4(1) /* 8 */ + S2B_8(2) /* 16 */ +# define S2B_QMIN 2 +# elif (LG_TINY_MIN == 3) + S2B_8(0) /* 8 */ + S2B_8(1) /* 16 */ +# define S2B_QMIN 1 +# else +# error "Unsupported LG_TINY_MIN" +# endif +# else + S2B_16(0) /* 16 */ +# define S2B_QMIN 0 +# endif + S2B_16(S2B_QMIN + 1) /* 32 */ + S2B_16(S2B_QMIN + 2) /* 48 */ + S2B_16(S2B_QMIN + 3) /* 64 */ + S2B_16(S2B_QMIN + 4) /* 80 */ + S2B_16(S2B_QMIN + 5) /* 96 */ + S2B_16(S2B_QMIN + 6) /* 112 */ + S2B_16(S2B_QMIN + 7) /* 128 */ +# define S2B_CMIN (S2B_QMIN + 8) +#else +/* 8-byte quantum ***********************/ +# ifdef JEMALLOC_TINY +# if (LG_TINY_MIN == 2) + S2B_4(0) /* 4 */ + S2B_4(1) /* 8 */ +# define S2B_QMIN 1 +# else +# error "Unsupported LG_TINY_MIN" +# endif +# else + S2B_8(0) /* 8 */ +# define S2B_QMIN 0 +# endif + S2B_8(S2B_QMIN + 1) /* 16 */ + S2B_8(S2B_QMIN + 2) /* 24 */ + S2B_8(S2B_QMIN + 3) /* 32 */ + S2B_8(S2B_QMIN + 4) /* 40 */ + S2B_8(S2B_QMIN + 5) /* 48 */ + S2B_8(S2B_QMIN + 6) /* 56 */ + S2B_8(S2B_QMIN + 7) /* 64 */ + S2B_8(S2B_QMIN + 8) /* 72 */ + S2B_8(S2B_QMIN + 9) /* 80 */ + S2B_8(S2B_QMIN + 10) /* 88 */ + S2B_8(S2B_QMIN + 11) /* 96 */ + S2B_8(S2B_QMIN + 12) /* 104 */ + S2B_8(S2B_QMIN + 13) /* 112 */ + S2B_8(S2B_QMIN + 14) /* 120 */ + S2B_8(S2B_QMIN + 15) /* 128 */ +# define S2B_CMIN (S2B_QMIN + 16) +#endif +/****************************************/ + S2B_64(S2B_CMIN + 0) /* 192 */ + S2B_64(S2B_CMIN + 1) /* 256 */ + S2B_64(S2B_CMIN + 2) /* 320 */ + S2B_64(S2B_CMIN + 3) /* 384 */ + S2B_64(S2B_CMIN + 4) /* 448 */ + S2B_64(S2B_CMIN + 5) /* 512 */ +# define S2B_SMIN (S2B_CMIN + 6) + S2B_256(S2B_SMIN + 0) /* 768 */ + S2B_256(S2B_SMIN + 1) /* 1024 */ + S2B_256(S2B_SMIN + 2) /* 1280 */ + S2B_256(S2B_SMIN + 3) /* 1536 */ + S2B_256(S2B_SMIN + 4) /* 1792 */ + S2B_256(S2B_SMIN + 5) /* 2048 */ + S2B_256(S2B_SMIN + 6) /* 2304 */ + S2B_256(S2B_SMIN + 7) /* 2560 */ + S2B_256(S2B_SMIN + 8) /* 2816 */ + S2B_256(S2B_SMIN + 9) /* 3072 */ + S2B_256(S2B_SMIN + 10) /* 3328 */ + S2B_256(S2B_SMIN + 11) /* 3584 */ + S2B_256(S2B_SMIN + 12) /* 3840 */ +#if (STATIC_PAGE_SHIFT == 13) + S2B_256(S2B_SMIN + 13) /* 4096 */ + S2B_256(S2B_SMIN + 14) /* 4352 */ + S2B_256(S2B_SMIN + 15) /* 4608 */ + S2B_256(S2B_SMIN + 16) /* 4864 */ + S2B_256(S2B_SMIN + 17) /* 5120 */ + S2B_256(S2B_SMIN + 18) /* 5376 */ + S2B_256(S2B_SMIN + 19) /* 5632 */ + S2B_256(S2B_SMIN + 20) /* 5888 */ + S2B_256(S2B_SMIN + 21) /* 6144 */ + S2B_256(S2B_SMIN + 22) /* 6400 */ + S2B_256(S2B_SMIN + 23) /* 6656 */ + S2B_256(S2B_SMIN + 24) /* 6912 */ + S2B_256(S2B_SMIN + 25) /* 7168 */ + S2B_256(S2B_SMIN + 26) /* 7424 */ + S2B_256(S2B_SMIN + 27) /* 7680 */ + S2B_256(S2B_SMIN + 28) /* 7936 */ +#endif +}; +#undef S2B_1 +#undef S2B_2 +#undef S2B_4 +#undef S2B_8 +#undef S2B_16 +#undef S2B_32 +#undef S2B_64 +#undef S2B_128 +#undef S2B_256 +#undef S2B_QMIN +#undef S2B_CMIN +#undef S2B_SMIN + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, + bool large, bool zero); +static arena_chunk_t *arena_chunk_alloc(arena_t *arena); +static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); +static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, + bool zero); +static void arena_purge(arena_t *arena, bool all); +static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); +static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, size_t oldsize, size_t newsize); +static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); +static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); +static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); +static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, + arena_bin_t *bin); +static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, arena_bin_t *bin); +static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, + arena_run_t *run, arena_bin_t *bin); +static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, + void *ptr, size_t oldsize, size_t size); +static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, + void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); +static bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, + size_t extra, bool zero); +static bool small_size2bin_init(void); +#ifdef JEMALLOC_DEBUG +static void small_size2bin_validate(void); +#endif +static bool small_size2bin_init_hard(void); +static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, + size_t min_run_size); +static bool bin_info_init(void); + +/******************************************************************************/ + +static inline int +arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) +{ + uintptr_t a_mapelm = (uintptr_t)a; + uintptr_t b_mapelm = (uintptr_t)b; + + assert(a != NULL); + assert(b != NULL); + + return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); +} + +/* Generate red-black tree functions. */ +rb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, + arena_chunk_map_t, u.rb_link, arena_run_comp) + +static inline int +arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) +{ + int ret; + size_t a_size = a->bits & ~PAGE_MASK; + size_t b_size = b->bits & ~PAGE_MASK; + + assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & + CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); + + ret = (a_size > b_size) - (a_size < b_size); + if (ret == 0) { + uintptr_t a_mapelm, b_mapelm; + + if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) + a_mapelm = (uintptr_t)a; + else { + /* + * Treat keys as though they are lower than anything + * else. + */ + a_mapelm = 0; + } + b_mapelm = (uintptr_t)b; + + ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); + } + + return (ret); +} + +/* Generate red-black tree functions. */ +rb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, + arena_chunk_map_t, u.rb_link, arena_avail_comp) + +static inline void * +arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) +{ + void *ret; + unsigned regind; + bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + + (uintptr_t)bin_info->bitmap_offset); + + dassert(run->magic == ARENA_RUN_MAGIC); + assert(run->nfree > 0); + assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); + + regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); + ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + + (uintptr_t)(bin_info->reg_size * regind)); + run->nfree--; + if (regind == run->nextind) + run->nextind++; + assert(regind < run->nextind); + return (ret); +} + +static inline void +arena_run_reg_dalloc(arena_run_t *run, void *ptr) +{ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + size_t binind = arena_bin_index(chunk->arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + unsigned regind = arena_run_regind(run, bin_info, ptr); + bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + + (uintptr_t)bin_info->bitmap_offset); + + assert(run->nfree < bin_info->nregs); + /* Freeing an interior pointer can cause assertion failure. */ + assert(((uintptr_t)ptr - ((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size + == 0); + assert((uintptr_t)ptr >= (uintptr_t)run + + (uintptr_t)bin_info->reg0_offset); + /* Freeing an unallocated pointer can cause assertion failure. */ + assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); + + bitmap_unset(bitmap, &bin_info->bitmap_info, regind); + run->nfree++; +} + +#ifdef JEMALLOC_DEBUG +static inline void +arena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) +{ + size_t i; + size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << PAGE_SHIFT)); + + for (i = 0; i < PAGE_SIZE / sizeof(size_t); i++) + assert(p[i] == 0); +} +#endif + +static void +arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, + bool zero) +{ + arena_chunk_t *chunk; + size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; + size_t flag_dirty; + arena_avail_tree_t *runs_avail; +#ifdef JEMALLOC_STATS + size_t cactive_diff; +#endif + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + old_ndirty = chunk->ndirty; + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) + >> PAGE_SHIFT); + flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; + runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : + &arena->runs_avail_clean; + total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> + PAGE_SHIFT; + assert((chunk->map[run_ind+total_pages-1-map_bias].bits & + CHUNK_MAP_DIRTY) == flag_dirty); + need_pages = (size >> PAGE_SHIFT); + assert(need_pages > 0); + assert(need_pages <= total_pages); + rem_pages = total_pages - need_pages; + + arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); +#ifdef JEMALLOC_STATS + /* Update stats_cactive if nactive is crossing a chunk multiple. */ + cactive_diff = CHUNK_CEILING((arena->nactive + need_pages) << + PAGE_SHIFT) - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); + if (cactive_diff != 0) + stats_cactive_add(cactive_diff); +#endif + arena->nactive += need_pages; + + /* Keep track of trailing unused pages for later use. */ + if (rem_pages > 0) { + if (flag_dirty != 0) { + chunk->map[run_ind+need_pages-map_bias].bits = + (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; + chunk->map[run_ind+total_pages-1-map_bias].bits = + (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; + } else { + chunk->map[run_ind+need_pages-map_bias].bits = + (rem_pages << PAGE_SHIFT) | + (chunk->map[run_ind+need_pages-map_bias].bits & + CHUNK_MAP_UNZEROED); + chunk->map[run_ind+total_pages-1-map_bias].bits = + (rem_pages << PAGE_SHIFT) | + (chunk->map[run_ind+total_pages-1-map_bias].bits & + CHUNK_MAP_UNZEROED); + } + arena_avail_tree_insert(runs_avail, + &chunk->map[run_ind+need_pages-map_bias]); + } + + /* Update dirty page accounting. */ + if (flag_dirty != 0) { + chunk->ndirty -= need_pages; + arena->ndirty -= need_pages; + } + + /* + * Update the page map separately for large vs. small runs, since it is + * possible to avoid iteration for large mallocs. + */ + if (large) { + if (zero) { + if (flag_dirty == 0) { + /* + * The run is clean, so some pages may be + * zeroed (i.e. never before touched). + */ + for (i = 0; i < need_pages; i++) { + if ((chunk->map[run_ind+i-map_bias].bits + & CHUNK_MAP_UNZEROED) != 0) { + memset((void *)((uintptr_t) + chunk + ((run_ind+i) << + PAGE_SHIFT)), 0, + PAGE_SIZE); + } +#ifdef JEMALLOC_DEBUG + else { + arena_chunk_validate_zeroed( + chunk, run_ind+i); + } +#endif + } + } else { + /* + * The run is dirty, so all pages must be + * zeroed. + */ + memset((void *)((uintptr_t)chunk + (run_ind << + PAGE_SHIFT)), 0, (need_pages << + PAGE_SHIFT)); + } + } + + /* + * Set the last element first, in case the run only contains one + * page (i.e. both statements set the same element). + */ + chunk->map[run_ind+need_pages-1-map_bias].bits = + CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; + chunk->map[run_ind-map_bias].bits = size | flag_dirty | + CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + } else { + assert(zero == false); + /* + * Propagate the dirty and unzeroed flags to the allocated + * small run, so that arena_dalloc_bin_run() has the ability to + * conditionally trim clean pages. + */ + chunk->map[run_ind-map_bias].bits = + (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | + CHUNK_MAP_ALLOCATED | flag_dirty; +#ifdef JEMALLOC_DEBUG + /* + * The first page will always be dirtied during small run + * initialization, so a validation failure here would not + * actually cause an observable failure. + */ + if (flag_dirty == 0 && + (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) + == 0) + arena_chunk_validate_zeroed(chunk, run_ind); +#endif + for (i = 1; i < need_pages - 1; i++) { + chunk->map[run_ind+i-map_bias].bits = (i << PAGE_SHIFT) + | (chunk->map[run_ind+i-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; +#ifdef JEMALLOC_DEBUG + if (flag_dirty == 0 && + (chunk->map[run_ind+i-map_bias].bits & + CHUNK_MAP_UNZEROED) == 0) + arena_chunk_validate_zeroed(chunk, run_ind+i); +#endif + } + chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages + - 1) << PAGE_SHIFT) | + (chunk->map[run_ind+need_pages-1-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; +#ifdef JEMALLOC_DEBUG + if (flag_dirty == 0 && + (chunk->map[run_ind+need_pages-1-map_bias].bits & + CHUNK_MAP_UNZEROED) == 0) { + arena_chunk_validate_zeroed(chunk, + run_ind+need_pages-1); + } +#endif + } +} + +static arena_chunk_t * +arena_chunk_alloc(arena_t *arena) +{ + arena_chunk_t *chunk; + size_t i; + + if (arena->spare != NULL) { + arena_avail_tree_t *runs_avail; + + chunk = arena->spare; + arena->spare = NULL; + + /* Insert the run into the appropriate runs_avail_* tree. */ + if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) + runs_avail = &arena->runs_avail_clean; + else + runs_avail = &arena->runs_avail_dirty; + assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); + assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) + == arena_maxclass); + assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == + (chunk->map[chunk_npages-1-map_bias].bits & + CHUNK_MAP_DIRTY)); + arena_avail_tree_insert(runs_avail, &chunk->map[0]); + } else { + bool zero; + size_t unzeroed; + + zero = false; + malloc_mutex_unlock(&arena->lock); + chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); + malloc_mutex_lock(&arena->lock); + if (chunk == NULL) + return (NULL); +#ifdef JEMALLOC_STATS + arena->stats.mapped += chunksize; +#endif + + chunk->arena = arena; + ql_elm_new(chunk, link_dirty); + chunk->dirtied = false; + + /* + * Claim that no pages are in use, since the header is merely + * overhead. + */ + chunk->ndirty = 0; + + /* + * Initialize the map to contain one maximal free untouched run. + * Mark the pages as zeroed iff chunk_alloc() returned a zeroed + * chunk. + */ + unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; + chunk->map[0].bits = arena_maxclass | unzeroed; + /* + * There is no need to initialize the internal page map entries + * unless the chunk is not zeroed. + */ + if (zero == false) { + for (i = map_bias+1; i < chunk_npages-1; i++) + chunk->map[i-map_bias].bits = unzeroed; + } +#ifdef JEMALLOC_DEBUG + else { + for (i = map_bias+1; i < chunk_npages-1; i++) + assert(chunk->map[i-map_bias].bits == unzeroed); + } +#endif + chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | + unzeroed; + + /* Insert the run into the runs_avail_clean tree. */ + arena_avail_tree_insert(&arena->runs_avail_clean, + &chunk->map[0]); + } + + return (chunk); +} + +static void +arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) +{ + arena_avail_tree_t *runs_avail; + + /* + * Remove run from the appropriate runs_avail_* tree, so that the arena + * does not use it. + */ + if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) + runs_avail = &arena->runs_avail_clean; + else + runs_avail = &arena->runs_avail_dirty; + arena_avail_tree_remove(runs_avail, &chunk->map[0]); + + if (arena->spare != NULL) { + arena_chunk_t *spare = arena->spare; + + arena->spare = chunk; + if (spare->dirtied) { + ql_remove(&chunk->arena->chunks_dirty, spare, + link_dirty); + arena->ndirty -= spare->ndirty; + } + malloc_mutex_unlock(&arena->lock); + chunk_dealloc((void *)spare, chunksize, true); + malloc_mutex_lock(&arena->lock); +#ifdef JEMALLOC_STATS + arena->stats.mapped -= chunksize; +#endif + } else + arena->spare = chunk; +} + +static arena_run_t * +arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) +{ + arena_chunk_t *chunk; + arena_run_t *run; + arena_chunk_map_t *mapelm, key; + + assert(size <= arena_maxclass); + assert((size & PAGE_MASK) == 0); + + /* Search the arena's chunks for the lowest best fit. */ + key.bits = size | CHUNK_MAP_KEY; + mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); + if (mapelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); + size_t pageind = (((uintptr_t)mapelm - + (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) + + map_bias; + + run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << + PAGE_SHIFT)); + arena_run_split(arena, run, size, large, zero); + return (run); + } + mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); + if (mapelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); + size_t pageind = (((uintptr_t)mapelm - + (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) + + map_bias; + + run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << + PAGE_SHIFT)); + arena_run_split(arena, run, size, large, zero); + return (run); + } + + /* + * No usable runs. Create a new chunk from which to allocate the run. + */ + chunk = arena_chunk_alloc(arena); + if (chunk != NULL) { + run = (arena_run_t *)((uintptr_t)chunk + (map_bias << + PAGE_SHIFT)); + arena_run_split(arena, run, size, large, zero); + return (run); + } + + /* + * arena_chunk_alloc() failed, but another thread may have made + * sufficient memory available while this one dropped arena->lock in + * arena_chunk_alloc(), so search one more time. + */ + mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); + if (mapelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); + size_t pageind = (((uintptr_t)mapelm - + (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) + + map_bias; + + run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << + PAGE_SHIFT)); + arena_run_split(arena, run, size, large, zero); + return (run); + } + mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); + if (mapelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); + size_t pageind = (((uintptr_t)mapelm - + (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) + + map_bias; + + run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << + PAGE_SHIFT)); + arena_run_split(arena, run, size, large, zero); + return (run); + } + + return (NULL); +} + +static inline void +arena_maybe_purge(arena_t *arena) +{ + + /* Enforce opt_lg_dirty_mult. */ + if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && + (arena->ndirty - arena->npurgatory) > chunk_npages && + (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - + arena->npurgatory)) + arena_purge(arena, false); +} + +static inline void +arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) +{ + ql_head(arena_chunk_map_t) mapelms; + arena_chunk_map_t *mapelm; + size_t pageind, flag_unzeroed; +#ifdef JEMALLOC_DEBUG + size_t ndirty; +#endif +#ifdef JEMALLOC_STATS + size_t nmadvise; +#endif + + ql_new(&mapelms); + + flag_unzeroed = +#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED + /* + * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous + * mappings, but not for file-backed mappings. + */ +# ifdef JEMALLOC_SWAP + swap_enabled ? CHUNK_MAP_UNZEROED : +# endif + 0; +#else + CHUNK_MAP_UNZEROED; +#endif + + /* + * If chunk is the spare, temporarily re-allocate it, 1) so that its + * run is reinserted into runs_avail_dirty, and 2) so that it cannot be + * completely discarded by another thread while arena->lock is dropped + * by this thread. Note that the arena_run_dalloc() call will + * implicitly deallocate the chunk, so no explicit action is required + * in this function to deallocate the chunk. + * + * Note that once a chunk contains dirty pages, it cannot again contain + * a single run unless 1) it is a dirty run, or 2) this function purges + * dirty pages and causes the transition to a single clean run. Thus + * (chunk == arena->spare) is possible, but it is not possible for + * this function to be called on the spare unless it contains a dirty + * run. + */ + if (chunk == arena->spare) { + assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); + arena_chunk_alloc(arena); + } + + /* Temporarily allocate all free dirty runs within chunk. */ + for (pageind = map_bias; pageind < chunk_npages;) { + mapelm = &chunk->map[pageind-map_bias]; + if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { + size_t npages; + + npages = mapelm->bits >> PAGE_SHIFT; + assert(pageind + npages <= chunk_npages); + if (mapelm->bits & CHUNK_MAP_DIRTY) { + size_t i; +#ifdef JEMALLOC_STATS + size_t cactive_diff; +#endif + + arena_avail_tree_remove( + &arena->runs_avail_dirty, mapelm); + + mapelm->bits = (npages << PAGE_SHIFT) | + flag_unzeroed | CHUNK_MAP_LARGE | + CHUNK_MAP_ALLOCATED; + /* + * Update internal elements in the page map, so + * that CHUNK_MAP_UNZEROED is properly set. + */ + for (i = 1; i < npages - 1; i++) { + chunk->map[pageind+i-map_bias].bits = + flag_unzeroed; + } + if (npages > 1) { + chunk->map[ + pageind+npages-1-map_bias].bits = + flag_unzeroed | CHUNK_MAP_LARGE | + CHUNK_MAP_ALLOCATED; + } + +#ifdef JEMALLOC_STATS + /* + * Update stats_cactive if nactive is crossing a + * chunk multiple. + */ + cactive_diff = CHUNK_CEILING((arena->nactive + + npages) << PAGE_SHIFT) - + CHUNK_CEILING(arena->nactive << PAGE_SHIFT); + if (cactive_diff != 0) + stats_cactive_add(cactive_diff); +#endif + arena->nactive += npages; + /* Append to list for later processing. */ + ql_elm_new(mapelm, u.ql_link); + ql_tail_insert(&mapelms, mapelm, u.ql_link); + } + + pageind += npages; + } else { + /* Skip allocated run. */ + if (mapelm->bits & CHUNK_MAP_LARGE) + pageind += mapelm->bits >> PAGE_SHIFT; + else { + arena_run_t *run = (arena_run_t *)((uintptr_t) + chunk + (uintptr_t)(pageind << PAGE_SHIFT)); + + assert((mapelm->bits >> PAGE_SHIFT) == 0); + dassert(run->magic == ARENA_RUN_MAGIC); + size_t binind = arena_bin_index(arena, + run->bin); + arena_bin_info_t *bin_info = + &arena_bin_info[binind]; + pageind += bin_info->run_size >> PAGE_SHIFT; + } + } + } + assert(pageind == chunk_npages); + +#ifdef JEMALLOC_DEBUG + ndirty = chunk->ndirty; +#endif +#ifdef JEMALLOC_STATS + arena->stats.purged += chunk->ndirty; +#endif + arena->ndirty -= chunk->ndirty; + chunk->ndirty = 0; + ql_remove(&arena->chunks_dirty, chunk, link_dirty); + chunk->dirtied = false; + + malloc_mutex_unlock(&arena->lock); +#ifdef JEMALLOC_STATS + nmadvise = 0; +#endif + ql_foreach(mapelm, &mapelms, u.ql_link) { + size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / + sizeof(arena_chunk_map_t)) + map_bias; + size_t npages = mapelm->bits >> PAGE_SHIFT; + + assert(pageind + npages <= chunk_npages); +#ifdef JEMALLOC_DEBUG + assert(ndirty >= npages); + ndirty -= npages; +#endif + +#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED + madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), + (npages << PAGE_SHIFT), MADV_DONTNEED); +#elif defined(JEMALLOC_PURGE_MADVISE_FREE) + madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), + (npages << PAGE_SHIFT), MADV_FREE); +#else +# error "No method defined for purging unused dirty pages." +#endif + +#ifdef JEMALLOC_STATS + nmadvise++; +#endif + } +#ifdef JEMALLOC_DEBUG + assert(ndirty == 0); +#endif + malloc_mutex_lock(&arena->lock); +#ifdef JEMALLOC_STATS + arena->stats.nmadvise += nmadvise; +#endif + + /* Deallocate runs. */ + for (mapelm = ql_first(&mapelms); mapelm != NULL; + mapelm = ql_first(&mapelms)) { + size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / + sizeof(arena_chunk_map_t)) + map_bias; + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)(pageind << PAGE_SHIFT)); + + ql_remove(&mapelms, mapelm, u.ql_link); + arena_run_dalloc(arena, run, false); + } +} + +static void +arena_purge(arena_t *arena, bool all) +{ + arena_chunk_t *chunk; + size_t npurgatory; +#ifdef JEMALLOC_DEBUG + size_t ndirty = 0; + + ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { + assert(chunk->dirtied); + ndirty += chunk->ndirty; + } + assert(ndirty == arena->ndirty); +#endif + assert(arena->ndirty > arena->npurgatory || all); + assert(arena->ndirty - arena->npurgatory > chunk_npages || all); + assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - + arena->npurgatory) || all); + +#ifdef JEMALLOC_STATS + arena->stats.npurge++; +#endif + + /* + * Compute the minimum number of pages that this thread should try to + * purge, and add the result to arena->npurgatory. This will keep + * multiple threads from racing to reduce ndirty below the threshold. + */ + npurgatory = arena->ndirty - arena->npurgatory; + if (all == false) { + assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); + npurgatory -= arena->nactive >> opt_lg_dirty_mult; + } + arena->npurgatory += npurgatory; + + while (npurgatory > 0) { + /* Get next chunk with dirty pages. */ + chunk = ql_first(&arena->chunks_dirty); + if (chunk == NULL) { + /* + * This thread was unable to purge as many pages as + * originally intended, due to races with other threads + * that either did some of the purging work, or re-used + * dirty pages. + */ + arena->npurgatory -= npurgatory; + return; + } + while (chunk->ndirty == 0) { + ql_remove(&arena->chunks_dirty, chunk, link_dirty); + chunk->dirtied = false; + chunk = ql_first(&arena->chunks_dirty); + if (chunk == NULL) { + /* Same logic as for above. */ + arena->npurgatory -= npurgatory; + return; + } + } + + if (chunk->ndirty > npurgatory) { + /* + * This thread will, at a minimum, purge all the dirty + * pages in chunk, so set npurgatory to reflect this + * thread's commitment to purge the pages. This tends + * to reduce the chances of the following scenario: + * + * 1) This thread sets arena->npurgatory such that + * (arena->ndirty - arena->npurgatory) is at the + * threshold. + * 2) This thread drops arena->lock. + * 3) Another thread causes one or more pages to be + * dirtied, and immediately determines that it must + * purge dirty pages. + * + * If this scenario *does* play out, that's okay, + * because all of the purging work being done really + * needs to happen. + */ + arena->npurgatory += chunk->ndirty - npurgatory; + npurgatory = chunk->ndirty; + } + + arena->npurgatory -= chunk->ndirty; + npurgatory -= chunk->ndirty; + arena_chunk_purge(arena, chunk); + } +} + +void +arena_purge_all(arena_t *arena) +{ + + malloc_mutex_lock(&arena->lock); + arena_purge(arena, true); + malloc_mutex_unlock(&arena->lock); +} + +static void +arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) +{ + arena_chunk_t *chunk; + size_t size, run_ind, run_pages, flag_dirty; + arena_avail_tree_t *runs_avail; +#ifdef JEMALLOC_STATS + size_t cactive_diff; +#endif + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) + >> PAGE_SHIFT); + assert(run_ind >= map_bias); + assert(run_ind < chunk_npages); + if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { + size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; + assert(size == PAGE_SIZE || + (chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & + ~PAGE_MASK) == 0); + assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & + CHUNK_MAP_LARGE) != 0); + assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & + CHUNK_MAP_ALLOCATED) != 0); + } else { + size_t binind = arena_bin_index(arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + size = bin_info->run_size; + } + run_pages = (size >> PAGE_SHIFT); +#ifdef JEMALLOC_STATS + /* Update stats_cactive if nactive is crossing a chunk multiple. */ + cactive_diff = CHUNK_CEILING(arena->nactive << PAGE_SHIFT) - + CHUNK_CEILING((arena->nactive - run_pages) << PAGE_SHIFT); + if (cactive_diff != 0) + stats_cactive_sub(cactive_diff); +#endif + arena->nactive -= run_pages; + + /* + * The run is dirty if the caller claims to have dirtied it, as well as + * if it was already dirty before being allocated. + */ + if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) + dirty = true; + flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; + runs_avail = dirty ? &arena->runs_avail_dirty : + &arena->runs_avail_clean; + + /* Mark pages as unallocated in the chunk map. */ + if (dirty) { + chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; + chunk->map[run_ind+run_pages-1-map_bias].bits = size | + CHUNK_MAP_DIRTY; + + chunk->ndirty += run_pages; + arena->ndirty += run_pages; + } else { + chunk->map[run_ind-map_bias].bits = size | + (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); + chunk->map[run_ind+run_pages-1-map_bias].bits = size | + (chunk->map[run_ind+run_pages-1-map_bias].bits & + CHUNK_MAP_UNZEROED); + } + + /* Try to coalesce forward. */ + if (run_ind + run_pages < chunk_npages && + (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) + == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & + CHUNK_MAP_DIRTY) == flag_dirty) { + size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & + ~PAGE_MASK; + size_t nrun_pages = nrun_size >> PAGE_SHIFT; + + /* + * Remove successor from runs_avail; the coalesced run is + * inserted later. + */ + assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits + & ~PAGE_MASK) == nrun_size); + assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits + & CHUNK_MAP_ALLOCATED) == 0); + assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits + & CHUNK_MAP_DIRTY) == flag_dirty); + arena_avail_tree_remove(runs_avail, + &chunk->map[run_ind+run_pages-map_bias]); + + size += nrun_size; + run_pages += nrun_pages; + + chunk->map[run_ind-map_bias].bits = size | + (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); + chunk->map[run_ind+run_pages-1-map_bias].bits = size | + (chunk->map[run_ind+run_pages-1-map_bias].bits & + CHUNK_MAP_FLAGS_MASK); + } + + /* Try to coalesce backward. */ + if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & + CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & + CHUNK_MAP_DIRTY) == flag_dirty) { + size_t prun_size = chunk->map[run_ind-1-map_bias].bits & + ~PAGE_MASK; + size_t prun_pages = prun_size >> PAGE_SHIFT; + + run_ind -= prun_pages; + + /* + * Remove predecessor from runs_avail; the coalesced run is + * inserted later. + */ + assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) + == prun_size); + assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) + == 0); + assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) + == flag_dirty); + arena_avail_tree_remove(runs_avail, + &chunk->map[run_ind-map_bias]); + + size += prun_size; + run_pages += prun_pages; + + chunk->map[run_ind-map_bias].bits = size | + (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); + chunk->map[run_ind+run_pages-1-map_bias].bits = size | + (chunk->map[run_ind+run_pages-1-map_bias].bits & + CHUNK_MAP_FLAGS_MASK); + } + + /* Insert into runs_avail, now that coalescing is complete. */ + assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == + (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); + assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == + (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); + arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); + + if (dirty) { + /* + * Insert into chunks_dirty before potentially calling + * arena_chunk_dealloc(), so that chunks_dirty and + * arena->ndirty are consistent. + */ + if (chunk->dirtied == false) { + ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); + chunk->dirtied = true; + } + } + + /* + * Deallocate chunk if it is now completely unused. The bit + * manipulation checks whether the first run is unallocated and extends + * to the end of the chunk. + */ + if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == + arena_maxclass) + arena_chunk_dealloc(arena, chunk); + + /* + * It is okay to do dirty page processing here even if the chunk was + * deallocated above, since in that case it is the spare. Waiting + * until after possible chunk deallocation to do dirty processing + * allows for an old spare to be fully deallocated, thus decreasing the + * chances of spuriously crossing the dirty page purging threshold. + */ + if (dirty) + arena_maybe_purge(arena); +} + +static void +arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, + size_t oldsize, size_t newsize) +{ + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; + size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; + size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; + + assert(oldsize > newsize); + + /* + * Update the chunk map so that arena_run_dalloc() can treat the + * leading run as separately allocated. Set the last element of each + * run first, in case of single-page runs. + */ + assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); + assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); + chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | + (chunk->map[pageind+head_npages-1-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + chunk->map[pageind-map_bias].bits = (oldsize - newsize) + | flag_dirty | (chunk->map[pageind-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + +#ifdef JEMALLOC_DEBUG + { + size_t tail_npages = newsize >> PAGE_SHIFT; + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] + .bits & ~PAGE_MASK) == 0); + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] + .bits & CHUNK_MAP_DIRTY) == flag_dirty); + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] + .bits & CHUNK_MAP_LARGE) != 0); + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] + .bits & CHUNK_MAP_ALLOCATED) != 0); + } +#endif + chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | + (chunk->map[pageind+head_npages-map_bias].bits & + CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + + arena_run_dalloc(arena, run, false); +} + +static void +arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, + size_t oldsize, size_t newsize, bool dirty) +{ + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; + size_t head_npages = newsize >> PAGE_SHIFT; + size_t tail_npages = (oldsize - newsize) >> PAGE_SHIFT; + size_t flag_dirty = chunk->map[pageind-map_bias].bits & + CHUNK_MAP_DIRTY; + + assert(oldsize > newsize); + + /* + * Update the chunk map so that arena_run_dalloc() can treat the + * trailing run as separately allocated. Set the last element of each + * run first, in case of single-page runs. + */ + assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); + assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); + chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | + (chunk->map[pageind+head_npages-1-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + chunk->map[pageind-map_bias].bits = newsize | flag_dirty | + (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | + CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & + ~PAGE_MASK) == 0); + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & + CHUNK_MAP_LARGE) != 0); + assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & + CHUNK_MAP_ALLOCATED) != 0); + chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = + flag_dirty | + (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | + flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & + CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + + arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), + dirty); +} + +static arena_run_t * +arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) +{ + arena_chunk_map_t *mapelm; + arena_run_t *run; + size_t binind; + arena_bin_info_t *bin_info; + + /* Look for a usable run. */ + mapelm = arena_run_tree_first(&bin->runs); + if (mapelm != NULL) { + arena_chunk_t *chunk; + size_t pageind; + + /* run is guaranteed to have available space. */ + arena_run_tree_remove(&bin->runs, mapelm); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); + pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / + sizeof(arena_chunk_map_t))) + map_bias; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + (mapelm->bits >> PAGE_SHIFT)) + << PAGE_SHIFT)); +#ifdef JEMALLOC_STATS + bin->stats.reruns++; +#endif + return (run); + } + /* No existing runs have any space available. */ + + binind = arena_bin_index(arena, bin); + bin_info = &arena_bin_info[binind]; + + /* Allocate a new run. */ + malloc_mutex_unlock(&bin->lock); + /******************************/ + malloc_mutex_lock(&arena->lock); + run = arena_run_alloc(arena, bin_info->run_size, false, false); + if (run != NULL) { + bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + + (uintptr_t)bin_info->bitmap_offset); + + /* Initialize run internals. */ + run->bin = bin; + run->nextind = 0; + run->nfree = bin_info->nregs; + bitmap_init(bitmap, &bin_info->bitmap_info); +#ifdef JEMALLOC_DEBUG + run->magic = ARENA_RUN_MAGIC; +#endif + } + malloc_mutex_unlock(&arena->lock); + /********************************/ + malloc_mutex_lock(&bin->lock); + if (run != NULL) { +#ifdef JEMALLOC_STATS + bin->stats.nruns++; + bin->stats.curruns++; + if (bin->stats.curruns > bin->stats.highruns) + bin->stats.highruns = bin->stats.curruns; +#endif + return (run); + } + + /* + * arena_run_alloc() failed, but another thread may have made + * sufficient memory available while this one dropped bin->lock above, + * so search one more time. + */ + mapelm = arena_run_tree_first(&bin->runs); + if (mapelm != NULL) { + arena_chunk_t *chunk; + size_t pageind; + + /* run is guaranteed to have available space. */ + arena_run_tree_remove(&bin->runs, mapelm); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); + pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / + sizeof(arena_chunk_map_t))) + map_bias; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + (mapelm->bits >> PAGE_SHIFT)) + << PAGE_SHIFT)); +#ifdef JEMALLOC_STATS + bin->stats.reruns++; +#endif + return (run); + } + + return (NULL); +} + +/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ +static void * +arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) +{ + void *ret; + size_t binind; + arena_bin_info_t *bin_info; + arena_run_t *run; + + binind = arena_bin_index(arena, bin); + bin_info = &arena_bin_info[binind]; + bin->runcur = NULL; + run = arena_bin_nonfull_run_get(arena, bin); + if (bin->runcur != NULL && bin->runcur->nfree > 0) { + /* + * Another thread updated runcur while this one ran without the + * bin lock in arena_bin_nonfull_run_get(). + */ + dassert(bin->runcur->magic == ARENA_RUN_MAGIC); + assert(bin->runcur->nfree > 0); + ret = arena_run_reg_alloc(bin->runcur, bin_info); + if (run != NULL) { + arena_chunk_t *chunk; + + /* + * arena_run_alloc() may have allocated run, or it may + * have pulled run from the bin's run tree. Therefore + * it is unsafe to make any assumptions about how run + * has previously been used, and arena_bin_lower_run() + * must be called, as if a region were just deallocated + * from the run. + */ + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + if (run->nfree == bin_info->nregs) + arena_dalloc_bin_run(arena, chunk, run, bin); + else + arena_bin_lower_run(arena, chunk, run, bin); + } + return (ret); + } + + if (run == NULL) + return (NULL); + + bin->runcur = run; + + dassert(bin->runcur->magic == ARENA_RUN_MAGIC); + assert(bin->runcur->nfree > 0); + + return (arena_run_reg_alloc(bin->runcur, bin_info)); +} + +#ifdef JEMALLOC_PROF +void +arena_prof_accum(arena_t *arena, uint64_t accumbytes) +{ + + if (prof_interval != 0) { + arena->prof_accumbytes += accumbytes; + if (arena->prof_accumbytes >= prof_interval) { + prof_idump(); + arena->prof_accumbytes -= prof_interval; + } + } +} +#endif + +#ifdef JEMALLOC_TCACHE +void +arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind +# ifdef JEMALLOC_PROF + , uint64_t prof_accumbytes +# endif + ) +{ + unsigned i, nfill; + arena_bin_t *bin; + arena_run_t *run; + void *ptr; + + assert(tbin->ncached == 0); + +#ifdef JEMALLOC_PROF + malloc_mutex_lock(&arena->lock); + arena_prof_accum(arena, prof_accumbytes); + malloc_mutex_unlock(&arena->lock); +#endif + bin = &arena->bins[binind]; + malloc_mutex_lock(&bin->lock); + for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> + tbin->lg_fill_div); i < nfill; i++) { + if ((run = bin->runcur) != NULL && run->nfree > 0) + ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); + else + ptr = arena_bin_malloc_hard(arena, bin); + if (ptr == NULL) + break; + /* Insert such that low regions get used first. */ + tbin->avail[nfill - 1 - i] = ptr; + } +#ifdef JEMALLOC_STATS + bin->stats.allocated += i * arena_bin_info[binind].reg_size; + bin->stats.nmalloc += i; + bin->stats.nrequests += tbin->tstats.nrequests; + bin->stats.nfills++; + tbin->tstats.nrequests = 0; +#endif + malloc_mutex_unlock(&bin->lock); + tbin->ncached = i; +} +#endif + +void * +arena_malloc_small(arena_t *arena, size_t size, bool zero) +{ + void *ret; + arena_bin_t *bin; + arena_run_t *run; + size_t binind; + + binind = SMALL_SIZE2BIN(size); + assert(binind < nbins); + bin = &arena->bins[binind]; + size = arena_bin_info[binind].reg_size; + + malloc_mutex_lock(&bin->lock); + if ((run = bin->runcur) != NULL && run->nfree > 0) + ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); + else + ret = arena_bin_malloc_hard(arena, bin); + + if (ret == NULL) { + malloc_mutex_unlock(&bin->lock); + return (NULL); + } + +#ifdef JEMALLOC_STATS + bin->stats.allocated += size; + bin->stats.nmalloc++; + bin->stats.nrequests++; +#endif + malloc_mutex_unlock(&bin->lock); +#ifdef JEMALLOC_PROF + if (isthreaded == false) { + malloc_mutex_lock(&arena->lock); + arena_prof_accum(arena, size); + malloc_mutex_unlock(&arena->lock); + } +#endif + + if (zero == false) { +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); +#endif + } else + memset(ret, 0, size); + + return (ret); +} + +void * +arena_malloc_large(arena_t *arena, size_t size, bool zero) +{ + void *ret; + + /* Large allocation. */ + size = PAGE_CEILING(size); + malloc_mutex_lock(&arena->lock); + ret = (void *)arena_run_alloc(arena, size, true, zero); + if (ret == NULL) { + malloc_mutex_unlock(&arena->lock); + return (NULL); + } +#ifdef JEMALLOC_STATS + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; + if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + } +#endif +#ifdef JEMALLOC_PROF + arena_prof_accum(arena, size); +#endif + malloc_mutex_unlock(&arena->lock); + + if (zero == false) { +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); +#endif + } + + return (ret); +} + +void * +arena_malloc(size_t size, bool zero) +{ + + assert(size != 0); + assert(QUANTUM_CEILING(size) <= arena_maxclass); + + if (size <= small_maxclass) { +#ifdef JEMALLOC_TCACHE + tcache_t *tcache; + + if ((tcache = tcache_get()) != NULL) + return (tcache_alloc_small(tcache, size, zero)); + else + +#endif + return (arena_malloc_small(choose_arena(), size, zero)); + } else { +#ifdef JEMALLOC_TCACHE + if (size <= tcache_maxclass) { + tcache_t *tcache; + + if ((tcache = tcache_get()) != NULL) + return (tcache_alloc_large(tcache, size, zero)); + else { + return (arena_malloc_large(choose_arena(), + size, zero)); + } + } else +#endif + return (arena_malloc_large(choose_arena(), size, zero)); + } +} + +/* Only handles large allocations that require more than page alignment. */ +void * +arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, + bool zero) +{ + void *ret; + size_t offset; + arena_chunk_t *chunk; + + assert((size & PAGE_MASK) == 0); + + alignment = PAGE_CEILING(alignment); + + malloc_mutex_lock(&arena->lock); + ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); + if (ret == NULL) { + malloc_mutex_unlock(&arena->lock); + return (NULL); + } + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); + + offset = (uintptr_t)ret & (alignment - 1); + assert((offset & PAGE_MASK) == 0); + assert(offset < alloc_size); + if (offset == 0) + arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); + else { + size_t leadsize, trailsize; + + leadsize = alignment - offset; + if (leadsize > 0) { + arena_run_trim_head(arena, chunk, ret, alloc_size, + alloc_size - leadsize); + ret = (void *)((uintptr_t)ret + leadsize); + } + + trailsize = alloc_size - leadsize - size; + if (trailsize != 0) { + /* Trim trailing space. */ + assert(trailsize < alloc_size); + arena_run_trim_tail(arena, chunk, ret, size + trailsize, + size, false); + } + } + +#ifdef JEMALLOC_STATS + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; + if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + } +#endif + malloc_mutex_unlock(&arena->lock); + +#ifdef JEMALLOC_FILL + if (zero == false) { + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); + } +#endif + return (ret); +} + +/* Return the size of the allocation pointed to by ptr. */ +size_t +arena_salloc(const void *ptr) +{ + size_t ret; + arena_chunk_t *chunk; + size_t pageind, mapbits; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + mapbits = chunk->map[pageind-map_bias].bits; + assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); + if ((mapbits & CHUNK_MAP_LARGE) == 0) { + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << + PAGE_SHIFT)); + dassert(run->magic == ARENA_RUN_MAGIC); + size_t binind = arena_bin_index(chunk->arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + assert(((uintptr_t)ptr - ((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == + 0); + ret = bin_info->reg_size; + } else { + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + ret = mapbits & ~PAGE_MASK; + assert(ret != 0); + } + + return (ret); +} + +#ifdef JEMALLOC_PROF +void +arena_prof_promoted(const void *ptr, size_t size) +{ + arena_chunk_t *chunk; + size_t pageind, binind; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + assert(isalloc(ptr) == PAGE_SIZE); + assert(size <= small_maxclass); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + binind = SMALL_SIZE2BIN(size); + assert(binind < nbins); + chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & + ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); +} + +size_t +arena_salloc_demote(const void *ptr) +{ + size_t ret; + arena_chunk_t *chunk; + size_t pageind, mapbits; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + mapbits = chunk->map[pageind-map_bias].bits; + assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); + if ((mapbits & CHUNK_MAP_LARGE) == 0) { + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << + PAGE_SHIFT)); + dassert(run->magic == ARENA_RUN_MAGIC); + size_t binind = arena_bin_index(chunk->arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + assert(((uintptr_t)ptr - ((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == + 0); + ret = bin_info->reg_size; + } else { + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + ret = mapbits & ~PAGE_MASK; + if (prof_promote && ret == PAGE_SIZE && (mapbits & + CHUNK_MAP_CLASS_MASK) != 0) { + size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> + CHUNK_MAP_CLASS_SHIFT) - 1; + assert(binind < nbins); + ret = arena_bin_info[binind].reg_size; + } + assert(ret != 0); + } + + return (ret); +} +#endif + +static void +arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, + arena_bin_t *bin) +{ + + /* Dissociate run from bin. */ + if (run == bin->runcur) + bin->runcur = NULL; + else { + size_t binind = arena_bin_index(chunk->arena, bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + + if (bin_info->nregs != 1) { + size_t run_pageind = (((uintptr_t)run - + (uintptr_t)chunk)) >> PAGE_SHIFT; + arena_chunk_map_t *run_mapelm = + &chunk->map[run_pageind-map_bias]; + /* + * This block's conditional is necessary because if the + * run only contains one region, then it never gets + * inserted into the non-full runs tree. + */ + arena_run_tree_remove(&bin->runs, run_mapelm); + } + } +} + +static void +arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, + arena_bin_t *bin) +{ + size_t binind; + arena_bin_info_t *bin_info; + size_t npages, run_ind, past; + + assert(run != bin->runcur); + assert(arena_run_tree_search(&bin->runs, &chunk->map[ + (((uintptr_t)run-(uintptr_t)chunk)>>PAGE_SHIFT)-map_bias]) == NULL); + + binind = arena_bin_index(chunk->arena, run->bin); + bin_info = &arena_bin_info[binind]; + + malloc_mutex_unlock(&bin->lock); + /******************************/ + npages = bin_info->run_size >> PAGE_SHIFT; + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); + past = (size_t)(PAGE_CEILING((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * + bin_info->reg_size) - (uintptr_t)chunk) >> PAGE_SHIFT); + malloc_mutex_lock(&arena->lock); + + /* + * If the run was originally clean, and some pages were never touched, + * trim the clean pages before deallocating the dirty portion of the + * run. + */ + if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past + - run_ind < npages) { + /* + * Trim clean pages. Convert to large run beforehand. Set the + * last map element first, in case this is a one-page run. + */ + chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | + (chunk->map[run_ind+npages-1-map_bias].bits & + CHUNK_MAP_FLAGS_MASK); + chunk->map[run_ind-map_bias].bits = bin_info->run_size | + CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & + CHUNK_MAP_FLAGS_MASK); + arena_run_trim_tail(arena, chunk, run, (npages << PAGE_SHIFT), + ((past - run_ind) << PAGE_SHIFT), false); + /* npages = past - run_ind; */ + } +#ifdef JEMALLOC_DEBUG + run->magic = 0; +#endif + arena_run_dalloc(arena, run, true); + malloc_mutex_unlock(&arena->lock); + /****************************/ + malloc_mutex_lock(&bin->lock); +#ifdef JEMALLOC_STATS + bin->stats.curruns--; +#endif +} + +static void +arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, + arena_bin_t *bin) +{ + + /* + * Make sure that bin->runcur always refers to the lowest non-full run, + * if one exists. + */ + if (bin->runcur == NULL) + bin->runcur = run; + else if ((uintptr_t)run < (uintptr_t)bin->runcur) { + /* Switch runcur. */ + if (bin->runcur->nfree > 0) { + arena_chunk_t *runcur_chunk = + CHUNK_ADDR2BASE(bin->runcur); + size_t runcur_pageind = (((uintptr_t)bin->runcur - + (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; + arena_chunk_map_t *runcur_mapelm = + &runcur_chunk->map[runcur_pageind-map_bias]; + + /* Insert runcur. */ + arena_run_tree_insert(&bin->runs, runcur_mapelm); + } + bin->runcur = run; + } else { + size_t run_pageind = (((uintptr_t)run - + (uintptr_t)chunk)) >> PAGE_SHIFT; + arena_chunk_map_t *run_mapelm = + &chunk->map[run_pageind-map_bias]; + + assert(arena_run_tree_search(&bin->runs, run_mapelm) == NULL); + arena_run_tree_insert(&bin->runs, run_mapelm); + } +} + +void +arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, + arena_chunk_map_t *mapelm) +{ + size_t pageind; + arena_run_t *run; + arena_bin_t *bin; +#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) + size_t size; +#endif + + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); + dassert(run->magic == ARENA_RUN_MAGIC); + bin = run->bin; + size_t binind = arena_bin_index(arena, bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; +#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) + size = bin_info->reg_size; +#endif + +#ifdef JEMALLOC_FILL + if (opt_junk) + memset(ptr, 0x5a, size); +#endif + + arena_run_reg_dalloc(run, ptr); + if (run->nfree == bin_info->nregs) { + arena_dissociate_bin_run(chunk, run, bin); + arena_dalloc_bin_run(arena, chunk, run, bin); + } else if (run->nfree == 1 && run != bin->runcur) + arena_bin_lower_run(arena, chunk, run, bin); + +#ifdef JEMALLOC_STATS + bin->stats.allocated -= size; + bin->stats.ndalloc++; +#endif +} + +#ifdef JEMALLOC_STATS +void +arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, + arena_stats_t *astats, malloc_bin_stats_t *bstats, + malloc_large_stats_t *lstats) +{ + unsigned i; + + malloc_mutex_lock(&arena->lock); + *nactive += arena->nactive; + *ndirty += arena->ndirty; + + astats->mapped += arena->stats.mapped; + astats->npurge += arena->stats.npurge; + astats->nmadvise += arena->stats.nmadvise; + astats->purged += arena->stats.purged; + astats->allocated_large += arena->stats.allocated_large; + astats->nmalloc_large += arena->stats.nmalloc_large; + astats->ndalloc_large += arena->stats.ndalloc_large; + astats->nrequests_large += arena->stats.nrequests_large; + + for (i = 0; i < nlclasses; i++) { + lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; + lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; + lstats[i].nrequests += arena->stats.lstats[i].nrequests; + lstats[i].highruns += arena->stats.lstats[i].highruns; + lstats[i].curruns += arena->stats.lstats[i].curruns; + } + malloc_mutex_unlock(&arena->lock); + + for (i = 0; i < nbins; i++) { + arena_bin_t *bin = &arena->bins[i]; + + malloc_mutex_lock(&bin->lock); + bstats[i].allocated += bin->stats.allocated; + bstats[i].nmalloc += bin->stats.nmalloc; + bstats[i].ndalloc += bin->stats.ndalloc; + bstats[i].nrequests += bin->stats.nrequests; +#ifdef JEMALLOC_TCACHE + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; +#endif + bstats[i].nruns += bin->stats.nruns; + bstats[i].reruns += bin->stats.reruns; + bstats[i].highruns += bin->stats.highruns; + bstats[i].curruns += bin->stats.curruns; + malloc_mutex_unlock(&bin->lock); + } +} +#endif + +void +arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) +{ + + /* Large allocation. */ +#ifdef JEMALLOC_FILL +# ifndef JEMALLOC_STATS + if (opt_junk) +# endif +#endif + { +#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> + PAGE_SHIFT; + size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; +#endif + +#ifdef JEMALLOC_FILL +# ifdef JEMALLOC_STATS + if (opt_junk) +# endif + memset(ptr, 0x5a, size); +#endif +#ifdef JEMALLOC_STATS + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= size; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; +#endif + } + + arena_run_dalloc(arena, (arena_run_t *)ptr, true); +} + +static void +arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t oldsize, size_t size) +{ + + assert(size < oldsize); + + /* + * Shrink the run, and make trailing pages available for other + * allocations. + */ + malloc_mutex_lock(&arena->lock); + arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, + true); +#ifdef JEMALLOC_STATS + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; + arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; + if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + } +#endif + malloc_mutex_unlock(&arena->lock); +} + +static bool +arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t oldsize, size_t size, size_t extra, bool zero) +{ + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + size_t npages = oldsize >> PAGE_SHIFT; + size_t followsize; + + assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); + + /* Try to extend the run. */ + assert(size + extra > oldsize); + malloc_mutex_lock(&arena->lock); + if (pageind + npages < chunk_npages && + (chunk->map[pageind+npages-map_bias].bits + & CHUNK_MAP_ALLOCATED) == 0 && (followsize = + chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - + oldsize) { + /* + * The next run is available and sufficiently large. Split the + * following run, then merge the first part with the existing + * allocation. + */ + size_t flag_dirty; + size_t splitsize = (oldsize + followsize <= size + extra) + ? followsize : size + extra - oldsize; + arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + + ((pageind+npages) << PAGE_SHIFT)), splitsize, true, zero); + + size = oldsize + splitsize; + npages = size >> PAGE_SHIFT; + + /* + * Mark the extended run as dirty if either portion of the run + * was dirty before allocation. This is rather pedantic, + * because there's not actually any sequence of events that + * could cause the resulting run to be passed to + * arena_run_dalloc() with the dirty argument set to false + * (which is when dirty flag consistency would really matter). + */ + flag_dirty = (chunk->map[pageind-map_bias].bits & + CHUNK_MAP_DIRTY) | + (chunk->map[pageind+npages-1-map_bias].bits & + CHUNK_MAP_DIRTY); + chunk->map[pageind-map_bias].bits = size | flag_dirty + | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | + CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + +#ifdef JEMALLOC_STATS + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; + arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; + if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { + arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = + arena->stats.lstats[(size >> PAGE_SHIFT) - + 1].curruns; + } +#endif + malloc_mutex_unlock(&arena->lock); + return (false); + } + malloc_mutex_unlock(&arena->lock); + + return (true); +} + +/* + * Try to resize a large allocation, in order to avoid copying. This will + * always fail if growing an object, and the following run is already in use. + */ +static bool +arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, + bool zero) +{ + size_t psize; + + psize = PAGE_CEILING(size + extra); + if (psize == oldsize) { + /* Same size class. */ +#ifdef JEMALLOC_FILL + if (opt_junk && size < oldsize) { + memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - + size); + } +#endif + return (false); + } else { + arena_chunk_t *chunk; + arena_t *arena; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = chunk->arena; + dassert(arena->magic == ARENA_MAGIC); + + if (psize < oldsize) { +#ifdef JEMALLOC_FILL + /* Fill before shrinking in order avoid a race. */ + if (opt_junk) { + memset((void *)((uintptr_t)ptr + size), 0x5a, + oldsize - size); + } +#endif + arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, + psize); + return (false); + } else { + bool ret = arena_ralloc_large_grow(arena, chunk, ptr, + oldsize, PAGE_CEILING(size), + psize - PAGE_CEILING(size), zero); +#ifdef JEMALLOC_FILL + if (ret == false && zero == false && opt_zero) { + memset((void *)((uintptr_t)ptr + oldsize), 0, + size - oldsize); + } +#endif + return (ret); + } + } +} + +void * +arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, + bool zero) +{ + + /* + * Avoid moving the allocation if the size class can be left the same. + */ + if (oldsize <= arena_maxclass) { + if (oldsize <= small_maxclass) { + assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size + == oldsize); + if ((size + extra <= small_maxclass && + SMALL_SIZE2BIN(size + extra) == + SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && + size + extra >= oldsize)) { +#ifdef JEMALLOC_FILL + if (opt_junk && size < oldsize) { + memset((void *)((uintptr_t)ptr + size), + 0x5a, oldsize - size); + } +#endif + return (ptr); + } + } else { + assert(size <= arena_maxclass); + if (size + extra > small_maxclass) { + if (arena_ralloc_large(ptr, oldsize, size, + extra, zero) == false) + return (ptr); + } + } + } + + /* Reallocation would require a move. */ + return (NULL); +} + +void * +arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero) +{ + void *ret; + size_t copysize; + + /* Try to avoid moving the allocation. */ + ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); + if (ret != NULL) + return (ret); + + /* + * size and oldsize are different enough that we need to move the + * object. In that case, fall back to allocating new space and + * copying. + */ + if (alignment != 0) { + size_t usize = sa2u(size + extra, alignment, NULL); + if (usize == 0) + return (NULL); + ret = ipalloc(usize, alignment, zero); + } else + ret = arena_malloc(size + extra, zero); + + if (ret == NULL) { + if (extra == 0) + return (NULL); + /* Try again, this time without extra. */ + if (alignment != 0) { + size_t usize = sa2u(size, alignment, NULL); + if (usize == 0) + return (NULL); + ret = ipalloc(usize, alignment, zero); + } else + ret = arena_malloc(size, zero); + + if (ret == NULL) + return (NULL); + } + + /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ + + /* + * Copy at most size bytes (not size+extra), since the caller has no + * expectation that the extra bytes will be reliably preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + memcpy(ret, ptr, copysize); + idalloc(ptr); + return (ret); +} + +bool +arena_new(arena_t *arena, unsigned ind) +{ + unsigned i; + arena_bin_t *bin; + + arena->ind = ind; + arena->nthreads = 0; + + if (malloc_mutex_init(&arena->lock)) + return (true); + +#ifdef JEMALLOC_STATS + memset(&arena->stats, 0, sizeof(arena_stats_t)); + arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses * + sizeof(malloc_large_stats_t)); + if (arena->stats.lstats == NULL) + return (true); + memset(arena->stats.lstats, 0, nlclasses * + sizeof(malloc_large_stats_t)); +# ifdef JEMALLOC_TCACHE + ql_new(&arena->tcache_ql); +# endif +#endif + +#ifdef JEMALLOC_PROF + arena->prof_accumbytes = 0; +#endif + + /* Initialize chunks. */ + ql_new(&arena->chunks_dirty); + arena->spare = NULL; + + arena->nactive = 0; + arena->ndirty = 0; + arena->npurgatory = 0; + + arena_avail_tree_new(&arena->runs_avail_clean); + arena_avail_tree_new(&arena->runs_avail_dirty); + + /* Initialize bins. */ + i = 0; +#ifdef JEMALLOC_TINY + /* (2^n)-spaced tiny bins. */ + for (; i < ntbins; i++) { + bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); + bin->runcur = NULL; + arena_run_tree_new(&bin->runs); +#ifdef JEMALLOC_STATS + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); +#endif + } +#endif + + /* Quantum-spaced bins. */ + for (; i < ntbins + nqbins; i++) { + bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); + bin->runcur = NULL; + arena_run_tree_new(&bin->runs); +#ifdef JEMALLOC_STATS + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); +#endif + } + + /* Cacheline-spaced bins. */ + for (; i < ntbins + nqbins + ncbins; i++) { + bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); + bin->runcur = NULL; + arena_run_tree_new(&bin->runs); +#ifdef JEMALLOC_STATS + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); +#endif + } + + /* Subpage-spaced bins. */ + for (; i < nbins; i++) { + bin = &arena->bins[i]; + if (malloc_mutex_init(&bin->lock)) + return (true); + bin->runcur = NULL; + arena_run_tree_new(&bin->runs); +#ifdef JEMALLOC_STATS + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); +#endif + } + +#ifdef JEMALLOC_DEBUG + arena->magic = ARENA_MAGIC; +#endif + + return (false); +} + +#ifdef JEMALLOC_DEBUG +static void +small_size2bin_validate(void) +{ + size_t i, size, binind; + + i = 1; +# ifdef JEMALLOC_TINY + /* Tiny. */ + for (; i < (1U << LG_TINY_MIN); i++) { + size = pow2_ceil(1U << LG_TINY_MIN); + binind = ffs((int)(size >> (LG_TINY_MIN + 1))); + assert(SMALL_SIZE2BIN(i) == binind); + } + for (; i < qspace_min; i++) { + size = pow2_ceil(i); + binind = ffs((int)(size >> (LG_TINY_MIN + 1))); + assert(SMALL_SIZE2BIN(i) == binind); + } +# endif + /* Quantum-spaced. */ + for (; i <= qspace_max; i++) { + size = QUANTUM_CEILING(i); + binind = ntbins + (size >> LG_QUANTUM) - 1; + assert(SMALL_SIZE2BIN(i) == binind); + } + /* Cacheline-spaced. */ + for (; i <= cspace_max; i++) { + size = CACHELINE_CEILING(i); + binind = ntbins + nqbins + ((size - cspace_min) >> + LG_CACHELINE); + assert(SMALL_SIZE2BIN(i) == binind); + } + /* Sub-page. */ + for (; i <= sspace_max; i++) { + size = SUBPAGE_CEILING(i); + binind = ntbins + nqbins + ncbins + ((size - sspace_min) + >> LG_SUBPAGE); + assert(SMALL_SIZE2BIN(i) == binind); + } +} +#endif + +static bool +small_size2bin_init(void) +{ + + if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT + || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT + || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> + LG_TINY_MIN) + 1)) + return (small_size2bin_init_hard()); + + small_size2bin = const_small_size2bin; +#ifdef JEMALLOC_DEBUG + small_size2bin_validate(); +#endif + return (false); +} + +static bool +small_size2bin_init_hard(void) +{ + size_t i, size, binind; + uint8_t *custom_small_size2bin; +#define CUSTOM_SMALL_SIZE2BIN(s) \ + custom_small_size2bin[(s-1) >> LG_TINY_MIN] + + assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT + || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT + || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> + LG_TINY_MIN) + 1)); + + custom_small_size2bin = (uint8_t *) + base_alloc(small_maxclass >> LG_TINY_MIN); + if (custom_small_size2bin == NULL) + return (true); + + i = 1; +#ifdef JEMALLOC_TINY + /* Tiny. */ + for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) { + size = pow2_ceil(1U << LG_TINY_MIN); + binind = ffs((int)(size >> (LG_TINY_MIN + 1))); + CUSTOM_SMALL_SIZE2BIN(i) = binind; + } + for (; i < qspace_min; i += TINY_MIN) { + size = pow2_ceil(i); + binind = ffs((int)(size >> (LG_TINY_MIN + 1))); + CUSTOM_SMALL_SIZE2BIN(i) = binind; + } +#endif + /* Quantum-spaced. */ + for (; i <= qspace_max; i += TINY_MIN) { + size = QUANTUM_CEILING(i); + binind = ntbins + (size >> LG_QUANTUM) - 1; + CUSTOM_SMALL_SIZE2BIN(i) = binind; + } + /* Cacheline-spaced. */ + for (; i <= cspace_max; i += TINY_MIN) { + size = CACHELINE_CEILING(i); + binind = ntbins + nqbins + ((size - cspace_min) >> + LG_CACHELINE); + CUSTOM_SMALL_SIZE2BIN(i) = binind; + } + /* Sub-page. */ + for (; i <= sspace_max; i += TINY_MIN) { + size = SUBPAGE_CEILING(i); + binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> + LG_SUBPAGE); + CUSTOM_SMALL_SIZE2BIN(i) = binind; + } + + small_size2bin = custom_small_size2bin; +#ifdef JEMALLOC_DEBUG + small_size2bin_validate(); +#endif + return (false); +#undef CUSTOM_SMALL_SIZE2BIN +} + +/* + * Calculate bin_info->run_size such that it meets the following constraints: + * + * *) bin_info->run_size >= min_run_size + * *) bin_info->run_size <= arena_maxclass + * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). + * *) bin_info->nregs <= RUN_MAXREGS + * + * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also + * calculated here, since these settings are all interdependent. + */ +static size_t +bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) +{ + size_t try_run_size, good_run_size; + uint32_t try_nregs, good_nregs; + uint32_t try_hdr_size, good_hdr_size; + uint32_t try_bitmap_offset, good_bitmap_offset; +#ifdef JEMALLOC_PROF + uint32_t try_ctx0_offset, good_ctx0_offset; +#endif + uint32_t try_reg0_offset, good_reg0_offset; + + assert(min_run_size >= PAGE_SIZE); + assert(min_run_size <= arena_maxclass); + + /* + * Calculate known-valid settings before entering the run_size + * expansion loop, so that the first part of the loop always copies + * valid settings. + * + * The do..while loop iteratively reduces the number of regions until + * the run header and the regions no longer overlap. A closed formula + * would be quite messy, since there is an interdependency between the + * header's mask length and the number of regions. + */ + try_run_size = min_run_size; + try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) + + 1; /* Counter-act try_nregs-- in loop. */ + if (try_nregs > RUN_MAXREGS) { + try_nregs = RUN_MAXREGS + + 1; /* Counter-act try_nregs-- in loop. */ + } + do { + try_nregs--; + try_hdr_size = sizeof(arena_run_t); + /* Pad to a long boundary. */ + try_hdr_size = LONG_CEILING(try_hdr_size); + try_bitmap_offset = try_hdr_size; + /* Add space for bitmap. */ + try_hdr_size += bitmap_size(try_nregs); +#ifdef JEMALLOC_PROF + if (opt_prof && prof_promote == false) { + /* Pad to a quantum boundary. */ + try_hdr_size = QUANTUM_CEILING(try_hdr_size); + try_ctx0_offset = try_hdr_size; + /* Add space for one (prof_ctx_t *) per region. */ + try_hdr_size += try_nregs * sizeof(prof_ctx_t *); + } else + try_ctx0_offset = 0; +#endif + try_reg0_offset = try_run_size - (try_nregs * + bin_info->reg_size); + } while (try_hdr_size > try_reg0_offset); + + /* run_size expansion loop. */ + do { + /* + * Copy valid settings before trying more aggressive settings. + */ + good_run_size = try_run_size; + good_nregs = try_nregs; + good_hdr_size = try_hdr_size; + good_bitmap_offset = try_bitmap_offset; +#ifdef JEMALLOC_PROF + good_ctx0_offset = try_ctx0_offset; +#endif + good_reg0_offset = try_reg0_offset; + + /* Try more aggressive settings. */ + try_run_size += PAGE_SIZE; + try_nregs = ((try_run_size - sizeof(arena_run_t)) / + bin_info->reg_size) + + 1; /* Counter-act try_nregs-- in loop. */ + if (try_nregs > RUN_MAXREGS) { + try_nregs = RUN_MAXREGS + + 1; /* Counter-act try_nregs-- in loop. */ + } + do { + try_nregs--; + try_hdr_size = sizeof(arena_run_t); + /* Pad to a long boundary. */ + try_hdr_size = LONG_CEILING(try_hdr_size); + try_bitmap_offset = try_hdr_size; + /* Add space for bitmap. */ + try_hdr_size += bitmap_size(try_nregs); +#ifdef JEMALLOC_PROF + if (opt_prof && prof_promote == false) { + /* Pad to a quantum boundary. */ + try_hdr_size = QUANTUM_CEILING(try_hdr_size); + try_ctx0_offset = try_hdr_size; + /* + * Add space for one (prof_ctx_t *) per region. + */ + try_hdr_size += try_nregs * + sizeof(prof_ctx_t *); + } +#endif + try_reg0_offset = try_run_size - (try_nregs * + bin_info->reg_size); + } while (try_hdr_size > try_reg0_offset); + } while (try_run_size <= arena_maxclass + && try_run_size <= arena_maxclass + && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX + && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size + && try_nregs < RUN_MAXREGS); + + assert(good_hdr_size <= good_reg0_offset); + + /* Copy final settings. */ + bin_info->run_size = good_run_size; + bin_info->nregs = good_nregs; + bin_info->bitmap_offset = good_bitmap_offset; +#ifdef JEMALLOC_PROF + bin_info->ctx0_offset = good_ctx0_offset; +#endif + bin_info->reg0_offset = good_reg0_offset; + + return (good_run_size); +} + +static bool +bin_info_init(void) +{ + arena_bin_info_t *bin_info; + unsigned i; + size_t prev_run_size; + + arena_bin_info = base_alloc(sizeof(arena_bin_info_t) * nbins); + if (arena_bin_info == NULL) + return (true); + + prev_run_size = PAGE_SIZE; + i = 0; +#ifdef JEMALLOC_TINY + /* (2^n)-spaced tiny bins. */ + for (; i < ntbins; i++) { + bin_info = &arena_bin_info[i]; + bin_info->reg_size = (1U << (LG_TINY_MIN + i)); + prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); + } +#endif + + /* Quantum-spaced bins. */ + for (; i < ntbins + nqbins; i++) { + bin_info = &arena_bin_info[i]; + bin_info->reg_size = (i - ntbins + 1) << LG_QUANTUM; + prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); + } + + /* Cacheline-spaced bins. */ + for (; i < ntbins + nqbins + ncbins; i++) { + bin_info = &arena_bin_info[i]; + bin_info->reg_size = cspace_min + ((i - (ntbins + nqbins)) << + LG_CACHELINE); + prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); + } + + /* Subpage-spaced bins. */ + for (; i < nbins; i++) { + bin_info = &arena_bin_info[i]; + bin_info->reg_size = sspace_min + ((i - (ntbins + nqbins + + ncbins)) << LG_SUBPAGE); + prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); + } + + return (false); +} + +bool +arena_boot(void) +{ + size_t header_size; + unsigned i; + + /* Set variables according to the value of opt_lg_[qc]space_max. */ + qspace_max = (1U << opt_lg_qspace_max); + cspace_min = CACHELINE_CEILING(qspace_max); + if (cspace_min == qspace_max) + cspace_min += CACHELINE; + cspace_max = (1U << opt_lg_cspace_max); + sspace_min = SUBPAGE_CEILING(cspace_max); + if (sspace_min == cspace_max) + sspace_min += SUBPAGE; + assert(sspace_min < PAGE_SIZE); + sspace_max = PAGE_SIZE - SUBPAGE; + +#ifdef JEMALLOC_TINY + assert(LG_QUANTUM >= LG_TINY_MIN); +#endif + assert(ntbins <= LG_QUANTUM); + nqbins = qspace_max >> LG_QUANTUM; + ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; + nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; + nbins = ntbins + nqbins + ncbins + nsbins; + + /* + * The small_size2bin lookup table uses uint8_t to encode each bin + * index, so we cannot support more than 256 small size classes. This + * limit is difficult to exceed (not even possible with 16B quantum and + * 4KiB pages), and such configurations are impractical, but + * nonetheless we need to protect against this case in order to avoid + * undefined behavior. + * + * Further constrain nbins to 255 if prof_promote is true, since all + * small size classes, plus a "not small" size class must be stored in + * 8 bits of arena_chunk_map_t's bits field. + */ +#ifdef JEMALLOC_PROF + if (opt_prof && prof_promote) { + if (nbins > 255) { + char line_buf[UMAX2S_BUFSIZE]; + malloc_write(": Too many small size classes ("); + malloc_write(u2s(nbins, 10, line_buf)); + malloc_write(" > max 255)\n"); + abort(); + } + } else +#endif + if (nbins > 256) { + char line_buf[UMAX2S_BUFSIZE]; + malloc_write(": Too many small size classes ("); + malloc_write(u2s(nbins, 10, line_buf)); + malloc_write(" > max 256)\n"); + abort(); + } + + /* + * Compute the header size such that it is large enough to contain the + * page map. The page map is biased to omit entries for the header + * itself, so some iteration is necessary to compute the map bias. + * + * 1) Compute safe header_size and map_bias values that include enough + * space for an unbiased page map. + * 2) Refine map_bias based on (1) to omit the header pages in the page + * map. The resulting map_bias may be one too small. + * 3) Refine map_bias based on (2). The result will be >= the result + * from (2), and will always be correct. + */ + map_bias = 0; + for (i = 0; i < 3; i++) { + header_size = offsetof(arena_chunk_t, map) + + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); + map_bias = (header_size >> PAGE_SHIFT) + ((header_size & + PAGE_MASK) != 0); + } + assert(map_bias > 0); + + arena_maxclass = chunksize - (map_bias << PAGE_SHIFT); + + if (small_size2bin_init()) + return (true); + + if (bin_info_init()) + return (true); + + return (false); +} diff --git a/deps/jemalloc.orig/src/atomic.c b/deps/jemalloc.orig/src/atomic.c new file mode 100644 index 00000000000..77ee313113b --- /dev/null +++ b/deps/jemalloc.orig/src/atomic.c @@ -0,0 +1,2 @@ +#define JEMALLOC_ATOMIC_C_ +#include "jemalloc/internal/jemalloc_internal.h" diff --git a/deps/jemalloc.orig/src/base.c b/deps/jemalloc.orig/src/base.c new file mode 100644 index 00000000000..cc85e8494ec --- /dev/null +++ b/deps/jemalloc.orig/src/base.c @@ -0,0 +1,106 @@ +#define JEMALLOC_BASE_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +malloc_mutex_t base_mtx; + +/* + * Current pages that are being used for internal memory allocations. These + * pages are carved up in cacheline-size quanta, so that there is no chance of + * false cache line sharing. + */ +static void *base_pages; +static void *base_next_addr; +static void *base_past_addr; /* Addr immediately past base_pages. */ +static extent_node_t *base_nodes; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static bool base_pages_alloc(size_t minsize); + +/******************************************************************************/ + +static bool +base_pages_alloc(size_t minsize) +{ + size_t csize; + bool zero; + + assert(minsize != 0); + csize = CHUNK_CEILING(minsize); + zero = false; + base_pages = chunk_alloc(csize, true, &zero); + if (base_pages == NULL) + return (true); + base_next_addr = base_pages; + base_past_addr = (void *)((uintptr_t)base_pages + csize); + + return (false); +} + +void * +base_alloc(size_t size) +{ + void *ret; + size_t csize; + + /* Round size up to nearest multiple of the cacheline size. */ + csize = CACHELINE_CEILING(size); + + malloc_mutex_lock(&base_mtx); + /* Make sure there's enough space for the allocation. */ + if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) { + if (base_pages_alloc(csize)) { + malloc_mutex_unlock(&base_mtx); + return (NULL); + } + } + /* Allocate. */ + ret = base_next_addr; + base_next_addr = (void *)((uintptr_t)base_next_addr + csize); + malloc_mutex_unlock(&base_mtx); + + return (ret); +} + +extent_node_t * +base_node_alloc(void) +{ + extent_node_t *ret; + + malloc_mutex_lock(&base_mtx); + if (base_nodes != NULL) { + ret = base_nodes; + base_nodes = *(extent_node_t **)ret; + malloc_mutex_unlock(&base_mtx); + } else { + malloc_mutex_unlock(&base_mtx); + ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); + } + + return (ret); +} + +void +base_node_dealloc(extent_node_t *node) +{ + + malloc_mutex_lock(&base_mtx); + *(extent_node_t **)node = base_nodes; + base_nodes = node; + malloc_mutex_unlock(&base_mtx); +} + +bool +base_boot(void) +{ + + base_nodes = NULL; + if (malloc_mutex_init(&base_mtx)) + return (true); + + return (false); +} diff --git a/deps/jemalloc.orig/src/bitmap.c b/deps/jemalloc.orig/src/bitmap.c new file mode 100644 index 00000000000..b47e2629093 --- /dev/null +++ b/deps/jemalloc.orig/src/bitmap.c @@ -0,0 +1,90 @@ +#define JEMALLOC_BITMAP_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static size_t bits2groups(size_t nbits); + +/******************************************************************************/ + +static size_t +bits2groups(size_t nbits) +{ + + return ((nbits >> LG_BITMAP_GROUP_NBITS) + + !!(nbits & BITMAP_GROUP_NBITS_MASK)); +} + +void +bitmap_info_init(bitmap_info_t *binfo, size_t nbits) +{ + unsigned i; + size_t group_count; + + assert(nbits > 0); + assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS)); + + /* + * Compute the number of groups necessary to store nbits bits, and + * progressively work upward through the levels until reaching a level + * that requires only one group. + */ + binfo->levels[0].group_offset = 0; + group_count = bits2groups(nbits); + for (i = 1; group_count > 1; i++) { + assert(i < BITMAP_MAX_LEVELS); + binfo->levels[i].group_offset = binfo->levels[i-1].group_offset + + group_count; + group_count = bits2groups(group_count); + } + binfo->levels[i].group_offset = binfo->levels[i-1].group_offset + + group_count; + binfo->nlevels = i; + binfo->nbits = nbits; +} + +size_t +bitmap_info_ngroups(const bitmap_info_t *binfo) +{ + + return (binfo->levels[binfo->nlevels].group_offset << LG_SIZEOF_BITMAP); +} + +size_t +bitmap_size(size_t nbits) +{ + bitmap_info_t binfo; + + bitmap_info_init(&binfo, nbits); + return (bitmap_info_ngroups(&binfo)); +} + +void +bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo) +{ + size_t extra; + unsigned i; + + /* + * Bits are actually inverted with regard to the external bitmap + * interface, so the bitmap starts out with all 1 bits, except for + * trailing unused bits (if any). Note that each group uses bit 0 to + * correspond to the first logical bit in the group, so extra bits + * are the most significant bits of the last group. + */ + memset(bitmap, 0xffU, binfo->levels[binfo->nlevels].group_offset << + LG_SIZEOF_BITMAP); + extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK)) + & BITMAP_GROUP_NBITS_MASK; + if (extra != 0) + bitmap[binfo->levels[1].group_offset - 1] >>= extra; + for (i = 1; i < binfo->nlevels; i++) { + size_t group_count = binfo->levels[i].group_offset - + binfo->levels[i-1].group_offset; + extra = (BITMAP_GROUP_NBITS - (group_count & + BITMAP_GROUP_NBITS_MASK)) & BITMAP_GROUP_NBITS_MASK; + if (extra != 0) + bitmap[binfo->levels[i+1].group_offset - 1] >>= extra; + } +} diff --git a/deps/jemalloc.orig/src/chunk.c b/deps/jemalloc.orig/src/chunk.c new file mode 100644 index 00000000000..d190c6f49b3 --- /dev/null +++ b/deps/jemalloc.orig/src/chunk.c @@ -0,0 +1,173 @@ +#define JEMALLOC_CHUNK_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +size_t opt_lg_chunk = LG_CHUNK_DEFAULT; +#ifdef JEMALLOC_SWAP +bool opt_overcommit = true; +#endif + +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) +malloc_mutex_t chunks_mtx; +chunk_stats_t stats_chunks; +#endif + +#ifdef JEMALLOC_IVSALLOC +rtree_t *chunks_rtree; +#endif + +/* Various chunk-related settings. */ +size_t chunksize; +size_t chunksize_mask; /* (chunksize - 1). */ +size_t chunk_npages; +size_t map_bias; +size_t arena_maxclass; /* Max size class for arenas. */ + +/******************************************************************************/ + +/* + * If the caller specifies (*zero == false), it is still possible to receive + * zeroed memory, in which case *zero is toggled to true. arena_chunk_alloc() + * takes advantage of this to avoid demanding zeroed chunks, but taking + * advantage of them if they are returned. + */ +void * +chunk_alloc(size_t size, bool base, bool *zero) +{ + void *ret; + + assert(size != 0); + assert((size & chunksize_mask) == 0); + +#ifdef JEMALLOC_SWAP + if (swap_enabled) { + ret = chunk_alloc_swap(size, zero); + if (ret != NULL) + goto RETURN; + } + + if (swap_enabled == false || opt_overcommit) { +#endif +#ifdef JEMALLOC_DSS + ret = chunk_alloc_dss(size, zero); + if (ret != NULL) + goto RETURN; +#endif + ret = chunk_alloc_mmap(size); + if (ret != NULL) { + *zero = true; + goto RETURN; + } +#ifdef JEMALLOC_SWAP + } +#endif + + /* All strategies for allocation failed. */ + ret = NULL; +RETURN: +#ifdef JEMALLOC_IVSALLOC + if (base == false && ret != NULL) { + if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) { + chunk_dealloc(ret, size, true); + return (NULL); + } + } +#endif +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + if (ret != NULL) { +# ifdef JEMALLOC_PROF + bool gdump; +# endif + malloc_mutex_lock(&chunks_mtx); +# ifdef JEMALLOC_STATS + stats_chunks.nchunks += (size / chunksize); +# endif + stats_chunks.curchunks += (size / chunksize); + if (stats_chunks.curchunks > stats_chunks.highchunks) { + stats_chunks.highchunks = stats_chunks.curchunks; +# ifdef JEMALLOC_PROF + gdump = true; +# endif + } +# ifdef JEMALLOC_PROF + else + gdump = false; +# endif + malloc_mutex_unlock(&chunks_mtx); +# ifdef JEMALLOC_PROF + if (opt_prof && opt_prof_gdump && gdump) + prof_gdump(); +# endif + } +#endif + + assert(CHUNK_ADDR2BASE(ret) == ret); + return (ret); +} + +void +chunk_dealloc(void *chunk, size_t size, bool unmap) +{ + + assert(chunk != NULL); + assert(CHUNK_ADDR2BASE(chunk) == chunk); + assert(size != 0); + assert((size & chunksize_mask) == 0); + +#ifdef JEMALLOC_IVSALLOC + rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); +#endif +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + malloc_mutex_lock(&chunks_mtx); + stats_chunks.curchunks -= (size / chunksize); + malloc_mutex_unlock(&chunks_mtx); +#endif + + if (unmap) { +#ifdef JEMALLOC_SWAP + if (swap_enabled && chunk_dealloc_swap(chunk, size) == false) + return; +#endif +#ifdef JEMALLOC_DSS + if (chunk_dealloc_dss(chunk, size) == false) + return; +#endif + chunk_dealloc_mmap(chunk, size); + } +} + +bool +chunk_boot(void) +{ + + /* Set variables according to the value of opt_lg_chunk. */ + chunksize = (ZU(1) << opt_lg_chunk); + assert(chunksize >= PAGE_SIZE); + chunksize_mask = chunksize - 1; + chunk_npages = (chunksize >> PAGE_SHIFT); + +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + if (malloc_mutex_init(&chunks_mtx)) + return (true); + memset(&stats_chunks, 0, sizeof(chunk_stats_t)); +#endif +#ifdef JEMALLOC_SWAP + if (chunk_swap_boot()) + return (true); +#endif + if (chunk_mmap_boot()) + return (true); +#ifdef JEMALLOC_DSS + if (chunk_dss_boot()) + return (true); +#endif +#ifdef JEMALLOC_IVSALLOC + chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk); + if (chunks_rtree == NULL) + return (true); +#endif + + return (false); +} diff --git a/deps/jemalloc.orig/src/chunk_dss.c b/deps/jemalloc.orig/src/chunk_dss.c new file mode 100644 index 00000000000..5c0e290e441 --- /dev/null +++ b/deps/jemalloc.orig/src/chunk_dss.c @@ -0,0 +1,284 @@ +#define JEMALLOC_CHUNK_DSS_C_ +#include "jemalloc/internal/jemalloc_internal.h" +#ifdef JEMALLOC_DSS +/******************************************************************************/ +/* Data. */ + +malloc_mutex_t dss_mtx; + +/* Base address of the DSS. */ +static void *dss_base; +/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ +static void *dss_prev; +/* Current upper limit on DSS addresses. */ +static void *dss_max; + +/* + * Trees of chunks that were previously allocated (trees differ only in node + * ordering). These are used when allocating chunks, in an attempt to re-use + * address space. Depending on function, different tree orderings are needed, + * which is why there are two trees with the same contents. + */ +static extent_tree_t dss_chunks_szad; +static extent_tree_t dss_chunks_ad; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void *chunk_recycle_dss(size_t size, bool *zero); +static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size); + +/******************************************************************************/ + +static void * +chunk_recycle_dss(size_t size, bool *zero) +{ + extent_node_t *node, key; + + key.addr = NULL; + key.size = size; + malloc_mutex_lock(&dss_mtx); + node = extent_tree_szad_nsearch(&dss_chunks_szad, &key); + if (node != NULL) { + void *ret = node->addr; + + /* Remove node from the tree. */ + extent_tree_szad_remove(&dss_chunks_szad, node); + if (node->size == size) { + extent_tree_ad_remove(&dss_chunks_ad, node); + base_node_dealloc(node); + } else { + /* + * Insert the remainder of node's address range as a + * smaller chunk. Its position within dss_chunks_ad + * does not change. + */ + assert(node->size > size); + node->addr = (void *)((uintptr_t)node->addr + size); + node->size -= size; + extent_tree_szad_insert(&dss_chunks_szad, node); + } + malloc_mutex_unlock(&dss_mtx); + + if (*zero) + memset(ret, 0, size); + return (ret); + } + malloc_mutex_unlock(&dss_mtx); + + return (NULL); +} + +void * +chunk_alloc_dss(size_t size, bool *zero) +{ + void *ret; + + ret = chunk_recycle_dss(size, zero); + if (ret != NULL) + return (ret); + + /* + * sbrk() uses a signed increment argument, so take care not to + * interpret a huge allocation request as a negative increment. + */ + if ((intptr_t)size < 0) + return (NULL); + + malloc_mutex_lock(&dss_mtx); + if (dss_prev != (void *)-1) { + intptr_t incr; + + /* + * The loop is necessary to recover from races with other + * threads that are using the DSS for something other than + * malloc. + */ + do { + /* Get the current end of the DSS. */ + dss_max = sbrk(0); + + /* + * Calculate how much padding is necessary to + * chunk-align the end of the DSS. + */ + incr = (intptr_t)size + - (intptr_t)CHUNK_ADDR2OFFSET(dss_max); + if (incr == (intptr_t)size) + ret = dss_max; + else { + ret = (void *)((intptr_t)dss_max + incr); + incr += size; + } + + dss_prev = sbrk(incr); + if (dss_prev == dss_max) { + /* Success. */ + dss_max = (void *)((intptr_t)dss_prev + incr); + malloc_mutex_unlock(&dss_mtx); + *zero = true; + return (ret); + } + } while (dss_prev != (void *)-1); + } + malloc_mutex_unlock(&dss_mtx); + + return (NULL); +} + +static extent_node_t * +chunk_dealloc_dss_record(void *chunk, size_t size) +{ + extent_node_t *xnode, *node, *prev, key; + + xnode = NULL; + while (true) { + key.addr = (void *)((uintptr_t)chunk + size); + node = extent_tree_ad_nsearch(&dss_chunks_ad, &key); + /* Try to coalesce forward. */ + if (node != NULL && node->addr == key.addr) { + /* + * Coalesce chunk with the following address range. + * This does not change the position within + * dss_chunks_ad, so only remove/insert from/into + * dss_chunks_szad. + */ + extent_tree_szad_remove(&dss_chunks_szad, node); + node->addr = chunk; + node->size += size; + extent_tree_szad_insert(&dss_chunks_szad, node); + break; + } else if (xnode == NULL) { + /* + * It is possible that base_node_alloc() will cause a + * new base chunk to be allocated, so take care not to + * deadlock on dss_mtx, and recover if another thread + * deallocates an adjacent chunk while this one is busy + * allocating xnode. + */ + malloc_mutex_unlock(&dss_mtx); + xnode = base_node_alloc(); + malloc_mutex_lock(&dss_mtx); + if (xnode == NULL) + return (NULL); + } else { + /* Coalescing forward failed, so insert a new node. */ + node = xnode; + xnode = NULL; + node->addr = chunk; + node->size = size; + extent_tree_ad_insert(&dss_chunks_ad, node); + extent_tree_szad_insert(&dss_chunks_szad, node); + break; + } + } + /* Discard xnode if it ended up unused do to a race. */ + if (xnode != NULL) + base_node_dealloc(xnode); + + /* Try to coalesce backward. */ + prev = extent_tree_ad_prev(&dss_chunks_ad, node); + if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == + chunk) { + /* + * Coalesce chunk with the previous address range. This does + * not change the position within dss_chunks_ad, so only + * remove/insert node from/into dss_chunks_szad. + */ + extent_tree_szad_remove(&dss_chunks_szad, prev); + extent_tree_ad_remove(&dss_chunks_ad, prev); + + extent_tree_szad_remove(&dss_chunks_szad, node); + node->addr = prev->addr; + node->size += prev->size; + extent_tree_szad_insert(&dss_chunks_szad, node); + + base_node_dealloc(prev); + } + + return (node); +} + +bool +chunk_in_dss(void *chunk) +{ + bool ret; + + malloc_mutex_lock(&dss_mtx); + if ((uintptr_t)chunk >= (uintptr_t)dss_base + && (uintptr_t)chunk < (uintptr_t)dss_max) + ret = true; + else + ret = false; + malloc_mutex_unlock(&dss_mtx); + + return (ret); +} + +bool +chunk_dealloc_dss(void *chunk, size_t size) +{ + bool ret; + + malloc_mutex_lock(&dss_mtx); + if ((uintptr_t)chunk >= (uintptr_t)dss_base + && (uintptr_t)chunk < (uintptr_t)dss_max) { + extent_node_t *node; + + /* Try to coalesce with other unused chunks. */ + node = chunk_dealloc_dss_record(chunk, size); + if (node != NULL) { + chunk = node->addr; + size = node->size; + } + + /* Get the current end of the DSS. */ + dss_max = sbrk(0); + + /* + * Try to shrink the DSS if this chunk is at the end of the + * DSS. The sbrk() call here is subject to a race condition + * with threads that use brk(2) or sbrk(2) directly, but the + * alternative would be to leak memory for the sake of poorly + * designed multi-threaded programs. + */ + if ((void *)((uintptr_t)chunk + size) == dss_max + && (dss_prev = sbrk(-(intptr_t)size)) == dss_max) { + /* Success. */ + dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size); + + if (node != NULL) { + extent_tree_szad_remove(&dss_chunks_szad, node); + extent_tree_ad_remove(&dss_chunks_ad, node); + base_node_dealloc(node); + } + } else + madvise(chunk, size, MADV_DONTNEED); + + ret = false; + goto RETURN; + } + + ret = true; +RETURN: + malloc_mutex_unlock(&dss_mtx); + return (ret); +} + +bool +chunk_dss_boot(void) +{ + + if (malloc_mutex_init(&dss_mtx)) + return (true); + dss_base = sbrk(0); + dss_prev = dss_base; + dss_max = dss_base; + extent_tree_szad_new(&dss_chunks_szad); + extent_tree_ad_new(&dss_chunks_ad); + + return (false); +} + +/******************************************************************************/ +#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc.orig/src/chunk_mmap.c b/deps/jemalloc.orig/src/chunk_mmap.c new file mode 100644 index 00000000000..164e86e7b38 --- /dev/null +++ b/deps/jemalloc.orig/src/chunk_mmap.c @@ -0,0 +1,239 @@ +#define JEMALLOC_CHUNK_MMAP_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +/* + * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and + * potentially avoid some system calls. + */ +#ifndef NO_TLS +static __thread bool mmap_unaligned_tls + JEMALLOC_ATTR(tls_model("initial-exec")); +#define MMAP_UNALIGNED_GET() mmap_unaligned_tls +#define MMAP_UNALIGNED_SET(v) do { \ + mmap_unaligned_tls = (v); \ +} while (0) +#else +static pthread_key_t mmap_unaligned_tsd; +#define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd)) +#define MMAP_UNALIGNED_SET(v) do { \ + pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \ +} while (0) +#endif + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void *pages_map(void *addr, size_t size, bool noreserve); +static void pages_unmap(void *addr, size_t size); +static void *chunk_alloc_mmap_slow(size_t size, bool unaligned, + bool noreserve); +static void *chunk_alloc_mmap_internal(size_t size, bool noreserve); + +/******************************************************************************/ + +static void * +pages_map(void *addr, size_t size, bool noreserve) +{ + void *ret; + + /* + * We don't use MAP_FIXED here, because it can cause the *replacement* + * of existing mappings, and we only want to create new mappings. + */ + int flags = MAP_PRIVATE | MAP_ANON; +#ifdef MAP_NORESERVE + if (noreserve) + flags |= MAP_NORESERVE; +#endif + ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); + assert(ret != NULL); + + if (ret == MAP_FAILED) + ret = NULL; + else if (addr != NULL && ret != addr) { + /* + * We succeeded in mapping memory, but not in the right place. + */ + if (munmap(ret, size) == -1) { + char buf[BUFERROR_BUF]; + + buferror(errno, buf, sizeof(buf)); + malloc_write(": Error in munmap(): "); + malloc_write(buf); + malloc_write("\n"); + if (opt_abort) + abort(); + } + ret = NULL; + } + + assert(ret == NULL || (addr == NULL && ret != addr) + || (addr != NULL && ret == addr)); + return (ret); +} + +static void +pages_unmap(void *addr, size_t size) +{ + + if (munmap(addr, size) == -1) { + char buf[BUFERROR_BUF]; + + buferror(errno, buf, sizeof(buf)); + malloc_write(": Error in munmap(): "); + malloc_write(buf); + malloc_write("\n"); + if (opt_abort) + abort(); + } +} + +static void * +chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve) +{ + void *ret; + size_t offset; + + /* Beware size_t wrap-around. */ + if (size + chunksize <= size) + return (NULL); + + ret = pages_map(NULL, size + chunksize, noreserve); + if (ret == NULL) + return (NULL); + + /* Clean up unneeded leading/trailing space. */ + offset = CHUNK_ADDR2OFFSET(ret); + if (offset != 0) { + /* Note that mmap() returned an unaligned mapping. */ + unaligned = true; + + /* Leading space. */ + pages_unmap(ret, chunksize - offset); + + ret = (void *)((uintptr_t)ret + + (chunksize - offset)); + + /* Trailing space. */ + pages_unmap((void *)((uintptr_t)ret + size), + offset); + } else { + /* Trailing space only. */ + pages_unmap((void *)((uintptr_t)ret + size), + chunksize); + } + + /* + * If mmap() returned an aligned mapping, reset mmap_unaligned so that + * the next chunk_alloc_mmap() execution tries the fast allocation + * method. + */ + if (unaligned == false) + MMAP_UNALIGNED_SET(false); + + return (ret); +} + +static void * +chunk_alloc_mmap_internal(size_t size, bool noreserve) +{ + void *ret; + + /* + * Ideally, there would be a way to specify alignment to mmap() (like + * NetBSD has), but in the absence of such a feature, we have to work + * hard to efficiently create aligned mappings. The reliable, but + * slow method is to create a mapping that is over-sized, then trim the + * excess. However, that always results in at least one call to + * pages_unmap(). + * + * A more optimistic approach is to try mapping precisely the right + * amount, then try to append another mapping if alignment is off. In + * practice, this works out well as long as the application is not + * interleaving mappings via direct mmap() calls. If we do run into a + * situation where there is an interleaved mapping and we are unable to + * extend an unaligned mapping, our best option is to switch to the + * slow method until mmap() returns another aligned mapping. This will + * tend to leave a gap in the memory map that is too small to cause + * later problems for the optimistic method. + * + * Another possible confounding factor is address space layout + * randomization (ASLR), which causes mmap(2) to disregard the + * requested address. mmap_unaligned tracks whether the previous + * chunk_alloc_mmap() execution received any unaligned or relocated + * mappings, and if so, the current execution will immediately fall + * back to the slow method. However, we keep track of whether the fast + * method would have succeeded, and if so, we make a note to try the + * fast method next time. + */ + + if (MMAP_UNALIGNED_GET() == false) { + size_t offset; + + ret = pages_map(NULL, size, noreserve); + if (ret == NULL) + return (NULL); + + offset = CHUNK_ADDR2OFFSET(ret); + if (offset != 0) { + MMAP_UNALIGNED_SET(true); + /* Try to extend chunk boundary. */ + if (pages_map((void *)((uintptr_t)ret + size), + chunksize - offset, noreserve) == NULL) { + /* + * Extension failed. Clean up, then revert to + * the reliable-but-expensive method. + */ + pages_unmap(ret, size); + ret = chunk_alloc_mmap_slow(size, true, + noreserve); + } else { + /* Clean up unneeded leading space. */ + pages_unmap(ret, chunksize - offset); + ret = (void *)((uintptr_t)ret + (chunksize - + offset)); + } + } + } else + ret = chunk_alloc_mmap_slow(size, false, noreserve); + + return (ret); +} + +void * +chunk_alloc_mmap(size_t size) +{ + + return (chunk_alloc_mmap_internal(size, false)); +} + +void * +chunk_alloc_mmap_noreserve(size_t size) +{ + + return (chunk_alloc_mmap_internal(size, true)); +} + +void +chunk_dealloc_mmap(void *chunk, size_t size) +{ + + pages_unmap(chunk, size); +} + +bool +chunk_mmap_boot(void) +{ + +#ifdef NO_TLS + if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) { + malloc_write(": Error in pthread_key_create()\n"); + return (true); + } +#endif + + return (false); +} diff --git a/deps/jemalloc/src/chunk_swap.c b/deps/jemalloc.orig/src/chunk_swap.c similarity index 100% rename from deps/jemalloc/src/chunk_swap.c rename to deps/jemalloc.orig/src/chunk_swap.c diff --git a/deps/jemalloc.orig/src/ckh.c b/deps/jemalloc.orig/src/ckh.c new file mode 100644 index 00000000000..43fcc25239d --- /dev/null +++ b/deps/jemalloc.orig/src/ckh.c @@ -0,0 +1,619 @@ +/* + ******************************************************************************* + * Implementation of (2^1+,2) cuckoo hashing, where 2^1+ indicates that each + * hash bucket contains 2^n cells, for n >= 1, and 2 indicates that two hash + * functions are employed. The original cuckoo hashing algorithm was described + * in: + * + * Pagh, R., F.F. Rodler (2004) Cuckoo Hashing. Journal of Algorithms + * 51(2):122-144. + * + * Generalization of cuckoo hashing was discussed in: + * + * Erlingsson, U., M. Manasse, F. McSherry (2006) A cool and practical + * alternative to traditional hash tables. In Proceedings of the 7th + * Workshop on Distributed Data and Structures (WDAS'06), Santa Clara, CA, + * January 2006. + * + * This implementation uses precisely two hash functions because that is the + * fewest that can work, and supporting multiple hashes is an implementation + * burden. Here is a reproduction of Figure 1 from Erlingsson et al. (2006) + * that shows approximate expected maximum load factors for various + * configurations: + * + * | #cells/bucket | + * #hashes | 1 | 2 | 4 | 8 | + * --------+-------+-------+-------+-------+ + * 1 | 0.006 | 0.006 | 0.03 | 0.12 | + * 2 | 0.49 | 0.86 |>0.93< |>0.96< | + * 3 | 0.91 | 0.97 | 0.98 | 0.999 | + * 4 | 0.97 | 0.99 | 0.999 | | + * + * The number of cells per bucket is chosen such that a bucket fits in one cache + * line. So, on 32- and 64-bit systems, we use (8,2) and (4,2) cuckoo hashing, + * respectively. + * + ******************************************************************************/ +#define JEMALLOC_CKH_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static bool ckh_grow(ckh_t *ckh); +static void ckh_shrink(ckh_t *ckh); + +/******************************************************************************/ + +/* + * Search bucket for key and return the cell number if found; SIZE_T_MAX + * otherwise. + */ +JEMALLOC_INLINE size_t +ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) +{ + ckhc_t *cell; + unsigned i; + + for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { + cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; + if (cell->key != NULL && ckh->keycomp(key, cell->key)) + return ((bucket << LG_CKH_BUCKET_CELLS) + i); + } + + return (SIZE_T_MAX); +} + +/* + * Search table for key and return cell number if found; SIZE_T_MAX otherwise. + */ +JEMALLOC_INLINE size_t +ckh_isearch(ckh_t *ckh, const void *key) +{ + size_t hash1, hash2, bucket, cell; + + assert(ckh != NULL); + dassert(ckh->magic == CKH_MAGIC); + + ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + + /* Search primary bucket. */ + bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + cell = ckh_bucket_search(ckh, bucket, key); + if (cell != SIZE_T_MAX) + return (cell); + + /* Search secondary bucket. */ + bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + cell = ckh_bucket_search(ckh, bucket, key); + return (cell); +} + +JEMALLOC_INLINE bool +ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, + const void *data) +{ + ckhc_t *cell; + unsigned offset, i; + + /* + * Cycle through the cells in the bucket, starting at a random position. + * The randomness avoids worst-case search overhead as buckets fill up. + */ + prn32(offset, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); + for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { + cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + + ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; + if (cell->key == NULL) { + cell->key = key; + cell->data = data; + ckh->count++; + return (false); + } + } + + return (true); +} + +/* + * No space is available in bucket. Randomly evict an item, then try to find an + * alternate location for that item. Iteratively repeat this + * eviction/relocation procedure until either success or detection of an + * eviction/relocation bucket cycle. + */ +JEMALLOC_INLINE bool +ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, + void const **argdata) +{ + const void *key, *data, *tkey, *tdata; + ckhc_t *cell; + size_t hash1, hash2, bucket, tbucket; + unsigned i; + + bucket = argbucket; + key = *argkey; + data = *argdata; + while (true) { + /* + * Choose a random item within the bucket to evict. This is + * critical to correct function, because without (eventually) + * evicting all items within a bucket during iteration, it + * would be possible to get stuck in an infinite loop if there + * were an item for which both hashes indicated the same + * bucket. + */ + prn32(i, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); + cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; + assert(cell->key != NULL); + + /* Swap cell->{key,data} and {key,data} (evict). */ + tkey = cell->key; tdata = cell->data; + cell->key = key; cell->data = data; + key = tkey; data = tdata; + +#ifdef CKH_COUNT + ckh->nrelocs++; +#endif + + /* Find the alternate bucket for the evicted item. */ + ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + tbucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + if (tbucket == bucket) { + tbucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + /* + * It may be that (tbucket == bucket) still, if the + * item's hashes both indicate this bucket. However, + * we are guaranteed to eventually escape this bucket + * during iteration, assuming pseudo-random item + * selection (true randomness would make infinite + * looping a remote possibility). The reason we can + * never get trapped forever is that there are two + * cases: + * + * 1) This bucket == argbucket, so we will quickly + * detect an eviction cycle and terminate. + * 2) An item was evicted to this bucket from another, + * which means that at least one item in this bucket + * has hashes that indicate distinct buckets. + */ + } + /* Check for a cycle. */ + if (tbucket == argbucket) { + *argkey = key; + *argdata = data; + return (true); + } + + bucket = tbucket; + if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) + return (false); + } +} + +JEMALLOC_INLINE bool +ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) +{ + size_t hash1, hash2, bucket; + const void *key = *argkey; + const void *data = *argdata; + + ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + + /* Try to insert in primary bucket. */ + bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) + return (false); + + /* Try to insert in secondary bucket. */ + bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) + return (false); + + /* + * Try to find a place for this item via iterative eviction/relocation. + */ + return (ckh_evict_reloc_insert(ckh, bucket, argkey, argdata)); +} + +/* + * Try to rebuild the hash table from scratch by inserting all items from the + * old table into the new. + */ +JEMALLOC_INLINE bool +ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) +{ + size_t count, i, nins; + const void *key, *data; + + count = ckh->count; + ckh->count = 0; + for (i = nins = 0; nins < count; i++) { + if (aTab[i].key != NULL) { + key = aTab[i].key; + data = aTab[i].data; + if (ckh_try_insert(ckh, &key, &data)) { + ckh->count = count; + return (true); + } + nins++; + } + } + + return (false); +} + +static bool +ckh_grow(ckh_t *ckh) +{ + bool ret; + ckhc_t *tab, *ttab; + size_t lg_curcells; + unsigned lg_prevbuckets; + +#ifdef CKH_COUNT + ckh->ngrows++; +#endif + + /* + * It is possible (though unlikely, given well behaved hashes) that the + * table will have to be doubled more than once in order to create a + * usable table. + */ + lg_prevbuckets = ckh->lg_curbuckets; + lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS; + while (true) { + size_t usize; + + lg_curcells++; + usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); + if (usize == 0) { + ret = true; + goto RETURN; + } + tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); + if (tab == NULL) { + ret = true; + goto RETURN; + } + /* Swap in new table. */ + ttab = ckh->tab; + ckh->tab = tab; + tab = ttab; + ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; + + if (ckh_rebuild(ckh, tab) == false) { + idalloc(tab); + break; + } + + /* Rebuilding failed, so back out partially rebuilt table. */ + idalloc(ckh->tab); + ckh->tab = tab; + ckh->lg_curbuckets = lg_prevbuckets; + } + + ret = false; +RETURN: + return (ret); +} + +static void +ckh_shrink(ckh_t *ckh) +{ + ckhc_t *tab, *ttab; + size_t lg_curcells, usize; + unsigned lg_prevbuckets; + + /* + * It is possible (though unlikely, given well behaved hashes) that the + * table rebuild will fail. + */ + lg_prevbuckets = ckh->lg_curbuckets; + lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; + usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); + if (usize == 0) + return; + tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); + if (tab == NULL) { + /* + * An OOM error isn't worth propagating, since it doesn't + * prevent this or future operations from proceeding. + */ + return; + } + /* Swap in new table. */ + ttab = ckh->tab; + ckh->tab = tab; + tab = ttab; + ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; + + if (ckh_rebuild(ckh, tab) == false) { + idalloc(tab); +#ifdef CKH_COUNT + ckh->nshrinks++; +#endif + return; + } + + /* Rebuilding failed, so back out partially rebuilt table. */ + idalloc(ckh->tab); + ckh->tab = tab; + ckh->lg_curbuckets = lg_prevbuckets; +#ifdef CKH_COUNT + ckh->nshrinkfails++; +#endif +} + +bool +ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) +{ + bool ret; + size_t mincells, usize; + unsigned lg_mincells; + + assert(minitems > 0); + assert(hash != NULL); + assert(keycomp != NULL); + +#ifdef CKH_COUNT + ckh->ngrows = 0; + ckh->nshrinks = 0; + ckh->nshrinkfails = 0; + ckh->ninserts = 0; + ckh->nrelocs = 0; +#endif + ckh->prn_state = 42; /* Value doesn't really matter. */ + ckh->count = 0; + + /* + * Find the minimum power of 2 that is large enough to fit aBaseCount + * entries. We are using (2+,2) cuckoo hashing, which has an expected + * maximum load factor of at least ~0.86, so 0.75 is a conservative load + * factor that will typically allow 2^aLgMinItems to fit without ever + * growing the table. + */ + assert(LG_CKH_BUCKET_CELLS > 0); + mincells = ((minitems + (3 - (minitems % 3))) / 3) << 2; + for (lg_mincells = LG_CKH_BUCKET_CELLS; + (ZU(1) << lg_mincells) < mincells; + lg_mincells++) + ; /* Do nothing. */ + ckh->lg_minbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; + ckh->lg_curbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; + ckh->hash = hash; + ckh->keycomp = keycomp; + + usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE, NULL); + if (usize == 0) { + ret = true; + goto RETURN; + } + ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); + if (ckh->tab == NULL) { + ret = true; + goto RETURN; + } + +#ifdef JEMALLOC_DEBUG + ckh->magic = CKH_MAGIC; +#endif + + ret = false; +RETURN: + return (ret); +} + +void +ckh_delete(ckh_t *ckh) +{ + + assert(ckh != NULL); + dassert(ckh->magic == CKH_MAGIC); + +#ifdef CKH_VERBOSE + malloc_printf( + "%s(%p): ngrows: %"PRIu64", nshrinks: %"PRIu64"," + " nshrinkfails: %"PRIu64", ninserts: %"PRIu64"," + " nrelocs: %"PRIu64"\n", __func__, ckh, + (unsigned long long)ckh->ngrows, + (unsigned long long)ckh->nshrinks, + (unsigned long long)ckh->nshrinkfails, + (unsigned long long)ckh->ninserts, + (unsigned long long)ckh->nrelocs); +#endif + + idalloc(ckh->tab); +#ifdef JEMALLOC_DEBUG + memset(ckh, 0x5a, sizeof(ckh_t)); +#endif +} + +size_t +ckh_count(ckh_t *ckh) +{ + + assert(ckh != NULL); + dassert(ckh->magic == CKH_MAGIC); + + return (ckh->count); +} + +bool +ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) +{ + size_t i, ncells; + + for (i = *tabind, ncells = (ZU(1) << (ckh->lg_curbuckets + + LG_CKH_BUCKET_CELLS)); i < ncells; i++) { + if (ckh->tab[i].key != NULL) { + if (key != NULL) + *key = (void *)ckh->tab[i].key; + if (data != NULL) + *data = (void *)ckh->tab[i].data; + *tabind = i + 1; + return (false); + } + } + + return (true); +} + +bool +ckh_insert(ckh_t *ckh, const void *key, const void *data) +{ + bool ret; + + assert(ckh != NULL); + dassert(ckh->magic == CKH_MAGIC); + assert(ckh_search(ckh, key, NULL, NULL)); + +#ifdef CKH_COUNT + ckh->ninserts++; +#endif + + while (ckh_try_insert(ckh, &key, &data)) { + if (ckh_grow(ckh)) { + ret = true; + goto RETURN; + } + } + + ret = false; +RETURN: + return (ret); +} + +bool +ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data) +{ + size_t cell; + + assert(ckh != NULL); + dassert(ckh->magic == CKH_MAGIC); + + cell = ckh_isearch(ckh, searchkey); + if (cell != SIZE_T_MAX) { + if (key != NULL) + *key = (void *)ckh->tab[cell].key; + if (data != NULL) + *data = (void *)ckh->tab[cell].data; + ckh->tab[cell].key = NULL; + ckh->tab[cell].data = NULL; /* Not necessary. */ + + ckh->count--; + /* Try to halve the table if it is less than 1/4 full. */ + if (ckh->count < (ZU(1) << (ckh->lg_curbuckets + + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets + > ckh->lg_minbuckets) { + /* Ignore error due to OOM. */ + ckh_shrink(ckh); + } + + return (false); + } + + return (true); +} + +bool +ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) +{ + size_t cell; + + assert(ckh != NULL); + dassert(ckh->magic == CKH_MAGIC); + + cell = ckh_isearch(ckh, searchkey); + if (cell != SIZE_T_MAX) { + if (key != NULL) + *key = (void *)ckh->tab[cell].key; + if (data != NULL) + *data = (void *)ckh->tab[cell].data; + return (false); + } + + return (true); +} + +void +ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) +{ + size_t ret1, ret2; + uint64_t h; + + assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); + assert(hash1 != NULL); + assert(hash2 != NULL); + + h = hash(key, strlen((const char *)key), 0x94122f335b332aeaLLU); + if (minbits <= 32) { + /* + * Avoid doing multiple hashes, since a single hash provides + * enough bits. + */ + ret1 = h & ZU(0xffffffffU); + ret2 = h >> 32; + } else { + ret1 = h; + ret2 = hash(key, strlen((const char *)key), + 0x8432a476666bbc13LLU); + } + + *hash1 = ret1; + *hash2 = ret2; +} + +bool +ckh_string_keycomp(const void *k1, const void *k2) +{ + + assert(k1 != NULL); + assert(k2 != NULL); + + return (strcmp((char *)k1, (char *)k2) ? false : true); +} + +void +ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, + size_t *hash2) +{ + size_t ret1, ret2; + uint64_t h; + union { + const void *v; + uint64_t i; + } u; + + assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); + assert(hash1 != NULL); + assert(hash2 != NULL); + + assert(sizeof(u.v) == sizeof(u.i)); +#if (LG_SIZEOF_PTR != LG_SIZEOF_INT) + u.i = 0; +#endif + u.v = key; + h = hash(&u.i, sizeof(u.i), 0xd983396e68886082LLU); + if (minbits <= 32) { + /* + * Avoid doing multiple hashes, since a single hash provides + * enough bits. + */ + ret1 = h & ZU(0xffffffffU); + ret2 = h >> 32; + } else { + assert(SIZEOF_PTR == 8); + ret1 = h; + ret2 = hash(&u.i, sizeof(u.i), 0x5e2be9aff8709a5dLLU); + } + + *hash1 = ret1; + *hash2 = ret2; +} + +bool +ckh_pointer_keycomp(const void *k1, const void *k2) +{ + + return ((k1 == k2) ? true : false); +} diff --git a/deps/jemalloc.orig/src/ctl.c b/deps/jemalloc.orig/src/ctl.c new file mode 100644 index 00000000000..e5336d36949 --- /dev/null +++ b/deps/jemalloc.orig/src/ctl.c @@ -0,0 +1,1670 @@ +#define JEMALLOC_CTL_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +/* + * ctl_mtx protects the following: + * - ctl_stats.* + * - opt_prof_active + * - swap_enabled + * - swap_prezeroed + */ +static malloc_mutex_t ctl_mtx; +static bool ctl_initialized; +static uint64_t ctl_epoch; +static ctl_stats_t ctl_stats; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +#define CTL_PROTO(n) \ +static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ + size_t *oldlenp, void *newp, size_t newlen); + +#define INDEX_PROTO(n) \ +const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \ + size_t i); + +#ifdef JEMALLOC_STATS +static bool ctl_arena_init(ctl_arena_stats_t *astats); +#endif +static void ctl_arena_clear(ctl_arena_stats_t *astats); +#ifdef JEMALLOC_STATS +static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, + arena_t *arena); +static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, + ctl_arena_stats_t *astats); +#endif +static void ctl_arena_refresh(arena_t *arena, unsigned i); +static void ctl_refresh(void); +static bool ctl_init(void); +static int ctl_lookup(const char *name, ctl_node_t const **nodesp, + size_t *mibp, size_t *depthp); + +CTL_PROTO(version) +CTL_PROTO(epoch) +#ifdef JEMALLOC_TCACHE +CTL_PROTO(tcache_flush) +#endif +CTL_PROTO(thread_arena) +#ifdef JEMALLOC_STATS +CTL_PROTO(thread_allocated) +CTL_PROTO(thread_allocatedp) +CTL_PROTO(thread_deallocated) +CTL_PROTO(thread_deallocatedp) +#endif +CTL_PROTO(config_debug) +CTL_PROTO(config_dss) +CTL_PROTO(config_dynamic_page_shift) +CTL_PROTO(config_fill) +CTL_PROTO(config_lazy_lock) +CTL_PROTO(config_prof) +CTL_PROTO(config_prof_libgcc) +CTL_PROTO(config_prof_libunwind) +CTL_PROTO(config_stats) +CTL_PROTO(config_swap) +CTL_PROTO(config_sysv) +CTL_PROTO(config_tcache) +CTL_PROTO(config_tiny) +CTL_PROTO(config_tls) +CTL_PROTO(config_xmalloc) +CTL_PROTO(opt_abort) +CTL_PROTO(opt_lg_qspace_max) +CTL_PROTO(opt_lg_cspace_max) +CTL_PROTO(opt_lg_chunk) +CTL_PROTO(opt_narenas) +CTL_PROTO(opt_lg_dirty_mult) +CTL_PROTO(opt_stats_print) +#ifdef JEMALLOC_FILL +CTL_PROTO(opt_junk) +CTL_PROTO(opt_zero) +#endif +#ifdef JEMALLOC_SYSV +CTL_PROTO(opt_sysv) +#endif +#ifdef JEMALLOC_XMALLOC +CTL_PROTO(opt_xmalloc) +#endif +#ifdef JEMALLOC_TCACHE +CTL_PROTO(opt_tcache) +CTL_PROTO(opt_lg_tcache_gc_sweep) +#endif +#ifdef JEMALLOC_PROF +CTL_PROTO(opt_prof) +CTL_PROTO(opt_prof_prefix) +CTL_PROTO(opt_prof_active) +CTL_PROTO(opt_lg_prof_bt_max) +CTL_PROTO(opt_lg_prof_sample) +CTL_PROTO(opt_lg_prof_interval) +CTL_PROTO(opt_prof_gdump) +CTL_PROTO(opt_prof_leak) +CTL_PROTO(opt_prof_accum) +CTL_PROTO(opt_lg_prof_tcmax) +#endif +#ifdef JEMALLOC_SWAP +CTL_PROTO(opt_overcommit) +#endif +CTL_PROTO(arenas_bin_i_size) +CTL_PROTO(arenas_bin_i_nregs) +CTL_PROTO(arenas_bin_i_run_size) +INDEX_PROTO(arenas_bin_i) +CTL_PROTO(arenas_lrun_i_size) +INDEX_PROTO(arenas_lrun_i) +CTL_PROTO(arenas_narenas) +CTL_PROTO(arenas_initialized) +CTL_PROTO(arenas_quantum) +CTL_PROTO(arenas_cacheline) +CTL_PROTO(arenas_subpage) +CTL_PROTO(arenas_pagesize) +CTL_PROTO(arenas_chunksize) +#ifdef JEMALLOC_TINY +CTL_PROTO(arenas_tspace_min) +CTL_PROTO(arenas_tspace_max) +#endif +CTL_PROTO(arenas_qspace_min) +CTL_PROTO(arenas_qspace_max) +CTL_PROTO(arenas_cspace_min) +CTL_PROTO(arenas_cspace_max) +CTL_PROTO(arenas_sspace_min) +CTL_PROTO(arenas_sspace_max) +#ifdef JEMALLOC_TCACHE +CTL_PROTO(arenas_tcache_max) +#endif +CTL_PROTO(arenas_ntbins) +CTL_PROTO(arenas_nqbins) +CTL_PROTO(arenas_ncbins) +CTL_PROTO(arenas_nsbins) +CTL_PROTO(arenas_nbins) +#ifdef JEMALLOC_TCACHE +CTL_PROTO(arenas_nhbins) +#endif +CTL_PROTO(arenas_nlruns) +CTL_PROTO(arenas_purge) +#ifdef JEMALLOC_PROF +CTL_PROTO(prof_active) +CTL_PROTO(prof_dump) +CTL_PROTO(prof_interval) +#endif +#ifdef JEMALLOC_STATS +CTL_PROTO(stats_chunks_current) +CTL_PROTO(stats_chunks_total) +CTL_PROTO(stats_chunks_high) +CTL_PROTO(stats_huge_allocated) +CTL_PROTO(stats_huge_nmalloc) +CTL_PROTO(stats_huge_ndalloc) +CTL_PROTO(stats_arenas_i_small_allocated) +CTL_PROTO(stats_arenas_i_small_nmalloc) +CTL_PROTO(stats_arenas_i_small_ndalloc) +CTL_PROTO(stats_arenas_i_small_nrequests) +CTL_PROTO(stats_arenas_i_large_allocated) +CTL_PROTO(stats_arenas_i_large_nmalloc) +CTL_PROTO(stats_arenas_i_large_ndalloc) +CTL_PROTO(stats_arenas_i_large_nrequests) +CTL_PROTO(stats_arenas_i_bins_j_allocated) +CTL_PROTO(stats_arenas_i_bins_j_nmalloc) +CTL_PROTO(stats_arenas_i_bins_j_ndalloc) +CTL_PROTO(stats_arenas_i_bins_j_nrequests) +#ifdef JEMALLOC_TCACHE +CTL_PROTO(stats_arenas_i_bins_j_nfills) +CTL_PROTO(stats_arenas_i_bins_j_nflushes) +#endif +CTL_PROTO(stats_arenas_i_bins_j_nruns) +CTL_PROTO(stats_arenas_i_bins_j_nreruns) +CTL_PROTO(stats_arenas_i_bins_j_highruns) +CTL_PROTO(stats_arenas_i_bins_j_curruns) +INDEX_PROTO(stats_arenas_i_bins_j) +CTL_PROTO(stats_arenas_i_lruns_j_nmalloc) +CTL_PROTO(stats_arenas_i_lruns_j_ndalloc) +CTL_PROTO(stats_arenas_i_lruns_j_nrequests) +CTL_PROTO(stats_arenas_i_lruns_j_highruns) +CTL_PROTO(stats_arenas_i_lruns_j_curruns) +INDEX_PROTO(stats_arenas_i_lruns_j) +#endif +CTL_PROTO(stats_arenas_i_nthreads) +CTL_PROTO(stats_arenas_i_pactive) +CTL_PROTO(stats_arenas_i_pdirty) +#ifdef JEMALLOC_STATS +CTL_PROTO(stats_arenas_i_mapped) +CTL_PROTO(stats_arenas_i_npurge) +CTL_PROTO(stats_arenas_i_nmadvise) +CTL_PROTO(stats_arenas_i_purged) +#endif +INDEX_PROTO(stats_arenas_i) +#ifdef JEMALLOC_STATS +CTL_PROTO(stats_cactive) +CTL_PROTO(stats_allocated) +CTL_PROTO(stats_active) +CTL_PROTO(stats_mapped) +#endif +#ifdef JEMALLOC_SWAP +# ifdef JEMALLOC_STATS +CTL_PROTO(swap_avail) +# endif +CTL_PROTO(swap_prezeroed) +CTL_PROTO(swap_nfds) +CTL_PROTO(swap_fds) +#endif + +/******************************************************************************/ +/* mallctl tree. */ + +/* Maximum tree depth. */ +#define CTL_MAX_DEPTH 6 + +#define NAME(n) true, {.named = {n +#define CHILD(c) sizeof(c##_node) / sizeof(ctl_node_t), c##_node}}, NULL +#define CTL(c) 0, NULL}}, c##_ctl + +/* + * Only handles internal indexed nodes, since there are currently no external + * ones. + */ +#define INDEX(i) false, {.indexed = {i##_index}}, NULL + +#ifdef JEMALLOC_TCACHE +static const ctl_node_t tcache_node[] = { + {NAME("flush"), CTL(tcache_flush)} +}; +#endif + +static const ctl_node_t thread_node[] = { + {NAME("arena"), CTL(thread_arena)} +#ifdef JEMALLOC_STATS + , + {NAME("allocated"), CTL(thread_allocated)}, + {NAME("allocatedp"), CTL(thread_allocatedp)}, + {NAME("deallocated"), CTL(thread_deallocated)}, + {NAME("deallocatedp"), CTL(thread_deallocatedp)} +#endif +}; + +static const ctl_node_t config_node[] = { + {NAME("debug"), CTL(config_debug)}, + {NAME("dss"), CTL(config_dss)}, + {NAME("dynamic_page_shift"), CTL(config_dynamic_page_shift)}, + {NAME("fill"), CTL(config_fill)}, + {NAME("lazy_lock"), CTL(config_lazy_lock)}, + {NAME("prof"), CTL(config_prof)}, + {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, + {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, + {NAME("stats"), CTL(config_stats)}, + {NAME("swap"), CTL(config_swap)}, + {NAME("sysv"), CTL(config_sysv)}, + {NAME("tcache"), CTL(config_tcache)}, + {NAME("tiny"), CTL(config_tiny)}, + {NAME("tls"), CTL(config_tls)}, + {NAME("xmalloc"), CTL(config_xmalloc)} +}; + +static const ctl_node_t opt_node[] = { + {NAME("abort"), CTL(opt_abort)}, + {NAME("lg_qspace_max"), CTL(opt_lg_qspace_max)}, + {NAME("lg_cspace_max"), CTL(opt_lg_cspace_max)}, + {NAME("lg_chunk"), CTL(opt_lg_chunk)}, + {NAME("narenas"), CTL(opt_narenas)}, + {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, + {NAME("stats_print"), CTL(opt_stats_print)} +#ifdef JEMALLOC_FILL + , + {NAME("junk"), CTL(opt_junk)}, + {NAME("zero"), CTL(opt_zero)} +#endif +#ifdef JEMALLOC_SYSV + , + {NAME("sysv"), CTL(opt_sysv)} +#endif +#ifdef JEMALLOC_XMALLOC + , + {NAME("xmalloc"), CTL(opt_xmalloc)} +#endif +#ifdef JEMALLOC_TCACHE + , + {NAME("tcache"), CTL(opt_tcache)}, + {NAME("lg_tcache_gc_sweep"), CTL(opt_lg_tcache_gc_sweep)} +#endif +#ifdef JEMALLOC_PROF + , + {NAME("prof"), CTL(opt_prof)}, + {NAME("prof_prefix"), CTL(opt_prof_prefix)}, + {NAME("prof_active"), CTL(opt_prof_active)}, + {NAME("lg_prof_bt_max"), CTL(opt_lg_prof_bt_max)}, + {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, + {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, + {NAME("prof_gdump"), CTL(opt_prof_gdump)}, + {NAME("prof_leak"), CTL(opt_prof_leak)}, + {NAME("prof_accum"), CTL(opt_prof_accum)}, + {NAME("lg_prof_tcmax"), CTL(opt_lg_prof_tcmax)} +#endif +#ifdef JEMALLOC_SWAP + , + {NAME("overcommit"), CTL(opt_overcommit)} +#endif +}; + +static const ctl_node_t arenas_bin_i_node[] = { + {NAME("size"), CTL(arenas_bin_i_size)}, + {NAME("nregs"), CTL(arenas_bin_i_nregs)}, + {NAME("run_size"), CTL(arenas_bin_i_run_size)} +}; +static const ctl_node_t super_arenas_bin_i_node[] = { + {NAME(""), CHILD(arenas_bin_i)} +}; + +static const ctl_node_t arenas_bin_node[] = { + {INDEX(arenas_bin_i)} +}; + +static const ctl_node_t arenas_lrun_i_node[] = { + {NAME("size"), CTL(arenas_lrun_i_size)} +}; +static const ctl_node_t super_arenas_lrun_i_node[] = { + {NAME(""), CHILD(arenas_lrun_i)} +}; + +static const ctl_node_t arenas_lrun_node[] = { + {INDEX(arenas_lrun_i)} +}; + +static const ctl_node_t arenas_node[] = { + {NAME("narenas"), CTL(arenas_narenas)}, + {NAME("initialized"), CTL(arenas_initialized)}, + {NAME("quantum"), CTL(arenas_quantum)}, + {NAME("cacheline"), CTL(arenas_cacheline)}, + {NAME("subpage"), CTL(arenas_subpage)}, + {NAME("pagesize"), CTL(arenas_pagesize)}, + {NAME("chunksize"), CTL(arenas_chunksize)}, +#ifdef JEMALLOC_TINY + {NAME("tspace_min"), CTL(arenas_tspace_min)}, + {NAME("tspace_max"), CTL(arenas_tspace_max)}, +#endif + {NAME("qspace_min"), CTL(arenas_qspace_min)}, + {NAME("qspace_max"), CTL(arenas_qspace_max)}, + {NAME("cspace_min"), CTL(arenas_cspace_min)}, + {NAME("cspace_max"), CTL(arenas_cspace_max)}, + {NAME("sspace_min"), CTL(arenas_sspace_min)}, + {NAME("sspace_max"), CTL(arenas_sspace_max)}, +#ifdef JEMALLOC_TCACHE + {NAME("tcache_max"), CTL(arenas_tcache_max)}, +#endif + {NAME("ntbins"), CTL(arenas_ntbins)}, + {NAME("nqbins"), CTL(arenas_nqbins)}, + {NAME("ncbins"), CTL(arenas_ncbins)}, + {NAME("nsbins"), CTL(arenas_nsbins)}, + {NAME("nbins"), CTL(arenas_nbins)}, +#ifdef JEMALLOC_TCACHE + {NAME("nhbins"), CTL(arenas_nhbins)}, +#endif + {NAME("bin"), CHILD(arenas_bin)}, + {NAME("nlruns"), CTL(arenas_nlruns)}, + {NAME("lrun"), CHILD(arenas_lrun)}, + {NAME("purge"), CTL(arenas_purge)} +}; + +#ifdef JEMALLOC_PROF +static const ctl_node_t prof_node[] = { + {NAME("active"), CTL(prof_active)}, + {NAME("dump"), CTL(prof_dump)}, + {NAME("interval"), CTL(prof_interval)} +}; +#endif + +#ifdef JEMALLOC_STATS +static const ctl_node_t stats_chunks_node[] = { + {NAME("current"), CTL(stats_chunks_current)}, + {NAME("total"), CTL(stats_chunks_total)}, + {NAME("high"), CTL(stats_chunks_high)} +}; + +static const ctl_node_t stats_huge_node[] = { + {NAME("allocated"), CTL(stats_huge_allocated)}, + {NAME("nmalloc"), CTL(stats_huge_nmalloc)}, + {NAME("ndalloc"), CTL(stats_huge_ndalloc)} +}; + +static const ctl_node_t stats_arenas_i_small_node[] = { + {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} +}; + +static const ctl_node_t stats_arenas_i_large_node[] = { + {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} +}; + +static const ctl_node_t stats_arenas_i_bins_j_node[] = { + {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, + {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, +#ifdef JEMALLOC_TCACHE + {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, + {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, +#endif + {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, + {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, + {NAME("highruns"), CTL(stats_arenas_i_bins_j_highruns)}, + {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} +}; +static const ctl_node_t super_stats_arenas_i_bins_j_node[] = { + {NAME(""), CHILD(stats_arenas_i_bins_j)} +}; + +static const ctl_node_t stats_arenas_i_bins_node[] = { + {INDEX(stats_arenas_i_bins_j)} +}; + +static const ctl_node_t stats_arenas_i_lruns_j_node[] = { + {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, + {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, + {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, + {NAME("highruns"), CTL(stats_arenas_i_lruns_j_highruns)}, + {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} +}; +static const ctl_node_t super_stats_arenas_i_lruns_j_node[] = { + {NAME(""), CHILD(stats_arenas_i_lruns_j)} +}; + +static const ctl_node_t stats_arenas_i_lruns_node[] = { + {INDEX(stats_arenas_i_lruns_j)} +}; +#endif + +static const ctl_node_t stats_arenas_i_node[] = { + {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, + {NAME("pactive"), CTL(stats_arenas_i_pactive)}, + {NAME("pdirty"), CTL(stats_arenas_i_pdirty)} +#ifdef JEMALLOC_STATS + , + {NAME("mapped"), CTL(stats_arenas_i_mapped)}, + {NAME("npurge"), CTL(stats_arenas_i_npurge)}, + {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, + {NAME("purged"), CTL(stats_arenas_i_purged)}, + {NAME("small"), CHILD(stats_arenas_i_small)}, + {NAME("large"), CHILD(stats_arenas_i_large)}, + {NAME("bins"), CHILD(stats_arenas_i_bins)}, + {NAME("lruns"), CHILD(stats_arenas_i_lruns)} +#endif +}; +static const ctl_node_t super_stats_arenas_i_node[] = { + {NAME(""), CHILD(stats_arenas_i)} +}; + +static const ctl_node_t stats_arenas_node[] = { + {INDEX(stats_arenas_i)} +}; + +static const ctl_node_t stats_node[] = { +#ifdef JEMALLOC_STATS + {NAME("cactive"), CTL(stats_cactive)}, + {NAME("allocated"), CTL(stats_allocated)}, + {NAME("active"), CTL(stats_active)}, + {NAME("mapped"), CTL(stats_mapped)}, + {NAME("chunks"), CHILD(stats_chunks)}, + {NAME("huge"), CHILD(stats_huge)}, +#endif + {NAME("arenas"), CHILD(stats_arenas)} +}; + +#ifdef JEMALLOC_SWAP +static const ctl_node_t swap_node[] = { +# ifdef JEMALLOC_STATS + {NAME("avail"), CTL(swap_avail)}, +# endif + {NAME("prezeroed"), CTL(swap_prezeroed)}, + {NAME("nfds"), CTL(swap_nfds)}, + {NAME("fds"), CTL(swap_fds)} +}; +#endif + +static const ctl_node_t root_node[] = { + {NAME("version"), CTL(version)}, + {NAME("epoch"), CTL(epoch)}, +#ifdef JEMALLOC_TCACHE + {NAME("tcache"), CHILD(tcache)}, +#endif + {NAME("thread"), CHILD(thread)}, + {NAME("config"), CHILD(config)}, + {NAME("opt"), CHILD(opt)}, + {NAME("arenas"), CHILD(arenas)}, +#ifdef JEMALLOC_PROF + {NAME("prof"), CHILD(prof)}, +#endif + {NAME("stats"), CHILD(stats)} +#ifdef JEMALLOC_SWAP + , + {NAME("swap"), CHILD(swap)} +#endif +}; +static const ctl_node_t super_root_node[] = { + {NAME(""), CHILD(root)} +}; + +#undef NAME +#undef CHILD +#undef CTL +#undef INDEX + +/******************************************************************************/ + +#ifdef JEMALLOC_STATS +static bool +ctl_arena_init(ctl_arena_stats_t *astats) +{ + + if (astats->bstats == NULL) { + astats->bstats = (malloc_bin_stats_t *)base_alloc(nbins * + sizeof(malloc_bin_stats_t)); + if (astats->bstats == NULL) + return (true); + } + if (astats->lstats == NULL) { + astats->lstats = (malloc_large_stats_t *)base_alloc(nlclasses * + sizeof(malloc_large_stats_t)); + if (astats->lstats == NULL) + return (true); + } + + return (false); +} +#endif + +static void +ctl_arena_clear(ctl_arena_stats_t *astats) +{ + + astats->pactive = 0; + astats->pdirty = 0; +#ifdef JEMALLOC_STATS + memset(&astats->astats, 0, sizeof(arena_stats_t)); + astats->allocated_small = 0; + astats->nmalloc_small = 0; + astats->ndalloc_small = 0; + astats->nrequests_small = 0; + memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t)); + memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); +#endif +} + +#ifdef JEMALLOC_STATS +static void +ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) +{ + unsigned i; + + arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty, + &cstats->astats, cstats->bstats, cstats->lstats); + + for (i = 0; i < nbins; i++) { + cstats->allocated_small += cstats->bstats[i].allocated; + cstats->nmalloc_small += cstats->bstats[i].nmalloc; + cstats->ndalloc_small += cstats->bstats[i].ndalloc; + cstats->nrequests_small += cstats->bstats[i].nrequests; + } +} + +static void +ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) +{ + unsigned i; + + sstats->pactive += astats->pactive; + sstats->pdirty += astats->pdirty; + + sstats->astats.mapped += astats->astats.mapped; + sstats->astats.npurge += astats->astats.npurge; + sstats->astats.nmadvise += astats->astats.nmadvise; + sstats->astats.purged += astats->astats.purged; + + sstats->allocated_small += astats->allocated_small; + sstats->nmalloc_small += astats->nmalloc_small; + sstats->ndalloc_small += astats->ndalloc_small; + sstats->nrequests_small += astats->nrequests_small; + + sstats->astats.allocated_large += astats->astats.allocated_large; + sstats->astats.nmalloc_large += astats->astats.nmalloc_large; + sstats->astats.ndalloc_large += astats->astats.ndalloc_large; + sstats->astats.nrequests_large += astats->astats.nrequests_large; + + for (i = 0; i < nlclasses; i++) { + sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; + sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; + sstats->lstats[i].nrequests += astats->lstats[i].nrequests; + sstats->lstats[i].highruns += astats->lstats[i].highruns; + sstats->lstats[i].curruns += astats->lstats[i].curruns; + } + + for (i = 0; i < nbins; i++) { + sstats->bstats[i].allocated += astats->bstats[i].allocated; + sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; + sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; + sstats->bstats[i].nrequests += astats->bstats[i].nrequests; +#ifdef JEMALLOC_TCACHE + sstats->bstats[i].nfills += astats->bstats[i].nfills; + sstats->bstats[i].nflushes += astats->bstats[i].nflushes; +#endif + sstats->bstats[i].nruns += astats->bstats[i].nruns; + sstats->bstats[i].reruns += astats->bstats[i].reruns; + sstats->bstats[i].highruns += astats->bstats[i].highruns; + sstats->bstats[i].curruns += astats->bstats[i].curruns; + } +} +#endif + +static void +ctl_arena_refresh(arena_t *arena, unsigned i) +{ + ctl_arena_stats_t *astats = &ctl_stats.arenas[i]; + ctl_arena_stats_t *sstats = &ctl_stats.arenas[narenas]; + + ctl_arena_clear(astats); + + sstats->nthreads += astats->nthreads; +#ifdef JEMALLOC_STATS + ctl_arena_stats_amerge(astats, arena); + /* Merge into sum stats as well. */ + ctl_arena_stats_smerge(sstats, astats); +#else + astats->pactive += arena->nactive; + astats->pdirty += arena->ndirty; + /* Merge into sum stats as well. */ + sstats->pactive += arena->nactive; + sstats->pdirty += arena->ndirty; +#endif +} + +static void +ctl_refresh(void) +{ + unsigned i; + arena_t *tarenas[narenas]; + +#ifdef JEMALLOC_STATS + malloc_mutex_lock(&chunks_mtx); + ctl_stats.chunks.current = stats_chunks.curchunks; + ctl_stats.chunks.total = stats_chunks.nchunks; + ctl_stats.chunks.high = stats_chunks.highchunks; + malloc_mutex_unlock(&chunks_mtx); + + malloc_mutex_lock(&huge_mtx); + ctl_stats.huge.allocated = huge_allocated; + ctl_stats.huge.nmalloc = huge_nmalloc; + ctl_stats.huge.ndalloc = huge_ndalloc; + malloc_mutex_unlock(&huge_mtx); +#endif + + /* + * Clear sum stats, since they will be merged into by + * ctl_arena_refresh(). + */ + ctl_stats.arenas[narenas].nthreads = 0; + ctl_arena_clear(&ctl_stats.arenas[narenas]); + + malloc_mutex_lock(&arenas_lock); + memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); + for (i = 0; i < narenas; i++) { + if (arenas[i] != NULL) + ctl_stats.arenas[i].nthreads = arenas[i]->nthreads; + else + ctl_stats.arenas[i].nthreads = 0; + } + malloc_mutex_unlock(&arenas_lock); + for (i = 0; i < narenas; i++) { + bool initialized = (tarenas[i] != NULL); + + ctl_stats.arenas[i].initialized = initialized; + if (initialized) + ctl_arena_refresh(tarenas[i], i); + } + +#ifdef JEMALLOC_STATS + ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small + + ctl_stats.arenas[narenas].astats.allocated_large + + ctl_stats.huge.allocated; + ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT) + + ctl_stats.huge.allocated; + ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); + +# ifdef JEMALLOC_SWAP + malloc_mutex_lock(&swap_mtx); + ctl_stats.swap_avail = swap_avail; + malloc_mutex_unlock(&swap_mtx); +# endif +#endif + + ctl_epoch++; +} + +static bool +ctl_init(void) +{ + bool ret; + + malloc_mutex_lock(&ctl_mtx); + if (ctl_initialized == false) { +#ifdef JEMALLOC_STATS + unsigned i; +#endif + + /* + * Allocate space for one extra arena stats element, which + * contains summed stats across all arenas. + */ + ctl_stats.arenas = (ctl_arena_stats_t *)base_alloc( + (narenas + 1) * sizeof(ctl_arena_stats_t)); + if (ctl_stats.arenas == NULL) { + ret = true; + goto RETURN; + } + memset(ctl_stats.arenas, 0, (narenas + 1) * + sizeof(ctl_arena_stats_t)); + + /* + * Initialize all stats structures, regardless of whether they + * ever get used. Lazy initialization would allow errors to + * cause inconsistent state to be viewable by the application. + */ +#ifdef JEMALLOC_STATS + for (i = 0; i <= narenas; i++) { + if (ctl_arena_init(&ctl_stats.arenas[i])) { + ret = true; + goto RETURN; + } + } +#endif + ctl_stats.arenas[narenas].initialized = true; + + ctl_epoch = 0; + ctl_refresh(); + ctl_initialized = true; + } + + ret = false; +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +static int +ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, + size_t *depthp) +{ + int ret; + const char *elm, *tdot, *dot; + size_t elen, i, j; + const ctl_node_t *node; + + elm = name; + /* Equivalent to strchrnul(). */ + dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); + elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); + if (elen == 0) { + ret = ENOENT; + goto RETURN; + } + node = super_root_node; + for (i = 0; i < *depthp; i++) { + assert(node->named); + assert(node->u.named.nchildren > 0); + if (node->u.named.children[0].named) { + const ctl_node_t *pnode = node; + + /* Children are named. */ + for (j = 0; j < node->u.named.nchildren; j++) { + const ctl_node_t *child = + &node->u.named.children[j]; + if (strlen(child->u.named.name) == elen + && strncmp(elm, child->u.named.name, + elen) == 0) { + node = child; + if (nodesp != NULL) + nodesp[i] = node; + mibp[i] = j; + break; + } + } + if (node == pnode) { + ret = ENOENT; + goto RETURN; + } + } else { + unsigned long index; + const ctl_node_t *inode; + + /* Children are indexed. */ + index = strtoul(elm, NULL, 10); + if (index == ULONG_MAX) { + ret = ENOENT; + goto RETURN; + } + + inode = &node->u.named.children[0]; + node = inode->u.indexed.index(mibp, *depthp, + index); + if (node == NULL) { + ret = ENOENT; + goto RETURN; + } + + if (nodesp != NULL) + nodesp[i] = node; + mibp[i] = (size_t)index; + } + + if (node->ctl != NULL) { + /* Terminal node. */ + if (*dot != '\0') { + /* + * The name contains more elements than are + * in this path through the tree. + */ + ret = ENOENT; + goto RETURN; + } + /* Complete lookup successful. */ + *depthp = i + 1; + break; + } + + /* Update elm. */ + if (*dot == '\0') { + /* No more elements. */ + ret = ENOENT; + goto RETURN; + } + elm = &dot[1]; + dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : + strchr(elm, '\0'); + elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); + } + + ret = 0; +RETURN: + return (ret); +} + +int +ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) +{ + int ret; + size_t depth; + ctl_node_t const *nodes[CTL_MAX_DEPTH]; + size_t mib[CTL_MAX_DEPTH]; + + if (ctl_initialized == false && ctl_init()) { + ret = EAGAIN; + goto RETURN; + } + + depth = CTL_MAX_DEPTH; + ret = ctl_lookup(name, nodes, mib, &depth); + if (ret != 0) + goto RETURN; + + if (nodes[depth-1]->ctl == NULL) { + /* The name refers to a partial path through the ctl tree. */ + ret = ENOENT; + goto RETURN; + } + + ret = nodes[depth-1]->ctl(mib, depth, oldp, oldlenp, newp, newlen); +RETURN: + return(ret); +} + +int +ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp) +{ + int ret; + + if (ctl_initialized == false && ctl_init()) { + ret = EAGAIN; + goto RETURN; + } + + ret = ctl_lookup(name, NULL, mibp, miblenp); +RETURN: + return(ret); +} + +int +ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + const ctl_node_t *node; + size_t i; + + if (ctl_initialized == false && ctl_init()) { + ret = EAGAIN; + goto RETURN; + } + + /* Iterate down the tree. */ + node = super_root_node; + for (i = 0; i < miblen; i++) { + if (node->u.named.children[0].named) { + /* Children are named. */ + if (node->u.named.nchildren <= mib[i]) { + ret = ENOENT; + goto RETURN; + } + node = &node->u.named.children[mib[i]]; + } else { + const ctl_node_t *inode; + + /* Indexed element. */ + inode = &node->u.named.children[0]; + node = inode->u.indexed.index(mib, miblen, mib[i]); + if (node == NULL) { + ret = ENOENT; + goto RETURN; + } + } + } + + /* Call the ctl function. */ + if (node->ctl == NULL) { + /* Partial MIB. */ + ret = ENOENT; + goto RETURN; + } + ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); + +RETURN: + return(ret); +} + +bool +ctl_boot(void) +{ + + if (malloc_mutex_init(&ctl_mtx)) + return (true); + + ctl_initialized = false; + + return (false); +} + +/******************************************************************************/ +/* *_ctl() functions. */ + +#define READONLY() do { \ + if (newp != NULL || newlen != 0) { \ + ret = EPERM; \ + goto RETURN; \ + } \ +} while (0) + +#define WRITEONLY() do { \ + if (oldp != NULL || oldlenp != NULL) { \ + ret = EPERM; \ + goto RETURN; \ + } \ +} while (0) + +#define VOID() do { \ + READONLY(); \ + WRITEONLY(); \ +} while (0) + +#define READ(v, t) do { \ + if (oldp != NULL && oldlenp != NULL) { \ + if (*oldlenp != sizeof(t)) { \ + size_t copylen = (sizeof(t) <= *oldlenp) \ + ? sizeof(t) : *oldlenp; \ + memcpy(oldp, (void *)&v, copylen); \ + ret = EINVAL; \ + goto RETURN; \ + } else \ + *(t *)oldp = v; \ + } \ +} while (0) + +#define WRITE(v, t) do { \ + if (newp != NULL) { \ + if (newlen != sizeof(t)) { \ + ret = EINVAL; \ + goto RETURN; \ + } \ + v = *(t *)newp; \ + } \ +} while (0) + +#define CTL_RO_GEN(n, v, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + \ + malloc_mutex_lock(&ctl_mtx); \ + READONLY(); \ + oldval = v; \ + READ(oldval, t); \ + \ + ret = 0; \ +RETURN: \ + malloc_mutex_unlock(&ctl_mtx); \ + return (ret); \ +} + +/* + * ctl_mtx is not acquired, under the assumption that no pertinent data will + * mutate during the call. + */ +#define CTL_RO_NL_GEN(n, v, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + \ + READONLY(); \ + oldval = v; \ + READ(oldval, t); \ + \ + ret = 0; \ +RETURN: \ + return (ret); \ +} + +#define CTL_RO_TRUE_GEN(n) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + bool oldval; \ + \ + READONLY(); \ + oldval = true; \ + READ(oldval, bool); \ + \ + ret = 0; \ +RETURN: \ + return (ret); \ +} + +#define CTL_RO_FALSE_GEN(n) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + bool oldval; \ + \ + READONLY(); \ + oldval = false; \ + READ(oldval, bool); \ + \ + ret = 0; \ +RETURN: \ + return (ret); \ +} + +CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) + +static int +epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + uint64_t newval; + + malloc_mutex_lock(&ctl_mtx); + newval = 0; + WRITE(newval, uint64_t); + if (newval != 0) + ctl_refresh(); + READ(ctl_epoch, uint64_t); + + ret = 0; +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +#ifdef JEMALLOC_TCACHE +static int +tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + tcache_t *tcache; + + VOID(); + + tcache = TCACHE_GET(); + if (tcache == NULL) { + ret = 0; + goto RETURN; + } + tcache_destroy(tcache); + TCACHE_SET(NULL); + + ret = 0; +RETURN: + return (ret); +} +#endif + +static int +thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + unsigned newind, oldind; + + newind = oldind = choose_arena()->ind; + WRITE(newind, unsigned); + READ(oldind, unsigned); + if (newind != oldind) { + arena_t *arena; + + if (newind >= narenas) { + /* New arena index is out of range. */ + ret = EFAULT; + goto RETURN; + } + + /* Initialize arena if necessary. */ + malloc_mutex_lock(&arenas_lock); + if ((arena = arenas[newind]) == NULL) + arena = arenas_extend(newind); + arenas[oldind]->nthreads--; + arenas[newind]->nthreads++; + malloc_mutex_unlock(&arenas_lock); + if (arena == NULL) { + ret = EAGAIN; + goto RETURN; + } + + /* Set new arena association. */ + ARENA_SET(arena); +#ifdef JEMALLOC_TCACHE + { + tcache_t *tcache = TCACHE_GET(); + if (tcache != NULL) + tcache->arena = arena; + } +#endif + } + + ret = 0; +RETURN: + return (ret); +} + +#ifdef JEMALLOC_STATS +CTL_RO_NL_GEN(thread_allocated, ALLOCATED_GET(), uint64_t); +CTL_RO_NL_GEN(thread_allocatedp, ALLOCATEDP_GET(), uint64_t *); +CTL_RO_NL_GEN(thread_deallocated, DEALLOCATED_GET(), uint64_t); +CTL_RO_NL_GEN(thread_deallocatedp, DEALLOCATEDP_GET(), uint64_t *); +#endif + +/******************************************************************************/ + +#ifdef JEMALLOC_DEBUG +CTL_RO_TRUE_GEN(config_debug) +#else +CTL_RO_FALSE_GEN(config_debug) +#endif + +#ifdef JEMALLOC_DSS +CTL_RO_TRUE_GEN(config_dss) +#else +CTL_RO_FALSE_GEN(config_dss) +#endif + +#ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT +CTL_RO_TRUE_GEN(config_dynamic_page_shift) +#else +CTL_RO_FALSE_GEN(config_dynamic_page_shift) +#endif + +#ifdef JEMALLOC_FILL +CTL_RO_TRUE_GEN(config_fill) +#else +CTL_RO_FALSE_GEN(config_fill) +#endif + +#ifdef JEMALLOC_LAZY_LOCK +CTL_RO_TRUE_GEN(config_lazy_lock) +#else +CTL_RO_FALSE_GEN(config_lazy_lock) +#endif + +#ifdef JEMALLOC_PROF +CTL_RO_TRUE_GEN(config_prof) +#else +CTL_RO_FALSE_GEN(config_prof) +#endif + +#ifdef JEMALLOC_PROF_LIBGCC +CTL_RO_TRUE_GEN(config_prof_libgcc) +#else +CTL_RO_FALSE_GEN(config_prof_libgcc) +#endif + +#ifdef JEMALLOC_PROF_LIBUNWIND +CTL_RO_TRUE_GEN(config_prof_libunwind) +#else +CTL_RO_FALSE_GEN(config_prof_libunwind) +#endif + +#ifdef JEMALLOC_STATS +CTL_RO_TRUE_GEN(config_stats) +#else +CTL_RO_FALSE_GEN(config_stats) +#endif + +#ifdef JEMALLOC_SWAP +CTL_RO_TRUE_GEN(config_swap) +#else +CTL_RO_FALSE_GEN(config_swap) +#endif + +#ifdef JEMALLOC_SYSV +CTL_RO_TRUE_GEN(config_sysv) +#else +CTL_RO_FALSE_GEN(config_sysv) +#endif + +#ifdef JEMALLOC_TCACHE +CTL_RO_TRUE_GEN(config_tcache) +#else +CTL_RO_FALSE_GEN(config_tcache) +#endif + +#ifdef JEMALLOC_TINY +CTL_RO_TRUE_GEN(config_tiny) +#else +CTL_RO_FALSE_GEN(config_tiny) +#endif + +#ifdef JEMALLOC_TLS +CTL_RO_TRUE_GEN(config_tls) +#else +CTL_RO_FALSE_GEN(config_tls) +#endif + +#ifdef JEMALLOC_XMALLOC +CTL_RO_TRUE_GEN(config_xmalloc) +#else +CTL_RO_FALSE_GEN(config_xmalloc) +#endif + +/******************************************************************************/ + +CTL_RO_NL_GEN(opt_abort, opt_abort, bool) +CTL_RO_NL_GEN(opt_lg_qspace_max, opt_lg_qspace_max, size_t) +CTL_RO_NL_GEN(opt_lg_cspace_max, opt_lg_cspace_max, size_t) +CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) +CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) +CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) +CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) +#ifdef JEMALLOC_FILL +CTL_RO_NL_GEN(opt_junk, opt_junk, bool) +CTL_RO_NL_GEN(opt_zero, opt_zero, bool) +#endif +#ifdef JEMALLOC_SYSV +CTL_RO_NL_GEN(opt_sysv, opt_sysv, bool) +#endif +#ifdef JEMALLOC_XMALLOC +CTL_RO_NL_GEN(opt_xmalloc, opt_xmalloc, bool) +#endif +#ifdef JEMALLOC_TCACHE +CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) +CTL_RO_NL_GEN(opt_lg_tcache_gc_sweep, opt_lg_tcache_gc_sweep, ssize_t) +#endif +#ifdef JEMALLOC_PROF +CTL_RO_NL_GEN(opt_prof, opt_prof, bool) +CTL_RO_NL_GEN(opt_prof_prefix, opt_prof_prefix, const char *) +CTL_RO_GEN(opt_prof_active, opt_prof_active, bool) /* Mutable. */ +CTL_RO_NL_GEN(opt_lg_prof_bt_max, opt_lg_prof_bt_max, size_t) +CTL_RO_NL_GEN(opt_lg_prof_sample, opt_lg_prof_sample, size_t) +CTL_RO_NL_GEN(opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) +CTL_RO_NL_GEN(opt_prof_gdump, opt_prof_gdump, bool) +CTL_RO_NL_GEN(opt_prof_leak, opt_prof_leak, bool) +CTL_RO_NL_GEN(opt_prof_accum, opt_prof_accum, bool) +CTL_RO_NL_GEN(opt_lg_prof_tcmax, opt_lg_prof_tcmax, ssize_t) +#endif +#ifdef JEMALLOC_SWAP +CTL_RO_NL_GEN(opt_overcommit, opt_overcommit, bool) +#endif + +/******************************************************************************/ + +CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) +CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) +const ctl_node_t * +arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > nbins) + return (NULL); + return (super_arenas_bin_i_node); +} + +CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << PAGE_SHIFT), size_t) +const ctl_node_t * +arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > nlclasses) + return (NULL); + return (super_arenas_lrun_i_node); +} + +CTL_RO_NL_GEN(arenas_narenas, narenas, unsigned) + +static int +arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + unsigned nread, i; + + malloc_mutex_lock(&ctl_mtx); + READONLY(); + if (*oldlenp != narenas * sizeof(bool)) { + ret = EINVAL; + nread = (*oldlenp < narenas * sizeof(bool)) + ? (*oldlenp / sizeof(bool)) : narenas; + } else { + ret = 0; + nread = narenas; + } + + for (i = 0; i < nread; i++) + ((bool *)oldp)[i] = ctl_stats.arenas[i].initialized; + +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) +CTL_RO_NL_GEN(arenas_cacheline, CACHELINE, size_t) +CTL_RO_NL_GEN(arenas_subpage, SUBPAGE, size_t) +CTL_RO_NL_GEN(arenas_pagesize, PAGE_SIZE, size_t) +CTL_RO_NL_GEN(arenas_chunksize, chunksize, size_t) +#ifdef JEMALLOC_TINY +CTL_RO_NL_GEN(arenas_tspace_min, (1U << LG_TINY_MIN), size_t) +CTL_RO_NL_GEN(arenas_tspace_max, (qspace_min >> 1), size_t) +#endif +CTL_RO_NL_GEN(arenas_qspace_min, qspace_min, size_t) +CTL_RO_NL_GEN(arenas_qspace_max, qspace_max, size_t) +CTL_RO_NL_GEN(arenas_cspace_min, cspace_min, size_t) +CTL_RO_NL_GEN(arenas_cspace_max, cspace_max, size_t) +CTL_RO_NL_GEN(arenas_sspace_min, sspace_min, size_t) +CTL_RO_NL_GEN(arenas_sspace_max, sspace_max, size_t) +#ifdef JEMALLOC_TCACHE +CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) +#endif +CTL_RO_NL_GEN(arenas_ntbins, ntbins, unsigned) +CTL_RO_NL_GEN(arenas_nqbins, nqbins, unsigned) +CTL_RO_NL_GEN(arenas_ncbins, ncbins, unsigned) +CTL_RO_NL_GEN(arenas_nsbins, nsbins, unsigned) +CTL_RO_NL_GEN(arenas_nbins, nbins, unsigned) +#ifdef JEMALLOC_TCACHE +CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) +#endif +CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) + +static int +arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + unsigned arena; + + WRITEONLY(); + arena = UINT_MAX; + WRITE(arena, unsigned); + if (newp != NULL && arena >= narenas) { + ret = EFAULT; + goto RETURN; + } else { + arena_t *tarenas[narenas]; + + malloc_mutex_lock(&arenas_lock); + memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); + malloc_mutex_unlock(&arenas_lock); + + if (arena == UINT_MAX) { + unsigned i; + for (i = 0; i < narenas; i++) { + if (tarenas[i] != NULL) + arena_purge_all(tarenas[i]); + } + } else { + assert(arena < narenas); + if (tarenas[arena] != NULL) + arena_purge_all(tarenas[arena]); + } + } + + ret = 0; +RETURN: + return (ret); +} + +/******************************************************************************/ + +#ifdef JEMALLOC_PROF +static int +prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + bool oldval; + + malloc_mutex_lock(&ctl_mtx); /* Protect opt_prof_active. */ + oldval = opt_prof_active; + if (newp != NULL) { + /* + * The memory barriers will tend to make opt_prof_active + * propagate faster on systems with weak memory ordering. + */ + mb_write(); + WRITE(opt_prof_active, bool); + mb_write(); + } + READ(oldval, bool); + + ret = 0; +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +static int +prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + const char *filename = NULL; + + WRITEONLY(); + WRITE(filename, const char *); + + if (prof_mdump(filename)) { + ret = EFAULT; + goto RETURN; + } + + ret = 0; +RETURN: + return (ret); +} + +CTL_RO_NL_GEN(prof_interval, prof_interval, uint64_t) +#endif + +/******************************************************************************/ + +#ifdef JEMALLOC_STATS +CTL_RO_GEN(stats_chunks_current, ctl_stats.chunks.current, size_t) +CTL_RO_GEN(stats_chunks_total, ctl_stats.chunks.total, uint64_t) +CTL_RO_GEN(stats_chunks_high, ctl_stats.chunks.high, size_t) +CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t) +CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t) +CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t) +CTL_RO_GEN(stats_arenas_i_small_allocated, + ctl_stats.arenas[mib[2]].allocated_small, size_t) +CTL_RO_GEN(stats_arenas_i_small_nmalloc, + ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t) +CTL_RO_GEN(stats_arenas_i_small_ndalloc, + ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) +CTL_RO_GEN(stats_arenas_i_small_nrequests, + ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) +CTL_RO_GEN(stats_arenas_i_large_allocated, + ctl_stats.arenas[mib[2]].astats.allocated_large, size_t) +CTL_RO_GEN(stats_arenas_i_large_nmalloc, + ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) +CTL_RO_GEN(stats_arenas_i_large_ndalloc, + ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) +CTL_RO_GEN(stats_arenas_i_large_nrequests, + ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t) + +CTL_RO_GEN(stats_arenas_i_bins_j_allocated, + ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) +CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc, + ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_nrequests, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) +#ifdef JEMALLOC_TCACHE +CTL_RO_GEN(stats_arenas_i_bins_j_nfills, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_nflushes, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nflushes, uint64_t) +#endif +CTL_RO_GEN(stats_arenas_i_bins_j_nruns, + ctl_stats.arenas[mib[2]].bstats[mib[4]].nruns, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_nreruns, + ctl_stats.arenas[mib[2]].bstats[mib[4]].reruns, uint64_t) +CTL_RO_GEN(stats_arenas_i_bins_j_highruns, + ctl_stats.arenas[mib[2]].bstats[mib[4]].highruns, size_t) +CTL_RO_GEN(stats_arenas_i_bins_j_curruns, + ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) + +const ctl_node_t * +stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) +{ + + if (j > nbins) + return (NULL); + return (super_stats_arenas_i_bins_j_node); +} + +CTL_RO_GEN(stats_arenas_i_lruns_j_nmalloc, + ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t) +CTL_RO_GEN(stats_arenas_i_lruns_j_ndalloc, + ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t) +CTL_RO_GEN(stats_arenas_i_lruns_j_nrequests, + ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t) +CTL_RO_GEN(stats_arenas_i_lruns_j_curruns, + ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) +CTL_RO_GEN(stats_arenas_i_lruns_j_highruns, + ctl_stats.arenas[mib[2]].lstats[mib[4]].highruns, size_t) + +const ctl_node_t * +stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) +{ + + if (j > nlclasses) + return (NULL); + return (super_stats_arenas_i_lruns_j_node); +} + +#endif +CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) +CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) +CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) +#ifdef JEMALLOC_STATS +CTL_RO_GEN(stats_arenas_i_mapped, ctl_stats.arenas[mib[2]].astats.mapped, + size_t) +CTL_RO_GEN(stats_arenas_i_npurge, ctl_stats.arenas[mib[2]].astats.npurge, + uint64_t) +CTL_RO_GEN(stats_arenas_i_nmadvise, ctl_stats.arenas[mib[2]].astats.nmadvise, + uint64_t) +CTL_RO_GEN(stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, + uint64_t) +#endif + +const ctl_node_t * +stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) +{ + const ctl_node_t * ret; + + malloc_mutex_lock(&ctl_mtx); + if (ctl_stats.arenas[i].initialized == false) { + ret = NULL; + goto RETURN; + } + + ret = super_stats_arenas_i_node; +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +#ifdef JEMALLOC_STATS +CTL_RO_GEN(stats_cactive, &stats_cactive, size_t *) +CTL_RO_GEN(stats_allocated, ctl_stats.allocated, size_t) +CTL_RO_GEN(stats_active, ctl_stats.active, size_t) +CTL_RO_GEN(stats_mapped, ctl_stats.mapped, size_t) +#endif + +/******************************************************************************/ + +#ifdef JEMALLOC_SWAP +# ifdef JEMALLOC_STATS +CTL_RO_GEN(swap_avail, ctl_stats.swap_avail, size_t) +# endif + +static int +swap_prezeroed_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + + malloc_mutex_lock(&ctl_mtx); + if (swap_enabled) { + READONLY(); + } else { + /* + * swap_prezeroed isn't actually used by the swap code until it + * is set during a successful chunk_swap_enabled() call. We + * use it here to store the value that we'll pass to + * chunk_swap_enable() in a swap.fds mallctl(). This is not + * very clean, but the obvious alternatives are even worse. + */ + WRITE(swap_prezeroed, bool); + } + + READ(swap_prezeroed, bool); + + ret = 0; +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +CTL_RO_GEN(swap_nfds, swap_nfds, size_t) + +static int +swap_fds_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + + malloc_mutex_lock(&ctl_mtx); + if (swap_enabled) { + READONLY(); + } else if (newp != NULL) { + size_t nfds = newlen / sizeof(int); + + { + int fds[nfds]; + + memcpy(fds, newp, nfds * sizeof(int)); + if (chunk_swap_enable(fds, nfds, swap_prezeroed)) { + ret = EFAULT; + goto RETURN; + } + } + } + + if (oldp != NULL && oldlenp != NULL) { + if (*oldlenp != swap_nfds * sizeof(int)) { + size_t copylen = (swap_nfds * sizeof(int) <= *oldlenp) + ? swap_nfds * sizeof(int) : *oldlenp; + + memcpy(oldp, swap_fds, copylen); + ret = EINVAL; + goto RETURN; + } else + memcpy(oldp, swap_fds, *oldlenp); + } + + ret = 0; +RETURN: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} +#endif diff --git a/deps/jemalloc.orig/src/extent.c b/deps/jemalloc.orig/src/extent.c new file mode 100644 index 00000000000..3c04d3aa5d1 --- /dev/null +++ b/deps/jemalloc.orig/src/extent.c @@ -0,0 +1,41 @@ +#define JEMALLOC_EXTENT_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ + +#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) +static inline int +extent_szad_comp(extent_node_t *a, extent_node_t *b) +{ + int ret; + size_t a_size = a->size; + size_t b_size = b->size; + + ret = (a_size > b_size) - (a_size < b_size); + if (ret == 0) { + uintptr_t a_addr = (uintptr_t)a->addr; + uintptr_t b_addr = (uintptr_t)b->addr; + + ret = (a_addr > b_addr) - (a_addr < b_addr); + } + + return (ret); +} + +/* Generate red-black tree functions. */ +rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad, + extent_szad_comp) +#endif + +static inline int +extent_ad_comp(extent_node_t *a, extent_node_t *b) +{ + uintptr_t a_addr = (uintptr_t)a->addr; + uintptr_t b_addr = (uintptr_t)b->addr; + + return ((a_addr > b_addr) - (a_addr < b_addr)); +} + +/* Generate red-black tree functions. */ +rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad, + extent_ad_comp) diff --git a/deps/jemalloc.orig/src/hash.c b/deps/jemalloc.orig/src/hash.c new file mode 100644 index 00000000000..cfa4da0275c --- /dev/null +++ b/deps/jemalloc.orig/src/hash.c @@ -0,0 +1,2 @@ +#define JEMALLOC_HASH_C_ +#include "jemalloc/internal/jemalloc_internal.h" diff --git a/deps/jemalloc.orig/src/huge.c b/deps/jemalloc.orig/src/huge.c new file mode 100644 index 00000000000..a4f9b054ed5 --- /dev/null +++ b/deps/jemalloc.orig/src/huge.c @@ -0,0 +1,386 @@ +#define JEMALLOC_HUGE_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +#ifdef JEMALLOC_STATS +uint64_t huge_nmalloc; +uint64_t huge_ndalloc; +size_t huge_allocated; +#endif + +malloc_mutex_t huge_mtx; + +/******************************************************************************/ + +/* Tree of chunks that are stand-alone huge allocations. */ +static extent_tree_t huge; + +void * +huge_malloc(size_t size, bool zero) +{ + void *ret; + size_t csize; + extent_node_t *node; + + /* Allocate one or more contiguous chunks for this request. */ + + csize = CHUNK_CEILING(size); + if (csize == 0) { + /* size is large enough to cause size_t wrap-around. */ + return (NULL); + } + + /* Allocate an extent node with which to track the chunk. */ + node = base_node_alloc(); + if (node == NULL) + return (NULL); + + ret = chunk_alloc(csize, false, &zero); + if (ret == NULL) { + base_node_dealloc(node); + return (NULL); + } + + /* Insert node into huge. */ + node->addr = ret; + node->size = csize; + + malloc_mutex_lock(&huge_mtx); + extent_tree_ad_insert(&huge, node); +#ifdef JEMALLOC_STATS + stats_cactive_add(csize); + huge_nmalloc++; + huge_allocated += csize; +#endif + malloc_mutex_unlock(&huge_mtx); + +#ifdef JEMALLOC_FILL + if (zero == false) { + if (opt_junk) + memset(ret, 0xa5, csize); + else if (opt_zero) + memset(ret, 0, csize); + } +#endif + + return (ret); +} + +/* Only handles large allocations that require more than chunk alignment. */ +void * +huge_palloc(size_t size, size_t alignment, bool zero) +{ + void *ret; + size_t alloc_size, chunk_size, offset; + extent_node_t *node; + + /* + * This allocation requires alignment that is even larger than chunk + * alignment. This means that huge_malloc() isn't good enough. + * + * Allocate almost twice as many chunks as are demanded by the size or + * alignment, in order to assure the alignment can be achieved, then + * unmap leading and trailing chunks. + */ + assert(alignment > chunksize); + + chunk_size = CHUNK_CEILING(size); + + if (size >= alignment) + alloc_size = chunk_size + alignment - chunksize; + else + alloc_size = (alignment << 1) - chunksize; + + /* Allocate an extent node with which to track the chunk. */ + node = base_node_alloc(); + if (node == NULL) + return (NULL); + + ret = chunk_alloc(alloc_size, false, &zero); + if (ret == NULL) { + base_node_dealloc(node); + return (NULL); + } + + offset = (uintptr_t)ret & (alignment - 1); + assert((offset & chunksize_mask) == 0); + assert(offset < alloc_size); + if (offset == 0) { + /* Trim trailing space. */ + chunk_dealloc((void *)((uintptr_t)ret + chunk_size), alloc_size + - chunk_size, true); + } else { + size_t trailsize; + + /* Trim leading space. */ + chunk_dealloc(ret, alignment - offset, true); + + ret = (void *)((uintptr_t)ret + (alignment - offset)); + + trailsize = alloc_size - (alignment - offset) - chunk_size; + if (trailsize != 0) { + /* Trim trailing space. */ + assert(trailsize < alloc_size); + chunk_dealloc((void *)((uintptr_t)ret + chunk_size), + trailsize, true); + } + } + + /* Insert node into huge. */ + node->addr = ret; + node->size = chunk_size; + + malloc_mutex_lock(&huge_mtx); + extent_tree_ad_insert(&huge, node); +#ifdef JEMALLOC_STATS + stats_cactive_add(chunk_size); + huge_nmalloc++; + huge_allocated += chunk_size; +#endif + malloc_mutex_unlock(&huge_mtx); + +#ifdef JEMALLOC_FILL + if (zero == false) { + if (opt_junk) + memset(ret, 0xa5, chunk_size); + else if (opt_zero) + memset(ret, 0, chunk_size); + } +#endif + + return (ret); +} + +void * +huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) +{ + + /* + * Avoid moving the allocation if the size class can be left the same. + */ + if (oldsize > arena_maxclass + && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) + && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { + assert(CHUNK_CEILING(oldsize) == oldsize); +#ifdef JEMALLOC_FILL + if (opt_junk && size < oldsize) { + memset((void *)((uintptr_t)ptr + size), 0x5a, + oldsize - size); + } +#endif + return (ptr); + } + + /* Reallocation would require a move. */ + return (NULL); +} + +void * +huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero) +{ + void *ret; + size_t copysize; + + /* Try to avoid moving the allocation. */ + ret = huge_ralloc_no_move(ptr, oldsize, size, extra); + if (ret != NULL) + return (ret); + + /* + * size and oldsize are different enough that we need to use a + * different size class. In that case, fall back to allocating new + * space and copying. + */ + if (alignment > chunksize) + ret = huge_palloc(size + extra, alignment, zero); + else + ret = huge_malloc(size + extra, zero); + + if (ret == NULL) { + if (extra == 0) + return (NULL); + /* Try again, this time without extra. */ + if (alignment > chunksize) + ret = huge_palloc(size, alignment, zero); + else + ret = huge_malloc(size, zero); + + if (ret == NULL) + return (NULL); + } + + /* + * Copy at most size bytes (not size+extra), since the caller has no + * expectation that the extra bytes will be reliably preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + + /* + * Use mremap(2) if this is a huge-->huge reallocation, and neither the + * source nor the destination are in swap or dss. + */ +#ifdef JEMALLOC_MREMAP_FIXED + if (oldsize >= chunksize +# ifdef JEMALLOC_SWAP + && (swap_enabled == false || (chunk_in_swap(ptr) == false && + chunk_in_swap(ret) == false)) +# endif +# ifdef JEMALLOC_DSS + && chunk_in_dss(ptr) == false && chunk_in_dss(ret) == false +# endif + ) { + size_t newsize = huge_salloc(ret); + + /* + * Remove ptr from the tree of huge allocations before + * performing the remap operation, in order to avoid the + * possibility of another thread acquiring that mapping before + * this one removes it from the tree. + */ + huge_dalloc(ptr, false); + if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED, + ret) == MAP_FAILED) { + /* + * Assuming no chunk management bugs in the allocator, + * the only documented way an error can occur here is + * if the application changed the map type for a + * portion of the old allocation. This is firmly in + * undefined behavior territory, so write a diagnostic + * message, and optionally abort. + */ + char buf[BUFERROR_BUF]; + + buferror(errno, buf, sizeof(buf)); + malloc_write(": Error in mremap(): "); + malloc_write(buf); + malloc_write("\n"); + if (opt_abort) + abort(); + memcpy(ret, ptr, copysize); + chunk_dealloc_mmap(ptr, oldsize); + } + } else +#endif + { + memcpy(ret, ptr, copysize); + idalloc(ptr); + } + return (ret); +} + +void +huge_dalloc(void *ptr, bool unmap) +{ + extent_node_t *node, key; + + malloc_mutex_lock(&huge_mtx); + + /* Extract from tree of huge allocations. */ + key.addr = ptr; + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + assert(node->addr == ptr); + extent_tree_ad_remove(&huge, node); + +#ifdef JEMALLOC_STATS + stats_cactive_sub(node->size); + huge_ndalloc++; + huge_allocated -= node->size; +#endif + + malloc_mutex_unlock(&huge_mtx); + + if (unmap) { + /* Unmap chunk. */ +#ifdef JEMALLOC_FILL +#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) + if (opt_junk) + memset(node->addr, 0x5a, node->size); +#endif +#endif + } + + chunk_dealloc(node->addr, node->size, unmap); + + base_node_dealloc(node); +} + +size_t +huge_salloc(const void *ptr) +{ + size_t ret; + extent_node_t *node, key; + + malloc_mutex_lock(&huge_mtx); + + /* Extract from tree of huge allocations. */ + key.addr = __DECONST(void *, ptr); + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + + ret = node->size; + + malloc_mutex_unlock(&huge_mtx); + + return (ret); +} + +#ifdef JEMALLOC_PROF +prof_ctx_t * +huge_prof_ctx_get(const void *ptr) +{ + prof_ctx_t *ret; + extent_node_t *node, key; + + malloc_mutex_lock(&huge_mtx); + + /* Extract from tree of huge allocations. */ + key.addr = __DECONST(void *, ptr); + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + + ret = node->prof_ctx; + + malloc_mutex_unlock(&huge_mtx); + + return (ret); +} + +void +huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +{ + extent_node_t *node, key; + + malloc_mutex_lock(&huge_mtx); + + /* Extract from tree of huge allocations. */ + key.addr = __DECONST(void *, ptr); + node = extent_tree_ad_search(&huge, &key); + assert(node != NULL); + + node->prof_ctx = ctx; + + malloc_mutex_unlock(&huge_mtx); +} +#endif + +bool +huge_boot(void) +{ + + /* Initialize chunks data. */ + if (malloc_mutex_init(&huge_mtx)) + return (true); + extent_tree_ad_new(&huge); + +#ifdef JEMALLOC_STATS + huge_nmalloc = 0; + huge_ndalloc = 0; + huge_allocated = 0; +#endif + + return (false); +} diff --git a/deps/jemalloc.orig/src/jemalloc.c b/deps/jemalloc.orig/src/jemalloc.c new file mode 100644 index 00000000000..a161c2e26e1 --- /dev/null +++ b/deps/jemalloc.orig/src/jemalloc.c @@ -0,0 +1,1881 @@ +#define JEMALLOC_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +malloc_mutex_t arenas_lock; +arena_t **arenas; +unsigned narenas; + +pthread_key_t arenas_tsd; +#ifndef NO_TLS +__thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); +#endif + +#ifdef JEMALLOC_STATS +# ifndef NO_TLS +__thread thread_allocated_t thread_allocated_tls; +# else +pthread_key_t thread_allocated_tsd; +# endif +#endif + +/* Set to true once the allocator has been initialized. */ +static bool malloc_initialized = false; + +/* Used to let the initializing thread recursively allocate. */ +static pthread_t malloc_initializer = (unsigned long)0; + +/* Used to avoid initialization races. */ +static malloc_mutex_t init_lock = +#ifdef JEMALLOC_OSSPIN + 0 +#else + MALLOC_MUTEX_INITIALIZER +#endif + ; + +#ifdef DYNAMIC_PAGE_SHIFT +size_t pagesize; +size_t pagesize_mask; +size_t lg_pagesize; +#endif + +unsigned ncpus; + +/* Runtime configuration options. */ +const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default")); +#ifdef JEMALLOC_DEBUG +bool opt_abort = true; +# ifdef JEMALLOC_FILL +bool opt_junk = true; +# endif +#else +bool opt_abort = false; +# ifdef JEMALLOC_FILL +bool opt_junk = false; +# endif +#endif +#ifdef JEMALLOC_SYSV +bool opt_sysv = false; +#endif +#ifdef JEMALLOC_XMALLOC +bool opt_xmalloc = false; +#endif +#ifdef JEMALLOC_FILL +bool opt_zero = false; +#endif +size_t opt_narenas = 0; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void wrtmessage(void *cbopaque, const char *s); +static void stats_print_atexit(void); +static unsigned malloc_ncpus(void); +static void arenas_cleanup(void *arg); +#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) +static void thread_allocated_cleanup(void *arg); +#endif +static bool malloc_conf_next(char const **opts_p, char const **k_p, + size_t *klen_p, char const **v_p, size_t *vlen_p); +static void malloc_conf_error(const char *msg, const char *k, size_t klen, + const char *v, size_t vlen); +static void malloc_conf_init(void); +static bool malloc_init_hard(void); +static int imemalign(void **memptr, size_t alignment, size_t size); + +/******************************************************************************/ +/* malloc_message() setup. */ + +#ifdef JEMALLOC_HAVE_ATTR +JEMALLOC_ATTR(visibility("hidden")) +#else +static +#endif +void +wrtmessage(void *cbopaque, const char *s) +{ +#ifdef JEMALLOC_CC_SILENCE + int result = +#endif + write(STDERR_FILENO, s, strlen(s)); +#ifdef JEMALLOC_CC_SILENCE + if (result < 0) + result = errno; +#endif +} + +void (*JEMALLOC_P(malloc_message))(void *, const char *s) + JEMALLOC_ATTR(visibility("default")) = wrtmessage; + +/******************************************************************************/ +/* + * Begin miscellaneous support functions. + */ + +/* Create a new arena and insert it into the arenas array at index ind. */ +arena_t * +arenas_extend(unsigned ind) +{ + arena_t *ret; + + /* Allocate enough space for trailing bins. */ + ret = (arena_t *)base_alloc(offsetof(arena_t, bins) + + (sizeof(arena_bin_t) * nbins)); + if (ret != NULL && arena_new(ret, ind) == false) { + arenas[ind] = ret; + return (ret); + } + /* Only reached if there is an OOM error. */ + + /* + * OOM here is quite inconvenient to propagate, since dealing with it + * would require a check for failure in the fast path. Instead, punt + * by using arenas[0]. In practice, this is an extremely unlikely + * failure. + */ + malloc_write(": Error initializing arena\n"); + if (opt_abort) + abort(); + + return (arenas[0]); +} + +/* + * Choose an arena based on a per-thread value (slow-path code only, called + * only by choose_arena()). + */ +arena_t * +choose_arena_hard(void) +{ + arena_t *ret; + + if (narenas > 1) { + unsigned i, choose, first_null; + + choose = 0; + first_null = narenas; + malloc_mutex_lock(&arenas_lock); + assert(arenas[0] != NULL); + for (i = 1; i < narenas; i++) { + if (arenas[i] != NULL) { + /* + * Choose the first arena that has the lowest + * number of threads assigned to it. + */ + if (arenas[i]->nthreads < + arenas[choose]->nthreads) + choose = i; + } else if (first_null == narenas) { + /* + * Record the index of the first uninitialized + * arena, in case all extant arenas are in use. + * + * NB: It is possible for there to be + * discontinuities in terms of initialized + * versus uninitialized arenas, due to the + * "thread.arena" mallctl. + */ + first_null = i; + } + } + + if (arenas[choose] == 0 || first_null == narenas) { + /* + * Use an unloaded arena, or the least loaded arena if + * all arenas are already initialized. + */ + ret = arenas[choose]; + } else { + /* Initialize a new arena. */ + ret = arenas_extend(first_null); + } + ret->nthreads++; + malloc_mutex_unlock(&arenas_lock); + } else { + ret = arenas[0]; + malloc_mutex_lock(&arenas_lock); + ret->nthreads++; + malloc_mutex_unlock(&arenas_lock); + } + + ARENA_SET(ret); + + return (ret); +} + +/* + * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so + * provide a wrapper. + */ +int +buferror(int errnum, char *buf, size_t buflen) +{ +#ifdef _GNU_SOURCE + char *b = strerror_r(errno, buf, buflen); + if (b != buf) { + strncpy(buf, b, buflen); + buf[buflen-1] = '\0'; + } + return (0); +#else + return (strerror_r(errno, buf, buflen)); +#endif +} + +static void +stats_print_atexit(void) +{ + +#if (defined(JEMALLOC_TCACHE) && defined(JEMALLOC_STATS)) + unsigned i; + + /* + * Merge stats from extant threads. This is racy, since individual + * threads do not lock when recording tcache stats events. As a + * consequence, the final stats may be slightly out of date by the time + * they are reported, if other threads continue to allocate. + */ + for (i = 0; i < narenas; i++) { + arena_t *arena = arenas[i]; + if (arena != NULL) { + tcache_t *tcache; + + /* + * tcache_stats_merge() locks bins, so if any code is + * introduced that acquires both arena and bin locks in + * the opposite order, deadlocks may result. + */ + malloc_mutex_lock(&arena->lock); + ql_foreach(tcache, &arena->tcache_ql, link) { + tcache_stats_merge(tcache, arena); + } + malloc_mutex_unlock(&arena->lock); + } + } +#endif + JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL); +} + +#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) +thread_allocated_t * +thread_allocated_get_hard(void) +{ + thread_allocated_t *thread_allocated = (thread_allocated_t *) + imalloc(sizeof(thread_allocated_t)); + if (thread_allocated == NULL) { + static thread_allocated_t static_thread_allocated = {0, 0}; + malloc_write(": Error allocating TSD;" + " mallctl(\"thread.{de,}allocated[p]\", ...)" + " will be inaccurate\n"); + if (opt_abort) + abort(); + return (&static_thread_allocated); + } + pthread_setspecific(thread_allocated_tsd, thread_allocated); + thread_allocated->allocated = 0; + thread_allocated->deallocated = 0; + return (thread_allocated); +} +#endif + +/* + * End miscellaneous support functions. + */ +/******************************************************************************/ +/* + * Begin initialization functions. + */ + +static unsigned +malloc_ncpus(void) +{ + unsigned ret; + long result; + + result = sysconf(_SC_NPROCESSORS_ONLN); + if (result == -1) { + /* Error. */ + ret = 1; + } + ret = (unsigned)result; + + return (ret); +} + +static void +arenas_cleanup(void *arg) +{ + arena_t *arena = (arena_t *)arg; + + malloc_mutex_lock(&arenas_lock); + arena->nthreads--; + malloc_mutex_unlock(&arenas_lock); +} + +#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) +static void +thread_allocated_cleanup(void *arg) +{ + uint64_t *allocated = (uint64_t *)arg; + + if (allocated != NULL) + idalloc(allocated); +} +#endif + +/* + * FreeBSD's pthreads implementation calls malloc(3), so the malloc + * implementation has to take pains to avoid infinite recursion during + * initialization. + */ +static inline bool +malloc_init(void) +{ + + if (malloc_initialized == false) + return (malloc_init_hard()); + + return (false); +} + +static bool +malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, + char const **v_p, size_t *vlen_p) +{ + bool accept; + const char *opts = *opts_p; + + *k_p = opts; + + for (accept = false; accept == false;) { + switch (*opts) { + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': case 'G': case 'H': case 'I': case 'J': + case 'K': case 'L': case 'M': case 'N': case 'O': + case 'P': case 'Q': case 'R': case 'S': case 'T': + case 'U': case 'V': case 'W': case 'X': case 'Y': + case 'Z': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': case 'g': case 'h': case 'i': case 'j': + case 'k': case 'l': case 'm': case 'n': case 'o': + case 'p': case 'q': case 'r': case 's': case 't': + case 'u': case 'v': case 'w': case 'x': case 'y': + case 'z': + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case '_': + opts++; + break; + case ':': + opts++; + *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; + *v_p = opts; + accept = true; + break; + case '\0': + if (opts != *opts_p) { + malloc_write(": Conf string " + "ends with key\n"); + } + return (true); + default: + malloc_write(": Malformed conf " + "string\n"); + return (true); + } + } + + for (accept = false; accept == false;) { + switch (*opts) { + case ',': + opts++; + /* + * Look ahead one character here, because the + * next time this function is called, it will + * assume that end of input has been cleanly + * reached if no input remains, but we have + * optimistically already consumed the comma if + * one exists. + */ + if (*opts == '\0') { + malloc_write(": Conf string " + "ends with comma\n"); + } + *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; + accept = true; + break; + case '\0': + *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; + accept = true; + break; + default: + opts++; + break; + } + } + + *opts_p = opts; + return (false); +} + +static void +malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, + size_t vlen) +{ + char buf[PATH_MAX + 1]; + + malloc_write(": "); + malloc_write(msg); + malloc_write(": "); + memcpy(buf, k, klen); + memcpy(&buf[klen], ":", 1); + memcpy(&buf[klen+1], v, vlen); + buf[klen+1+vlen] = '\0'; + malloc_write(buf); + malloc_write("\n"); +} + +static void +malloc_conf_init(void) +{ + unsigned i; + char buf[PATH_MAX + 1]; + const char *opts, *k, *v; + size_t klen, vlen; + + for (i = 0; i < 3; i++) { + /* Get runtime configuration. */ + switch (i) { + case 0: + if (JEMALLOC_P(malloc_conf) != NULL) { + /* + * Use options that were compiled into the + * program. + */ + opts = JEMALLOC_P(malloc_conf); + } else { + /* No configuration specified. */ + buf[0] = '\0'; + opts = buf; + } + break; + case 1: { + int linklen; + const char *linkname = +#ifdef JEMALLOC_PREFIX + "/etc/"JEMALLOC_PREFIX"malloc.conf" +#else + "/etc/malloc.conf" +#endif + ; + + if ((linklen = readlink(linkname, buf, + sizeof(buf) - 1)) != -1) { + /* + * Use the contents of the "/etc/malloc.conf" + * symbolic link's name. + */ + buf[linklen] = '\0'; + opts = buf; + } else { + /* No configuration specified. */ + buf[0] = '\0'; + opts = buf; + } + break; + } + case 2: { + const char *envname = +#ifdef JEMALLOC_PREFIX + JEMALLOC_CPREFIX"MALLOC_CONF" +#else + "MALLOC_CONF" +#endif + ; + + if ((opts = getenv(envname)) != NULL) { + /* + * Do nothing; opts is already initialized to + * the value of the MALLOC_CONF environment + * variable. + */ + } else { + /* No configuration specified. */ + buf[0] = '\0'; + opts = buf; + } + break; + } + default: + /* NOTREACHED */ + assert(false); + buf[0] = '\0'; + opts = buf; + } + + while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, + &vlen) == false) { +#define CONF_HANDLE_BOOL(n) \ + if (sizeof(#n)-1 == klen && strncmp(#n, k, \ + klen) == 0) { \ + if (strncmp("true", v, vlen) == 0 && \ + vlen == sizeof("true")-1) \ + opt_##n = true; \ + else if (strncmp("false", v, vlen) == \ + 0 && vlen == sizeof("false")-1) \ + opt_##n = false; \ + else { \ + malloc_conf_error( \ + "Invalid conf value", \ + k, klen, v, vlen); \ + } \ + continue; \ + } +#define CONF_HANDLE_SIZE_T(n, min, max) \ + if (sizeof(#n)-1 == klen && strncmp(#n, k, \ + klen) == 0) { \ + unsigned long ul; \ + char *end; \ + \ + errno = 0; \ + ul = strtoul(v, &end, 0); \ + if (errno != 0 || (uintptr_t)end - \ + (uintptr_t)v != vlen) { \ + malloc_conf_error( \ + "Invalid conf value", \ + k, klen, v, vlen); \ + } else if (ul < min || ul > max) { \ + malloc_conf_error( \ + "Out-of-range conf value", \ + k, klen, v, vlen); \ + } else \ + opt_##n = ul; \ + continue; \ + } +#define CONF_HANDLE_SSIZE_T(n, min, max) \ + if (sizeof(#n)-1 == klen && strncmp(#n, k, \ + klen) == 0) { \ + long l; \ + char *end; \ + \ + errno = 0; \ + l = strtol(v, &end, 0); \ + if (errno != 0 || (uintptr_t)end - \ + (uintptr_t)v != vlen) { \ + malloc_conf_error( \ + "Invalid conf value", \ + k, klen, v, vlen); \ + } else if (l < (ssize_t)min || l > \ + (ssize_t)max) { \ + malloc_conf_error( \ + "Out-of-range conf value", \ + k, klen, v, vlen); \ + } else \ + opt_##n = l; \ + continue; \ + } +#define CONF_HANDLE_CHAR_P(n, d) \ + if (sizeof(#n)-1 == klen && strncmp(#n, k, \ + klen) == 0) { \ + size_t cpylen = (vlen <= \ + sizeof(opt_##n)-1) ? vlen : \ + sizeof(opt_##n)-1; \ + strncpy(opt_##n, v, cpylen); \ + opt_##n[cpylen] = '\0'; \ + continue; \ + } + + CONF_HANDLE_BOOL(abort) + CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM, + PAGE_SHIFT-1) + CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM, + PAGE_SHIFT-1) + /* + * Chunks always require at least one * header page, + * plus one data page. + */ + CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1, + (sizeof(size_t) << 3) - 1) + CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX) + CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1, + (sizeof(size_t) << 3) - 1) + CONF_HANDLE_BOOL(stats_print) +#ifdef JEMALLOC_FILL + CONF_HANDLE_BOOL(junk) + CONF_HANDLE_BOOL(zero) +#endif +#ifdef JEMALLOC_SYSV + CONF_HANDLE_BOOL(sysv) +#endif +#ifdef JEMALLOC_XMALLOC + CONF_HANDLE_BOOL(xmalloc) +#endif +#ifdef JEMALLOC_TCACHE + CONF_HANDLE_BOOL(tcache) + CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1, + (sizeof(size_t) << 3) - 1) + CONF_HANDLE_SSIZE_T(lg_tcache_max, -1, + (sizeof(size_t) << 3) - 1) +#endif +#ifdef JEMALLOC_PROF + CONF_HANDLE_BOOL(prof) + CONF_HANDLE_CHAR_P(prof_prefix, "jeprof") + CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0, LG_PROF_BT_MAX) + CONF_HANDLE_BOOL(prof_active) + CONF_HANDLE_SSIZE_T(lg_prof_sample, 0, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(prof_accum) + CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1, + (sizeof(size_t) << 3) - 1) + CONF_HANDLE_SSIZE_T(lg_prof_interval, -1, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(prof_gdump) + CONF_HANDLE_BOOL(prof_leak) +#endif +#ifdef JEMALLOC_SWAP + CONF_HANDLE_BOOL(overcommit) +#endif + malloc_conf_error("Invalid conf pair", k, klen, v, + vlen); +#undef CONF_HANDLE_BOOL +#undef CONF_HANDLE_SIZE_T +#undef CONF_HANDLE_SSIZE_T +#undef CONF_HANDLE_CHAR_P + } + + /* Validate configuration of options that are inter-related. */ + if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) { + malloc_write(": Invalid lg_[qc]space_max " + "relationship; restoring defaults\n"); + opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; + opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; + } + } +} + +static bool +malloc_init_hard(void) +{ + arena_t *init_arenas[1]; + + malloc_mutex_lock(&init_lock); + if (malloc_initialized || malloc_initializer == pthread_self()) { + /* + * Another thread initialized the allocator before this one + * acquired init_lock, or this thread is the initializing + * thread, and it is recursively allocating. + */ + malloc_mutex_unlock(&init_lock); + return (false); + } + if (malloc_initializer != (unsigned long)0) { + /* Busy-wait until the initializing thread completes. */ + do { + malloc_mutex_unlock(&init_lock); + CPU_SPINWAIT; + malloc_mutex_lock(&init_lock); + } while (malloc_initialized == false); + malloc_mutex_unlock(&init_lock); + return (false); + } + +#ifdef DYNAMIC_PAGE_SHIFT + /* Get page size. */ + { + long result; + + result = sysconf(_SC_PAGESIZE); + assert(result != -1); + pagesize = (size_t)result; + + /* + * We assume that pagesize is a power of 2 when calculating + * pagesize_mask and lg_pagesize. + */ + assert(((result - 1) & result) == 0); + pagesize_mask = result - 1; + lg_pagesize = ffs((int)result) - 1; + } +#endif + +#ifdef JEMALLOC_PROF + prof_boot0(); +#endif + + malloc_conf_init(); + + /* Register fork handlers. */ + if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, + jemalloc_postfork) != 0) { + malloc_write(": Error in pthread_atfork()\n"); + if (opt_abort) + abort(); + } + + if (ctl_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (opt_stats_print) { + /* Print statistics at exit. */ + if (atexit(stats_print_atexit) != 0) { + malloc_write(": Error in atexit()\n"); + if (opt_abort) + abort(); + } + } + + if (chunk_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (base_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + +#ifdef JEMALLOC_PROF + prof_boot1(); +#endif + + if (arena_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + +#ifdef JEMALLOC_TCACHE + if (tcache_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } +#endif + + if (huge_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + +#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) + /* Initialize allocation counters before any allocations can occur. */ + if (pthread_key_create(&thread_allocated_tsd, thread_allocated_cleanup) + != 0) { + malloc_mutex_unlock(&init_lock); + return (true); + } +#endif + + if (malloc_mutex_init(&arenas_lock)) + return (true); + + if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + /* + * Create enough scaffolding to allow recursive allocation in + * malloc_ncpus(). + */ + narenas = 1; + arenas = init_arenas; + memset(arenas, 0, sizeof(arena_t *) * narenas); + + /* + * Initialize one arena here. The rest are lazily created in + * choose_arena_hard(). + */ + arenas_extend(0); + if (arenas[0] == NULL) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + /* + * Assign the initial arena to the initial thread, in order to avoid + * spurious creation of an extra arena if the application switches to + * threaded mode. + */ + ARENA_SET(arenas[0]); + arenas[0]->nthreads++; + +#ifdef JEMALLOC_PROF + if (prof_boot2()) { + malloc_mutex_unlock(&init_lock); + return (true); + } +#endif + + /* Get number of CPUs. */ + malloc_initializer = pthread_self(); + malloc_mutex_unlock(&init_lock); + ncpus = malloc_ncpus(); + malloc_mutex_lock(&init_lock); + + if (opt_narenas == 0) { + /* + * For SMP systems, create more than one arena per CPU by + * default. + */ + if (ncpus > 1) + opt_narenas = ncpus << 2; + else + opt_narenas = 1; + } + narenas = opt_narenas; + /* + * Make sure that the arenas array can be allocated. In practice, this + * limit is enough to allow the allocator to function, but the ctl + * machinery will fail to allocate memory at far lower limits. + */ + if (narenas > chunksize / sizeof(arena_t *)) { + char buf[UMAX2S_BUFSIZE]; + + narenas = chunksize / sizeof(arena_t *); + malloc_write(": Reducing narenas to limit ("); + malloc_write(u2s(narenas, 10, buf)); + malloc_write(")\n"); + } + + /* Allocate and initialize arenas. */ + arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); + if (arenas == NULL) { + malloc_mutex_unlock(&init_lock); + return (true); + } + /* + * Zero the array. In practice, this should always be pre-zeroed, + * since it was just mmap()ed, but let's be sure. + */ + memset(arenas, 0, sizeof(arena_t *) * narenas); + /* Copy the pointer to the one arena that was already initialized. */ + arenas[0] = init_arenas[0]; + +#ifdef JEMALLOC_ZONE + /* Register the custom zone. */ + malloc_zone_register(create_zone()); + + /* + * Convert the default szone to an "overlay zone" that is capable of + * deallocating szone-allocated objects, but allocating new objects + * from jemalloc. + */ + szone2ozone(malloc_default_zone()); +#endif + + malloc_initialized = true; + malloc_mutex_unlock(&init_lock); + return (false); +} + +#ifdef JEMALLOC_ZONE +JEMALLOC_ATTR(constructor) +void +jemalloc_darwin_init(void) +{ + + if (malloc_init_hard()) + abort(); +} +#endif + +/* + * End initialization functions. + */ +/******************************************************************************/ +/* + * Begin malloc(3)-compatible functions. + */ + +JEMALLOC_ATTR(malloc) +JEMALLOC_ATTR(visibility("default")) +void * +JEMALLOC_P(malloc)(size_t size) +{ + void *ret; +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + size_t usize +# ifdef JEMALLOC_CC_SILENCE + = 0 +# endif + ; +#endif +#ifdef JEMALLOC_PROF + prof_thr_cnt_t *cnt +# ifdef JEMALLOC_CC_SILENCE + = NULL +# endif + ; +#endif + + if (malloc_init()) { + ret = NULL; + goto OOM; + } + + if (size == 0) { +#ifdef JEMALLOC_SYSV + if (opt_sysv == false) +#endif + size = 1; +#ifdef JEMALLOC_SYSV + else { +# ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in malloc(): " + "invalid size 0\n"); + abort(); + } +# endif + ret = NULL; + goto RETURN; + } +#endif + } + +#ifdef JEMALLOC_PROF + if (opt_prof) { + usize = s2u(size); + PROF_ALLOC_PREP(1, usize, cnt); + if (cnt == NULL) { + ret = NULL; + goto OOM; + } + if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= + small_maxclass) { + ret = imalloc(small_maxclass+1); + if (ret != NULL) + arena_prof_promoted(ret, usize); + } else + ret = imalloc(size); + } else +#endif + { +#ifdef JEMALLOC_STATS + usize = s2u(size); +#endif + ret = imalloc(size); + } + +OOM: + if (ret == NULL) { +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in malloc(): " + "out of memory\n"); + abort(); + } +#endif + errno = ENOMEM; + } + +#ifdef JEMALLOC_SYSV +RETURN: +#endif +#ifdef JEMALLOC_PROF + if (opt_prof && ret != NULL) + prof_malloc(ret, usize, cnt); +#endif +#ifdef JEMALLOC_STATS + if (ret != NULL) { + assert(usize == isalloc(ret)); + ALLOCATED_ADD(usize, 0); + } +#endif + return (ret); +} + +JEMALLOC_ATTR(nonnull(1)) +#ifdef JEMALLOC_PROF +/* + * Avoid any uncertainty as to how many backtrace frames to ignore in + * PROF_ALLOC_PREP(). + */ +JEMALLOC_ATTR(noinline) +#endif +static int +imemalign(void **memptr, size_t alignment, size_t size) +{ + int ret; + size_t usize +#ifdef JEMALLOC_CC_SILENCE + = 0 +#endif + ; + void *result; +#ifdef JEMALLOC_PROF + prof_thr_cnt_t *cnt +# ifdef JEMALLOC_CC_SILENCE + = NULL +# endif + ; +#endif + + if (malloc_init()) + result = NULL; + else { + if (size == 0) { +#ifdef JEMALLOC_SYSV + if (opt_sysv == false) +#endif + size = 1; +#ifdef JEMALLOC_SYSV + else { +# ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in " + "posix_memalign(): invalid size " + "0\n"); + abort(); + } +# endif + result = NULL; + *memptr = NULL; + ret = 0; + goto RETURN; + } +#endif + } + + /* Make sure that alignment is a large enough power of 2. */ + if (((alignment - 1) & alignment) != 0 + || alignment < sizeof(void *)) { +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in " + "posix_memalign(): invalid alignment\n"); + abort(); + } +#endif + result = NULL; + ret = EINVAL; + goto RETURN; + } + + usize = sa2u(size, alignment, NULL); + if (usize == 0) { + result = NULL; + ret = ENOMEM; + goto RETURN; + } + +#ifdef JEMALLOC_PROF + if (opt_prof) { + PROF_ALLOC_PREP(2, usize, cnt); + if (cnt == NULL) { + result = NULL; + ret = EINVAL; + } else { + if (prof_promote && (uintptr_t)cnt != + (uintptr_t)1U && usize <= small_maxclass) { + assert(sa2u(small_maxclass+1, + alignment, NULL) != 0); + result = ipalloc(sa2u(small_maxclass+1, + alignment, NULL), alignment, false); + if (result != NULL) { + arena_prof_promoted(result, + usize); + } + } else { + result = ipalloc(usize, alignment, + false); + } + } + } else +#endif + result = ipalloc(usize, alignment, false); + } + + if (result == NULL) { +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in posix_memalign(): " + "out of memory\n"); + abort(); + } +#endif + ret = ENOMEM; + goto RETURN; + } + + *memptr = result; + ret = 0; + +RETURN: +#ifdef JEMALLOC_STATS + if (result != NULL) { + assert(usize == isalloc(result)); + ALLOCATED_ADD(usize, 0); + } +#endif +#ifdef JEMALLOC_PROF + if (opt_prof && result != NULL) + prof_malloc(result, usize, cnt); +#endif + return (ret); +} + +JEMALLOC_ATTR(nonnull(1)) +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) +{ + + return imemalign(memptr, alignment, size); +} + +JEMALLOC_ATTR(malloc) +JEMALLOC_ATTR(visibility("default")) +void * +JEMALLOC_P(calloc)(size_t num, size_t size) +{ + void *ret; + size_t num_size; +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + size_t usize +# ifdef JEMALLOC_CC_SILENCE + = 0 +# endif + ; +#endif +#ifdef JEMALLOC_PROF + prof_thr_cnt_t *cnt +# ifdef JEMALLOC_CC_SILENCE + = NULL +# endif + ; +#endif + + if (malloc_init()) { + num_size = 0; + ret = NULL; + goto RETURN; + } + + num_size = num * size; + if (num_size == 0) { +#ifdef JEMALLOC_SYSV + if ((opt_sysv == false) && ((num == 0) || (size == 0))) +#endif + num_size = 1; +#ifdef JEMALLOC_SYSV + else { + ret = NULL; + goto RETURN; + } +#endif + /* + * Try to avoid division here. We know that it isn't possible to + * overflow during multiplication if neither operand uses any of the + * most significant half of the bits in a size_t. + */ + } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) + && (num_size / size != num)) { + /* size_t overflow. */ + ret = NULL; + goto RETURN; + } + +#ifdef JEMALLOC_PROF + if (opt_prof) { + usize = s2u(num_size); + PROF_ALLOC_PREP(1, usize, cnt); + if (cnt == NULL) { + ret = NULL; + goto RETURN; + } + if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize + <= small_maxclass) { + ret = icalloc(small_maxclass+1); + if (ret != NULL) + arena_prof_promoted(ret, usize); + } else + ret = icalloc(num_size); + } else +#endif + { +#ifdef JEMALLOC_STATS + usize = s2u(num_size); +#endif + ret = icalloc(num_size); + } + +RETURN: + if (ret == NULL) { +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in calloc(): out of " + "memory\n"); + abort(); + } +#endif + errno = ENOMEM; + } + +#ifdef JEMALLOC_PROF + if (opt_prof && ret != NULL) + prof_malloc(ret, usize, cnt); +#endif +#ifdef JEMALLOC_STATS + if (ret != NULL) { + assert(usize == isalloc(ret)); + ALLOCATED_ADD(usize, 0); + } +#endif + return (ret); +} + +JEMALLOC_ATTR(visibility("default")) +void * +JEMALLOC_P(realloc)(void *ptr, size_t size) +{ + void *ret; +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + size_t usize +# ifdef JEMALLOC_CC_SILENCE + = 0 +# endif + ; + size_t old_size = 0; +#endif +#ifdef JEMALLOC_PROF + prof_thr_cnt_t *cnt +# ifdef JEMALLOC_CC_SILENCE + = NULL +# endif + ; + prof_ctx_t *old_ctx +# ifdef JEMALLOC_CC_SILENCE + = NULL +# endif + ; +#endif + + if (size == 0) { +#ifdef JEMALLOC_SYSV + if (opt_sysv == false) +#endif + size = 1; +#ifdef JEMALLOC_SYSV + else { + if (ptr != NULL) { +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + old_size = isalloc(ptr); +#endif +#ifdef JEMALLOC_PROF + if (opt_prof) { + old_ctx = prof_ctx_get(ptr); + cnt = NULL; + } +#endif + idalloc(ptr); + } +#ifdef JEMALLOC_PROF + else if (opt_prof) { + old_ctx = NULL; + cnt = NULL; + } +#endif + ret = NULL; + goto RETURN; + } +#endif + } + + if (ptr != NULL) { + assert(malloc_initialized || malloc_initializer == + pthread_self()); + +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + old_size = isalloc(ptr); +#endif +#ifdef JEMALLOC_PROF + if (opt_prof) { + usize = s2u(size); + old_ctx = prof_ctx_get(ptr); + PROF_ALLOC_PREP(1, usize, cnt); + if (cnt == NULL) { + old_ctx = NULL; + ret = NULL; + goto OOM; + } + if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && + usize <= small_maxclass) { + ret = iralloc(ptr, small_maxclass+1, 0, 0, + false, false); + if (ret != NULL) + arena_prof_promoted(ret, usize); + else + old_ctx = NULL; + } else { + ret = iralloc(ptr, size, 0, 0, false, false); + if (ret == NULL) + old_ctx = NULL; + } + } else +#endif + { +#ifdef JEMALLOC_STATS + usize = s2u(size); +#endif + ret = iralloc(ptr, size, 0, 0, false, false); + } + +#ifdef JEMALLOC_PROF +OOM: +#endif + if (ret == NULL) { +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in realloc(): " + "out of memory\n"); + abort(); + } +#endif + errno = ENOMEM; + } + } else { +#ifdef JEMALLOC_PROF + if (opt_prof) + old_ctx = NULL; +#endif + if (malloc_init()) { +#ifdef JEMALLOC_PROF + if (opt_prof) + cnt = NULL; +#endif + ret = NULL; + } else { +#ifdef JEMALLOC_PROF + if (opt_prof) { + usize = s2u(size); + PROF_ALLOC_PREP(1, usize, cnt); + if (cnt == NULL) + ret = NULL; + else { + if (prof_promote && (uintptr_t)cnt != + (uintptr_t)1U && usize <= + small_maxclass) { + ret = imalloc(small_maxclass+1); + if (ret != NULL) { + arena_prof_promoted(ret, + usize); + } + } else + ret = imalloc(size); + } + } else +#endif + { +#ifdef JEMALLOC_STATS + usize = s2u(size); +#endif + ret = imalloc(size); + } + } + + if (ret == NULL) { +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in realloc(): " + "out of memory\n"); + abort(); + } +#endif + errno = ENOMEM; + } + } + +#ifdef JEMALLOC_SYSV +RETURN: +#endif +#ifdef JEMALLOC_PROF + if (opt_prof) + prof_realloc(ret, usize, cnt, old_size, old_ctx); +#endif +#ifdef JEMALLOC_STATS + if (ret != NULL) { + assert(usize == isalloc(ret)); + ALLOCATED_ADD(usize, old_size); + } +#endif + return (ret); +} + +JEMALLOC_ATTR(visibility("default")) +void +JEMALLOC_P(free)(void *ptr) +{ + + if (ptr != NULL) { +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + size_t usize; +#endif + + assert(malloc_initialized || malloc_initializer == + pthread_self()); + +#ifdef JEMALLOC_STATS + usize = isalloc(ptr); +#endif +#ifdef JEMALLOC_PROF + if (opt_prof) { +# ifndef JEMALLOC_STATS + usize = isalloc(ptr); +# endif + prof_free(ptr, usize); + } +#endif +#ifdef JEMALLOC_STATS + ALLOCATED_ADD(0, usize); +#endif + idalloc(ptr); + } +} + +/* + * End malloc(3)-compatible functions. + */ +/******************************************************************************/ +/* + * Begin non-standard override functions. + * + * These overrides are omitted if the JEMALLOC_PREFIX is defined, since the + * entire point is to avoid accidental mixed allocator usage. + */ +#ifndef JEMALLOC_PREFIX + +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_ATTR(malloc) +JEMALLOC_ATTR(visibility("default")) +void * +JEMALLOC_P(memalign)(size_t alignment, size_t size) +{ + void *ret; +#ifdef JEMALLOC_CC_SILENCE + int result = +#endif + imemalign(&ret, alignment, size); +#ifdef JEMALLOC_CC_SILENCE + if (result != 0) + return (NULL); +#endif + return (ret); +} +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_ATTR(malloc) +JEMALLOC_ATTR(visibility("default")) +void * +JEMALLOC_P(valloc)(size_t size) +{ + void *ret; +#ifdef JEMALLOC_CC_SILENCE + int result = +#endif + imemalign(&ret, PAGE_SIZE, size); +#ifdef JEMALLOC_CC_SILENCE + if (result != 0) + return (NULL); +#endif + return (ret); +} +#endif + +#endif /* JEMALLOC_PREFIX */ +/* + * End non-standard override functions. + */ +/******************************************************************************/ +/* + * Begin non-standard functions. + */ + +JEMALLOC_ATTR(visibility("default")) +size_t +JEMALLOC_P(malloc_usable_size)(const void *ptr) +{ + size_t ret; + + assert(malloc_initialized || malloc_initializer == pthread_self()); + +#ifdef JEMALLOC_IVSALLOC + ret = ivsalloc(ptr); +#else + assert(ptr != NULL); + ret = isalloc(ptr); +#endif + + return (ret); +} + +JEMALLOC_ATTR(visibility("default")) +void +JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), + void *cbopaque, const char *opts) +{ + + stats_print(write_cb, cbopaque, opts); +} + +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_byname(name, oldp, oldlenp, newp, newlen)); +} + +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_nametomib(name, mibp, miblenp)); +} + +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); +} + +JEMALLOC_INLINE void * +iallocm(size_t usize, size_t alignment, bool zero) +{ + + assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment, + NULL))); + + if (alignment != 0) + return (ipalloc(usize, alignment, zero)); + else if (zero) + return (icalloc(usize)); + else + return (imalloc(usize)); +} + +JEMALLOC_ATTR(nonnull(1)) +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) +{ + void *p; + size_t usize; + size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + & (SIZE_T_MAX-1)); + bool zero = flags & ALLOCM_ZERO; +#ifdef JEMALLOC_PROF + prof_thr_cnt_t *cnt; +#endif + + assert(ptr != NULL); + assert(size != 0); + + if (malloc_init()) + goto OOM; + + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL); + if (usize == 0) + goto OOM; + +#ifdef JEMALLOC_PROF + if (opt_prof) { + PROF_ALLOC_PREP(1, usize, cnt); + if (cnt == NULL) + goto OOM; + if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= + small_maxclass) { + size_t usize_promoted = (alignment == 0) ? + s2u(small_maxclass+1) : sa2u(small_maxclass+1, + alignment, NULL); + assert(usize_promoted != 0); + p = iallocm(usize_promoted, alignment, zero); + if (p == NULL) + goto OOM; + arena_prof_promoted(p, usize); + } else { + p = iallocm(usize, alignment, zero); + if (p == NULL) + goto OOM; + } + prof_malloc(p, usize, cnt); + if (rsize != NULL) + *rsize = usize; + } else +#endif + { + p = iallocm(usize, alignment, zero); + if (p == NULL) + goto OOM; +#ifndef JEMALLOC_STATS + if (rsize != NULL) +#endif + { +#ifdef JEMALLOC_STATS + if (rsize != NULL) +#endif + *rsize = usize; + } + } + + *ptr = p; +#ifdef JEMALLOC_STATS + assert(usize == isalloc(p)); + ALLOCATED_ADD(usize, 0); +#endif + return (ALLOCM_SUCCESS); +OOM: +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in allocm(): " + "out of memory\n"); + abort(); + } +#endif + *ptr = NULL; + return (ALLOCM_ERR_OOM); +} + +JEMALLOC_ATTR(nonnull(1)) +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, + int flags) +{ + void *p, *q; + size_t usize; +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + size_t old_size; +#endif + size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + & (SIZE_T_MAX-1)); + bool zero = flags & ALLOCM_ZERO; + bool no_move = flags & ALLOCM_NO_MOVE; +#ifdef JEMALLOC_PROF + prof_thr_cnt_t *cnt; +#endif + + assert(ptr != NULL); + assert(*ptr != NULL); + assert(size != 0); + assert(SIZE_T_MAX - size >= extra); + assert(malloc_initialized || malloc_initializer == pthread_self()); + + p = *ptr; +#ifdef JEMALLOC_PROF + if (opt_prof) { + /* + * usize isn't knowable before iralloc() returns when extra is + * non-zero. Therefore, compute its maximum possible value and + * use that in PROF_ALLOC_PREP() to decide whether to capture a + * backtrace. prof_realloc() will use the actual usize to + * decide whether to sample. + */ + size_t max_usize = (alignment == 0) ? s2u(size+extra) : + sa2u(size+extra, alignment, NULL); + prof_ctx_t *old_ctx = prof_ctx_get(p); + old_size = isalloc(p); + PROF_ALLOC_PREP(1, max_usize, cnt); + if (cnt == NULL) + goto OOM; + /* + * Use minimum usize to determine whether promotion may happen. + */ + if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U + && ((alignment == 0) ? s2u(size) : sa2u(size, + alignment, NULL)) <= small_maxclass) { + q = iralloc(p, small_maxclass+1, (small_maxclass+1 >= + size+extra) ? 0 : size+extra - (small_maxclass+1), + alignment, zero, no_move); + if (q == NULL) + goto ERR; + if (max_usize < PAGE_SIZE) { + usize = max_usize; + arena_prof_promoted(q, usize); + } else + usize = isalloc(q); + } else { + q = iralloc(p, size, extra, alignment, zero, no_move); + if (q == NULL) + goto ERR; + usize = isalloc(q); + } + prof_realloc(q, usize, cnt, old_size, old_ctx); + if (rsize != NULL) + *rsize = usize; + } else +#endif + { +#ifdef JEMALLOC_STATS + old_size = isalloc(p); +#endif + q = iralloc(p, size, extra, alignment, zero, no_move); + if (q == NULL) + goto ERR; +#ifndef JEMALLOC_STATS + if (rsize != NULL) +#endif + { + usize = isalloc(q); +#ifdef JEMALLOC_STATS + if (rsize != NULL) +#endif + *rsize = usize; + } + } + + *ptr = q; +#ifdef JEMALLOC_STATS + ALLOCATED_ADD(usize, old_size); +#endif + return (ALLOCM_SUCCESS); +ERR: + if (no_move) + return (ALLOCM_ERR_NOT_MOVED); +#ifdef JEMALLOC_PROF +OOM: +#endif +#ifdef JEMALLOC_XMALLOC + if (opt_xmalloc) { + malloc_write(": Error in rallocm(): " + "out of memory\n"); + abort(); + } +#endif + return (ALLOCM_ERR_OOM); +} + +JEMALLOC_ATTR(nonnull(1)) +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) +{ + size_t sz; + + assert(malloc_initialized || malloc_initializer == pthread_self()); + +#ifdef JEMALLOC_IVSALLOC + sz = ivsalloc(ptr); +#else + assert(ptr != NULL); + sz = isalloc(ptr); +#endif + assert(rsize != NULL); + *rsize = sz; + + return (ALLOCM_SUCCESS); +} + +JEMALLOC_ATTR(nonnull(1)) +JEMALLOC_ATTR(visibility("default")) +int +JEMALLOC_P(dallocm)(void *ptr, int flags) +{ +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + size_t usize; +#endif + + assert(ptr != NULL); + assert(malloc_initialized || malloc_initializer == pthread_self()); + +#ifdef JEMALLOC_STATS + usize = isalloc(ptr); +#endif +#ifdef JEMALLOC_PROF + if (opt_prof) { +# ifndef JEMALLOC_STATS + usize = isalloc(ptr); +# endif + prof_free(ptr, usize); + } +#endif +#ifdef JEMALLOC_STATS + ALLOCATED_ADD(0, usize); +#endif + idalloc(ptr); + + return (ALLOCM_SUCCESS); +} + +/* + * End non-standard functions. + */ +/******************************************************************************/ + +/* + * The following functions are used by threading libraries for protection of + * malloc during fork(). + */ + +void +jemalloc_prefork(void) +{ + unsigned i; + + /* Acquire all mutexes in a safe order. */ + + malloc_mutex_lock(&arenas_lock); + for (i = 0; i < narenas; i++) { + if (arenas[i] != NULL) + malloc_mutex_lock(&arenas[i]->lock); + } + + malloc_mutex_lock(&base_mtx); + + malloc_mutex_lock(&huge_mtx); + +#ifdef JEMALLOC_DSS + malloc_mutex_lock(&dss_mtx); +#endif + +#ifdef JEMALLOC_SWAP + malloc_mutex_lock(&swap_mtx); +#endif +} + +void +jemalloc_postfork(void) +{ + unsigned i; + + /* Release all mutexes, now that fork() has completed. */ + +#ifdef JEMALLOC_SWAP + malloc_mutex_unlock(&swap_mtx); +#endif + +#ifdef JEMALLOC_DSS + malloc_mutex_unlock(&dss_mtx); +#endif + + malloc_mutex_unlock(&huge_mtx); + + malloc_mutex_unlock(&base_mtx); + + for (i = 0; i < narenas; i++) { + if (arenas[i] != NULL) + malloc_mutex_unlock(&arenas[i]->lock); + } + malloc_mutex_unlock(&arenas_lock); +} + +/******************************************************************************/ diff --git a/deps/jemalloc.orig/src/mb.c b/deps/jemalloc.orig/src/mb.c new file mode 100644 index 00000000000..dc2c0a256fd --- /dev/null +++ b/deps/jemalloc.orig/src/mb.c @@ -0,0 +1,2 @@ +#define JEMALLOC_MB_C_ +#include "jemalloc/internal/jemalloc_internal.h" diff --git a/deps/jemalloc.orig/src/mutex.c b/deps/jemalloc.orig/src/mutex.c new file mode 100644 index 00000000000..ca89ef1c962 --- /dev/null +++ b/deps/jemalloc.orig/src/mutex.c @@ -0,0 +1,90 @@ +#define JEMALLOC_MUTEX_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +#ifdef JEMALLOC_LAZY_LOCK +bool isthreaded = false; +#endif + +#ifdef JEMALLOC_LAZY_LOCK +static void pthread_create_once(void); +#endif + +/******************************************************************************/ +/* + * We intercept pthread_create() calls in order to toggle isthreaded if the + * process goes multi-threaded. + */ + +#ifdef JEMALLOC_LAZY_LOCK +static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, + void *(*)(void *), void *__restrict); + +static void +pthread_create_once(void) +{ + + pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); + if (pthread_create_fptr == NULL) { + malloc_write(": Error in dlsym(RTLD_NEXT, " + "\"pthread_create\")\n"); + abort(); + } + + isthreaded = true; +} + +JEMALLOC_ATTR(visibility("default")) +int +pthread_create(pthread_t *__restrict thread, + const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), + void *__restrict arg) +{ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + + pthread_once(&once_control, pthread_create_once); + + return (pthread_create_fptr(thread, attr, start_routine, arg)); +} +#endif + +/******************************************************************************/ + +bool +malloc_mutex_init(malloc_mutex_t *mutex) +{ +#ifdef JEMALLOC_OSSPIN + *mutex = 0; +#else + pthread_mutexattr_t attr; + + if (pthread_mutexattr_init(&attr) != 0) + return (true); +#ifdef PTHREAD_MUTEX_ADAPTIVE_NP + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP); +#else + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); +#endif + if (pthread_mutex_init(mutex, &attr) != 0) { + pthread_mutexattr_destroy(&attr); + return (true); + } + pthread_mutexattr_destroy(&attr); + +#endif + return (false); +} + +void +malloc_mutex_destroy(malloc_mutex_t *mutex) +{ + +#ifndef JEMALLOC_OSSPIN + if (pthread_mutex_destroy(mutex) != 0) { + malloc_write(": Error in pthread_mutex_destroy()\n"); + abort(); + } +#endif +} diff --git a/deps/jemalloc.orig/src/prof.c b/deps/jemalloc.orig/src/prof.c new file mode 100644 index 00000000000..8a144b4e46c --- /dev/null +++ b/deps/jemalloc.orig/src/prof.c @@ -0,0 +1,1244 @@ +#define JEMALLOC_PROF_C_ +#include "jemalloc/internal/jemalloc_internal.h" +#ifdef JEMALLOC_PROF +/******************************************************************************/ + +#ifdef JEMALLOC_PROF_LIBUNWIND +#define UNW_LOCAL_ONLY +#include +#endif + +#ifdef JEMALLOC_PROF_LIBGCC +#include +#endif + +/******************************************************************************/ +/* Data. */ + +bool opt_prof = false; +bool opt_prof_active = true; +size_t opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT; +size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; +ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; +bool opt_prof_gdump = false; +bool opt_prof_leak = false; +bool opt_prof_accum = true; +ssize_t opt_lg_prof_tcmax = LG_PROF_TCMAX_DEFAULT; +char opt_prof_prefix[PATH_MAX + 1]; + +uint64_t prof_interval; +bool prof_promote; + +unsigned prof_bt_max; + +#ifndef NO_TLS +__thread prof_tdata_t *prof_tdata_tls + JEMALLOC_ATTR(tls_model("initial-exec")); +#endif +pthread_key_t prof_tdata_tsd; + +/* + * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data + * structure that knows about all backtraces currently captured. + */ +static ckh_t bt2ctx; +static malloc_mutex_t bt2ctx_mtx; + +static malloc_mutex_t prof_dump_seq_mtx; +static uint64_t prof_dump_seq; +static uint64_t prof_dump_iseq; +static uint64_t prof_dump_mseq; +static uint64_t prof_dump_useq; + +/* + * This buffer is rather large for stack allocation, so use a single buffer for + * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since + * it must be locked anyway during dumping. + */ +static char prof_dump_buf[PROF_DUMP_BUF_SIZE]; +static unsigned prof_dump_buf_end; +static int prof_dump_fd; + +/* Do not dump any profiles until bootstrapping is complete. */ +static bool prof_booted = false; + +static malloc_mutex_t enq_mtx; +static bool enq; +static bool enq_idump; +static bool enq_gdump; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static prof_bt_t *bt_dup(prof_bt_t *bt); +static void bt_destroy(prof_bt_t *bt); +#ifdef JEMALLOC_PROF_LIBGCC +static _Unwind_Reason_Code prof_unwind_init_callback( + struct _Unwind_Context *context, void *arg); +static _Unwind_Reason_Code prof_unwind_callback( + struct _Unwind_Context *context, void *arg); +#endif +static bool prof_flush(bool propagate_err); +static bool prof_write(const char *s, bool propagate_err); +static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, + size_t *leak_nctx); +static void prof_ctx_destroy(prof_ctx_t *ctx); +static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); +static bool prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, + bool propagate_err); +static bool prof_dump_maps(bool propagate_err); +static bool prof_dump(const char *filename, bool leakcheck, + bool propagate_err); +static void prof_dump_filename(char *filename, char v, int64_t vseq); +static void prof_fdump(void); +static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, + size_t *hash2); +static bool prof_bt_keycomp(const void *k1, const void *k2); +static void prof_tdata_cleanup(void *arg); + +/******************************************************************************/ + +void +bt_init(prof_bt_t *bt, void **vec) +{ + + bt->vec = vec; + bt->len = 0; +} + +static void +bt_destroy(prof_bt_t *bt) +{ + + idalloc(bt); +} + +static prof_bt_t * +bt_dup(prof_bt_t *bt) +{ + prof_bt_t *ret; + + /* + * Create a single allocation that has space for vec immediately + * following the prof_bt_t structure. The backtraces that get + * stored in the backtrace caches are copied from stack-allocated + * temporary variables, so size is known at creation time. Making this + * a contiguous object improves cache locality. + */ + ret = (prof_bt_t *)imalloc(QUANTUM_CEILING(sizeof(prof_bt_t)) + + (bt->len * sizeof(void *))); + if (ret == NULL) + return (NULL); + ret->vec = (void **)((uintptr_t)ret + + QUANTUM_CEILING(sizeof(prof_bt_t))); + memcpy(ret->vec, bt->vec, bt->len * sizeof(void *)); + ret->len = bt->len; + + return (ret); +} + +static inline void +prof_enter(void) +{ + + malloc_mutex_lock(&enq_mtx); + enq = true; + malloc_mutex_unlock(&enq_mtx); + + malloc_mutex_lock(&bt2ctx_mtx); +} + +static inline void +prof_leave(void) +{ + bool idump, gdump; + + malloc_mutex_unlock(&bt2ctx_mtx); + + malloc_mutex_lock(&enq_mtx); + enq = false; + idump = enq_idump; + enq_idump = false; + gdump = enq_gdump; + enq_gdump = false; + malloc_mutex_unlock(&enq_mtx); + + if (idump) + prof_idump(); + if (gdump) + prof_gdump(); +} + +#ifdef JEMALLOC_PROF_LIBUNWIND +void +prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +{ + unw_context_t uc; + unw_cursor_t cursor; + unsigned i; + int err; + + assert(bt->len == 0); + assert(bt->vec != NULL); + assert(max <= (1U << opt_lg_prof_bt_max)); + + unw_getcontext(&uc); + unw_init_local(&cursor, &uc); + + /* Throw away (nignore+1) stack frames, if that many exist. */ + for (i = 0; i < nignore + 1; i++) { + err = unw_step(&cursor); + if (err <= 0) + return; + } + + /* + * Iterate over stack frames until there are no more, or until no space + * remains in bt. + */ + for (i = 0; i < max; i++) { + unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); + bt->len++; + err = unw_step(&cursor); + if (err <= 0) + break; + } +} +#endif +#ifdef JEMALLOC_PROF_LIBGCC +static _Unwind_Reason_Code +prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) +{ + + return (_URC_NO_REASON); +} + +static _Unwind_Reason_Code +prof_unwind_callback(struct _Unwind_Context *context, void *arg) +{ + prof_unwind_data_t *data = (prof_unwind_data_t *)arg; + + if (data->nignore > 0) + data->nignore--; + else { + data->bt->vec[data->bt->len] = (void *)_Unwind_GetIP(context); + data->bt->len++; + if (data->bt->len == data->max) + return (_URC_END_OF_STACK); + } + + return (_URC_NO_REASON); +} + +void +prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +{ + prof_unwind_data_t data = {bt, nignore, max}; + + _Unwind_Backtrace(prof_unwind_callback, &data); +} +#endif +#ifdef JEMALLOC_PROF_GCC +void +prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +{ +#define BT_FRAME(i) \ + if ((i) < nignore + max) { \ + void *p; \ + if (__builtin_frame_address(i) == 0) \ + return; \ + p = __builtin_return_address(i); \ + if (p == NULL) \ + return; \ + if (i >= nignore) { \ + bt->vec[(i) - nignore] = p; \ + bt->len = (i) - nignore + 1; \ + } \ + } else \ + return; + + assert(nignore <= 3); + assert(max <= (1U << opt_lg_prof_bt_max)); + + BT_FRAME(0) + BT_FRAME(1) + BT_FRAME(2) + BT_FRAME(3) + BT_FRAME(4) + BT_FRAME(5) + BT_FRAME(6) + BT_FRAME(7) + BT_FRAME(8) + BT_FRAME(9) + + BT_FRAME(10) + BT_FRAME(11) + BT_FRAME(12) + BT_FRAME(13) + BT_FRAME(14) + BT_FRAME(15) + BT_FRAME(16) + BT_FRAME(17) + BT_FRAME(18) + BT_FRAME(19) + + BT_FRAME(20) + BT_FRAME(21) + BT_FRAME(22) + BT_FRAME(23) + BT_FRAME(24) + BT_FRAME(25) + BT_FRAME(26) + BT_FRAME(27) + BT_FRAME(28) + BT_FRAME(29) + + BT_FRAME(30) + BT_FRAME(31) + BT_FRAME(32) + BT_FRAME(33) + BT_FRAME(34) + BT_FRAME(35) + BT_FRAME(36) + BT_FRAME(37) + BT_FRAME(38) + BT_FRAME(39) + + BT_FRAME(40) + BT_FRAME(41) + BT_FRAME(42) + BT_FRAME(43) + BT_FRAME(44) + BT_FRAME(45) + BT_FRAME(46) + BT_FRAME(47) + BT_FRAME(48) + BT_FRAME(49) + + BT_FRAME(50) + BT_FRAME(51) + BT_FRAME(52) + BT_FRAME(53) + BT_FRAME(54) + BT_FRAME(55) + BT_FRAME(56) + BT_FRAME(57) + BT_FRAME(58) + BT_FRAME(59) + + BT_FRAME(60) + BT_FRAME(61) + BT_FRAME(62) + BT_FRAME(63) + BT_FRAME(64) + BT_FRAME(65) + BT_FRAME(66) + BT_FRAME(67) + BT_FRAME(68) + BT_FRAME(69) + + BT_FRAME(70) + BT_FRAME(71) + BT_FRAME(72) + BT_FRAME(73) + BT_FRAME(74) + BT_FRAME(75) + BT_FRAME(76) + BT_FRAME(77) + BT_FRAME(78) + BT_FRAME(79) + + BT_FRAME(80) + BT_FRAME(81) + BT_FRAME(82) + BT_FRAME(83) + BT_FRAME(84) + BT_FRAME(85) + BT_FRAME(86) + BT_FRAME(87) + BT_FRAME(88) + BT_FRAME(89) + + BT_FRAME(90) + BT_FRAME(91) + BT_FRAME(92) + BT_FRAME(93) + BT_FRAME(94) + BT_FRAME(95) + BT_FRAME(96) + BT_FRAME(97) + BT_FRAME(98) + BT_FRAME(99) + + BT_FRAME(100) + BT_FRAME(101) + BT_FRAME(102) + BT_FRAME(103) + BT_FRAME(104) + BT_FRAME(105) + BT_FRAME(106) + BT_FRAME(107) + BT_FRAME(108) + BT_FRAME(109) + + BT_FRAME(110) + BT_FRAME(111) + BT_FRAME(112) + BT_FRAME(113) + BT_FRAME(114) + BT_FRAME(115) + BT_FRAME(116) + BT_FRAME(117) + BT_FRAME(118) + BT_FRAME(119) + + BT_FRAME(120) + BT_FRAME(121) + BT_FRAME(122) + BT_FRAME(123) + BT_FRAME(124) + BT_FRAME(125) + BT_FRAME(126) + BT_FRAME(127) + + /* Extras to compensate for nignore. */ + BT_FRAME(128) + BT_FRAME(129) + BT_FRAME(130) +#undef BT_FRAME +} +#endif + +prof_thr_cnt_t * +prof_lookup(prof_bt_t *bt) +{ + union { + prof_thr_cnt_t *p; + void *v; + } ret; + prof_tdata_t *prof_tdata; + + prof_tdata = PROF_TCACHE_GET(); + if (prof_tdata == NULL) { + prof_tdata = prof_tdata_init(); + if (prof_tdata == NULL) + return (NULL); + } + + if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { + union { + prof_bt_t *p; + void *v; + } btkey; + union { + prof_ctx_t *p; + void *v; + } ctx; + bool new_ctx; + + /* + * This thread's cache lacks bt. Look for it in the global + * cache. + */ + prof_enter(); + if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { + /* bt has never been seen before. Insert it. */ + ctx.v = imalloc(sizeof(prof_ctx_t)); + if (ctx.v == NULL) { + prof_leave(); + return (NULL); + } + btkey.p = bt_dup(bt); + if (btkey.v == NULL) { + prof_leave(); + idalloc(ctx.v); + return (NULL); + } + ctx.p->bt = btkey.p; + if (malloc_mutex_init(&ctx.p->lock)) { + prof_leave(); + idalloc(btkey.v); + idalloc(ctx.v); + return (NULL); + } + memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); + ql_new(&ctx.p->cnts_ql); + if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { + /* OOM. */ + prof_leave(); + malloc_mutex_destroy(&ctx.p->lock); + idalloc(btkey.v); + idalloc(ctx.v); + return (NULL); + } + /* + * Artificially raise curobjs, in order to avoid a race + * condition with prof_ctx_merge()/prof_ctx_destroy(). + * + * No locking is necessary for ctx here because no other + * threads have had the opportunity to fetch it from + * bt2ctx yet. + */ + ctx.p->cnt_merged.curobjs++; + new_ctx = true; + } else { + /* + * Artificially raise curobjs, in order to avoid a race + * condition with prof_ctx_merge()/prof_ctx_destroy(). + */ + malloc_mutex_lock(&ctx.p->lock); + ctx.p->cnt_merged.curobjs++; + malloc_mutex_unlock(&ctx.p->lock); + new_ctx = false; + } + prof_leave(); + + /* Link a prof_thd_cnt_t into ctx for this thread. */ + if (opt_lg_prof_tcmax >= 0 && ckh_count(&prof_tdata->bt2cnt) + == (ZU(1) << opt_lg_prof_tcmax)) { + assert(ckh_count(&prof_tdata->bt2cnt) > 0); + /* + * Flush the least recently used cnt in order to keep + * bt2cnt from becoming too large. + */ + ret.p = ql_last(&prof_tdata->lru_ql, lru_link); + assert(ret.v != NULL); + if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, + NULL, NULL)) + assert(false); + ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); + prof_ctx_merge(ret.p->ctx, ret.p); + /* ret can now be re-used. */ + } else { + assert(opt_lg_prof_tcmax < 0 || + ckh_count(&prof_tdata->bt2cnt) < (ZU(1) << + opt_lg_prof_tcmax)); + /* Allocate and partially initialize a new cnt. */ + ret.v = imalloc(sizeof(prof_thr_cnt_t)); + if (ret.p == NULL) { + if (new_ctx) + prof_ctx_destroy(ctx.p); + return (NULL); + } + ql_elm_new(ret.p, cnts_link); + ql_elm_new(ret.p, lru_link); + } + /* Finish initializing ret. */ + ret.p->ctx = ctx.p; + ret.p->epoch = 0; + memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); + if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) { + if (new_ctx) + prof_ctx_destroy(ctx.p); + idalloc(ret.v); + return (NULL); + } + ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); + malloc_mutex_lock(&ctx.p->lock); + ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); + ctx.p->cnt_merged.curobjs--; + malloc_mutex_unlock(&ctx.p->lock); + } else { + /* Move ret to the front of the LRU. */ + ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); + ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); + } + + return (ret.p); +} + +static bool +prof_flush(bool propagate_err) +{ + bool ret = false; + ssize_t err; + + err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); + if (err == -1) { + if (propagate_err == false) { + malloc_write(": write() failed during heap " + "profile flush\n"); + if (opt_abort) + abort(); + } + ret = true; + } + prof_dump_buf_end = 0; + + return (ret); +} + +static bool +prof_write(const char *s, bool propagate_err) +{ + unsigned i, slen, n; + + i = 0; + slen = strlen(s); + while (i < slen) { + /* Flush the buffer if it is full. */ + if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) + if (prof_flush(propagate_err) && propagate_err) + return (true); + + if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) { + /* Finish writing. */ + n = slen - i; + } else { + /* Write as much of s as will fit. */ + n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end; + } + memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); + prof_dump_buf_end += n; + i += n; + } + + return (false); +} + +static void +prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) +{ + prof_thr_cnt_t *thr_cnt; + prof_cnt_t tcnt; + + malloc_mutex_lock(&ctx->lock); + + memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); + ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { + volatile unsigned *epoch = &thr_cnt->epoch; + + while (true) { + unsigned epoch0 = *epoch; + + /* Make sure epoch is even. */ + if (epoch0 & 1U) + continue; + + memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); + + /* Terminate if epoch didn't change while reading. */ + if (*epoch == epoch0) + break; + } + + ctx->cnt_summed.curobjs += tcnt.curobjs; + ctx->cnt_summed.curbytes += tcnt.curbytes; + if (opt_prof_accum) { + ctx->cnt_summed.accumobjs += tcnt.accumobjs; + ctx->cnt_summed.accumbytes += tcnt.accumbytes; + } + } + + if (ctx->cnt_summed.curobjs != 0) + (*leak_nctx)++; + + /* Add to cnt_all. */ + cnt_all->curobjs += ctx->cnt_summed.curobjs; + cnt_all->curbytes += ctx->cnt_summed.curbytes; + if (opt_prof_accum) { + cnt_all->accumobjs += ctx->cnt_summed.accumobjs; + cnt_all->accumbytes += ctx->cnt_summed.accumbytes; + } + + malloc_mutex_unlock(&ctx->lock); +} + +static void +prof_ctx_destroy(prof_ctx_t *ctx) +{ + + /* + * Check that ctx is still unused by any thread cache before destroying + * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in + * order to avoid a race condition with this function, as does + * prof_ctx_merge() in order to avoid a race between the main body of + * prof_ctx_merge() and entry into this function. + */ + prof_enter(); + malloc_mutex_lock(&ctx->lock); + if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { + assert(ctx->cnt_merged.curbytes == 0); + assert(ctx->cnt_merged.accumobjs == 0); + assert(ctx->cnt_merged.accumbytes == 0); + /* Remove ctx from bt2ctx. */ + if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) + assert(false); + prof_leave(); + /* Destroy ctx. */ + malloc_mutex_unlock(&ctx->lock); + bt_destroy(ctx->bt); + malloc_mutex_destroy(&ctx->lock); + idalloc(ctx); + } else { + /* + * Compensate for increment in prof_ctx_merge() or + * prof_lookup(). + */ + ctx->cnt_merged.curobjs--; + malloc_mutex_unlock(&ctx->lock); + prof_leave(); + } +} + +static void +prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) +{ + bool destroy; + + /* Merge cnt stats and detach from ctx. */ + malloc_mutex_lock(&ctx->lock); + ctx->cnt_merged.curobjs += cnt->cnts.curobjs; + ctx->cnt_merged.curbytes += cnt->cnts.curbytes; + ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; + ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; + ql_remove(&ctx->cnts_ql, cnt, cnts_link); + if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && + ctx->cnt_merged.curobjs == 0) { + /* + * Artificially raise ctx->cnt_merged.curobjs in order to keep + * another thread from winning the race to destroy ctx while + * this one has ctx->lock dropped. Without this, it would be + * possible for another thread to: + * + * 1) Sample an allocation associated with ctx. + * 2) Deallocate the sampled object. + * 3) Successfully prof_ctx_destroy(ctx). + * + * The result would be that ctx no longer exists by the time + * this thread accesses it in prof_ctx_destroy(). + */ + ctx->cnt_merged.curobjs++; + destroy = true; + } else + destroy = false; + malloc_mutex_unlock(&ctx->lock); + if (destroy) + prof_ctx_destroy(ctx); +} + +static bool +prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, bool propagate_err) +{ + char buf[UMAX2S_BUFSIZE]; + unsigned i; + + if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { + assert(ctx->cnt_summed.curbytes == 0); + assert(ctx->cnt_summed.accumobjs == 0); + assert(ctx->cnt_summed.accumbytes == 0); + return (false); + } + + if (prof_write(u2s(ctx->cnt_summed.curobjs, 10, buf), propagate_err) + || prof_write(": ", propagate_err) + || prof_write(u2s(ctx->cnt_summed.curbytes, 10, buf), + propagate_err) + || prof_write(" [", propagate_err) + || prof_write(u2s(ctx->cnt_summed.accumobjs, 10, buf), + propagate_err) + || prof_write(": ", propagate_err) + || prof_write(u2s(ctx->cnt_summed.accumbytes, 10, buf), + propagate_err) + || prof_write("] @", propagate_err)) + return (true); + + for (i = 0; i < bt->len; i++) { + if (prof_write(" 0x", propagate_err) + || prof_write(u2s((uintptr_t)bt->vec[i], 16, buf), + propagate_err)) + return (true); + } + + if (prof_write("\n", propagate_err)) + return (true); + + return (false); +} + +static bool +prof_dump_maps(bool propagate_err) +{ + int mfd; + char buf[UMAX2S_BUFSIZE]; + char *s; + unsigned i, slen; + /* /proc//maps\0 */ + char mpath[6 + UMAX2S_BUFSIZE + + 5 + 1]; + + i = 0; + + s = "/proc/"; + slen = strlen(s); + memcpy(&mpath[i], s, slen); + i += slen; + + s = u2s(getpid(), 10, buf); + slen = strlen(s); + memcpy(&mpath[i], s, slen); + i += slen; + + s = "/maps"; + slen = strlen(s); + memcpy(&mpath[i], s, slen); + i += slen; + + mpath[i] = '\0'; + + mfd = open(mpath, O_RDONLY); + if (mfd != -1) { + ssize_t nread; + + if (prof_write("\nMAPPED_LIBRARIES:\n", propagate_err) && + propagate_err) + return (true); + nread = 0; + do { + prof_dump_buf_end += nread; + if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) { + /* Make space in prof_dump_buf before read(). */ + if (prof_flush(propagate_err) && propagate_err) + return (true); + } + nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], + PROF_DUMP_BUF_SIZE - prof_dump_buf_end); + } while (nread > 0); + close(mfd); + } else + return (true); + + return (false); +} + +static bool +prof_dump(const char *filename, bool leakcheck, bool propagate_err) +{ + prof_cnt_t cnt_all; + size_t tabind; + union { + prof_bt_t *p; + void *v; + } bt; + union { + prof_ctx_t *p; + void *v; + } ctx; + char buf[UMAX2S_BUFSIZE]; + size_t leak_nctx; + + prof_enter(); + prof_dump_fd = creat(filename, 0644); + if (prof_dump_fd == -1) { + if (propagate_err == false) { + malloc_write(": creat(\""); + malloc_write(filename); + malloc_write("\", 0644) failed\n"); + if (opt_abort) + abort(); + } + goto ERROR; + } + + /* Merge per thread profile stats, and sum them in cnt_all. */ + memset(&cnt_all, 0, sizeof(prof_cnt_t)); + leak_nctx = 0; + for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) + prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); + + /* Dump profile header. */ + if (prof_write("heap profile: ", propagate_err) + || prof_write(u2s(cnt_all.curobjs, 10, buf), propagate_err) + || prof_write(": ", propagate_err) + || prof_write(u2s(cnt_all.curbytes, 10, buf), propagate_err) + || prof_write(" [", propagate_err) + || prof_write(u2s(cnt_all.accumobjs, 10, buf), propagate_err) + || prof_write(": ", propagate_err) + || prof_write(u2s(cnt_all.accumbytes, 10, buf), propagate_err)) + goto ERROR; + + if (opt_lg_prof_sample == 0) { + if (prof_write("] @ heapprofile\n", propagate_err)) + goto ERROR; + } else { + if (prof_write("] @ heap_v2/", propagate_err) + || prof_write(u2s((uint64_t)1U << opt_lg_prof_sample, 10, + buf), propagate_err) + || prof_write("\n", propagate_err)) + goto ERROR; + } + + /* Dump per ctx profile stats. */ + for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) + == false;) { + if (prof_dump_ctx(ctx.p, bt.p, propagate_err)) + goto ERROR; + } + + /* Dump /proc//maps if possible. */ + if (prof_dump_maps(propagate_err)) + goto ERROR; + + if (prof_flush(propagate_err)) + goto ERROR; + close(prof_dump_fd); + prof_leave(); + + if (leakcheck && cnt_all.curbytes != 0) { + malloc_write(": Leak summary: "); + malloc_write(u2s(cnt_all.curbytes, 10, buf)); + malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, "); + malloc_write(u2s(cnt_all.curobjs, 10, buf)); + malloc_write((cnt_all.curobjs != 1) ? " objects, " : + " object, "); + malloc_write(u2s(leak_nctx, 10, buf)); + malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n"); + malloc_write(": Run pprof on \""); + malloc_write(filename); + malloc_write("\" for leak detail\n"); + } + + return (false); +ERROR: + prof_leave(); + return (true); +} + +#define DUMP_FILENAME_BUFSIZE (PATH_MAX+ UMAX2S_BUFSIZE \ + + 1 \ + + UMAX2S_BUFSIZE \ + + 2 \ + + UMAX2S_BUFSIZE \ + + 5 + 1) +static void +prof_dump_filename(char *filename, char v, int64_t vseq) +{ + char buf[UMAX2S_BUFSIZE]; + char *s; + unsigned i, slen; + + /* + * Construct a filename of the form: + * + * ...v.heap\0 + */ + + i = 0; + + s = opt_prof_prefix; + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + s = "."; + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + s = u2s(getpid(), 10, buf); + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + s = "."; + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + s = u2s(prof_dump_seq, 10, buf); + prof_dump_seq++; + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + s = "."; + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + filename[i] = v; + i++; + + if (vseq != 0xffffffffffffffffLLU) { + s = u2s(vseq, 10, buf); + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + } + + s = ".heap"; + slen = strlen(s); + memcpy(&filename[i], s, slen); + i += slen; + + filename[i] = '\0'; +} + +static void +prof_fdump(void) +{ + char filename[DUMP_FILENAME_BUFSIZE]; + + if (prof_booted == false) + return; + + if (opt_prof_prefix[0] != '\0') { + malloc_mutex_lock(&prof_dump_seq_mtx); + prof_dump_filename(filename, 'f', 0xffffffffffffffffLLU); + malloc_mutex_unlock(&prof_dump_seq_mtx); + prof_dump(filename, opt_prof_leak, false); + } +} + +void +prof_idump(void) +{ + char filename[DUMP_FILENAME_BUFSIZE]; + + if (prof_booted == false) + return; + malloc_mutex_lock(&enq_mtx); + if (enq) { + enq_idump = true; + malloc_mutex_unlock(&enq_mtx); + return; + } + malloc_mutex_unlock(&enq_mtx); + + if (opt_prof_prefix[0] != '\0') { + malloc_mutex_lock(&prof_dump_seq_mtx); + prof_dump_filename(filename, 'i', prof_dump_iseq); + prof_dump_iseq++; + malloc_mutex_unlock(&prof_dump_seq_mtx); + prof_dump(filename, false, false); + } +} + +bool +prof_mdump(const char *filename) +{ + char filename_buf[DUMP_FILENAME_BUFSIZE]; + + if (opt_prof == false || prof_booted == false) + return (true); + + if (filename == NULL) { + /* No filename specified, so automatically generate one. */ + if (opt_prof_prefix[0] == '\0') + return (true); + malloc_mutex_lock(&prof_dump_seq_mtx); + prof_dump_filename(filename_buf, 'm', prof_dump_mseq); + prof_dump_mseq++; + malloc_mutex_unlock(&prof_dump_seq_mtx); + filename = filename_buf; + } + return (prof_dump(filename, false, true)); +} + +void +prof_gdump(void) +{ + char filename[DUMP_FILENAME_BUFSIZE]; + + if (prof_booted == false) + return; + malloc_mutex_lock(&enq_mtx); + if (enq) { + enq_gdump = true; + malloc_mutex_unlock(&enq_mtx); + return; + } + malloc_mutex_unlock(&enq_mtx); + + if (opt_prof_prefix[0] != '\0') { + malloc_mutex_lock(&prof_dump_seq_mtx); + prof_dump_filename(filename, 'u', prof_dump_useq); + prof_dump_useq++; + malloc_mutex_unlock(&prof_dump_seq_mtx); + prof_dump(filename, false, false); + } +} + +static void +prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) +{ + size_t ret1, ret2; + uint64_t h; + prof_bt_t *bt = (prof_bt_t *)key; + + assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); + assert(hash1 != NULL); + assert(hash2 != NULL); + + h = hash(bt->vec, bt->len * sizeof(void *), 0x94122f335b332aeaLLU); + if (minbits <= 32) { + /* + * Avoid doing multiple hashes, since a single hash provides + * enough bits. + */ + ret1 = h & ZU(0xffffffffU); + ret2 = h >> 32; + } else { + ret1 = h; + ret2 = hash(bt->vec, bt->len * sizeof(void *), + 0x8432a476666bbc13LLU); + } + + *hash1 = ret1; + *hash2 = ret2; +} + +static bool +prof_bt_keycomp(const void *k1, const void *k2) +{ + const prof_bt_t *bt1 = (prof_bt_t *)k1; + const prof_bt_t *bt2 = (prof_bt_t *)k2; + + if (bt1->len != bt2->len) + return (false); + return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); +} + +prof_tdata_t * +prof_tdata_init(void) +{ + prof_tdata_t *prof_tdata; + + /* Initialize an empty cache for this thread. */ + prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); + if (prof_tdata == NULL) + return (NULL); + + if (ckh_new(&prof_tdata->bt2cnt, PROF_CKH_MINITEMS, + prof_bt_hash, prof_bt_keycomp)) { + idalloc(prof_tdata); + return (NULL); + } + ql_new(&prof_tdata->lru_ql); + + prof_tdata->vec = imalloc(sizeof(void *) * prof_bt_max); + if (prof_tdata->vec == NULL) { + ckh_delete(&prof_tdata->bt2cnt); + idalloc(prof_tdata); + return (NULL); + } + + prof_tdata->prn_state = 0; + prof_tdata->threshold = 0; + prof_tdata->accum = 0; + + PROF_TCACHE_SET(prof_tdata); + + return (prof_tdata); +} + +static void +prof_tdata_cleanup(void *arg) +{ + prof_thr_cnt_t *cnt; + prof_tdata_t *prof_tdata = (prof_tdata_t *)arg; + + /* + * Delete the hash table. All of its contents can still be iterated + * over via the LRU. + */ + ckh_delete(&prof_tdata->bt2cnt); + + /* Iteratively merge cnt's into the global stats and delete them. */ + while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { + ql_remove(&prof_tdata->lru_ql, cnt, lru_link); + prof_ctx_merge(cnt->ctx, cnt); + idalloc(cnt); + } + + idalloc(prof_tdata->vec); + + idalloc(prof_tdata); + PROF_TCACHE_SET(NULL); +} + +void +prof_boot0(void) +{ + + memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, + sizeof(PROF_PREFIX_DEFAULT)); +} + +void +prof_boot1(void) +{ + + /* + * opt_prof and prof_promote must be in their final state before any + * arenas are initialized, so this function must be executed early. + */ + + if (opt_prof_leak && opt_prof == false) { + /* + * Enable opt_prof, but in such a way that profiles are never + * automatically dumped. + */ + opt_prof = true; + opt_prof_gdump = false; + prof_interval = 0; + } else if (opt_prof) { + if (opt_lg_prof_interval >= 0) { + prof_interval = (((uint64_t)1U) << + opt_lg_prof_interval); + } else + prof_interval = 0; + } + + prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT); +} + +bool +prof_boot2(void) +{ + + if (opt_prof) { + if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, + prof_bt_keycomp)) + return (true); + if (malloc_mutex_init(&bt2ctx_mtx)) + return (true); + if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup) + != 0) { + malloc_write( + ": Error in pthread_key_create()\n"); + abort(); + } + + prof_bt_max = (1U << opt_lg_prof_bt_max); + if (malloc_mutex_init(&prof_dump_seq_mtx)) + return (true); + + if (malloc_mutex_init(&enq_mtx)) + return (true); + enq = false; + enq_idump = false; + enq_gdump = false; + + if (atexit(prof_fdump) != 0) { + malloc_write(": Error in atexit()\n"); + if (opt_abort) + abort(); + } + } + +#ifdef JEMALLOC_PROF_LIBGCC + /* + * Cause the backtracing machinery to allocate its internal state + * before enabling profiling. + */ + _Unwind_Backtrace(prof_unwind_init_callback, NULL); +#endif + + prof_booted = true; + + return (false); +} + +/******************************************************************************/ +#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc.orig/src/rtree.c b/deps/jemalloc.orig/src/rtree.c new file mode 100644 index 00000000000..eb0ff1e24af --- /dev/null +++ b/deps/jemalloc.orig/src/rtree.c @@ -0,0 +1,46 @@ +#define JEMALLOC_RTREE_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +rtree_t * +rtree_new(unsigned bits) +{ + rtree_t *ret; + unsigned bits_per_level, height, i; + + bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; + height = bits / bits_per_level; + if (height * bits_per_level != bits) + height++; + assert(height * bits_per_level >= bits); + + ret = (rtree_t*)base_alloc(offsetof(rtree_t, level2bits) + + (sizeof(unsigned) * height)); + if (ret == NULL) + return (NULL); + memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) * + height)); + + if (malloc_mutex_init(&ret->mutex)) { + /* Leak the rtree. */ + return (NULL); + } + ret->height = height; + if (bits_per_level * height > bits) + ret->level2bits[0] = bits % bits_per_level; + else + ret->level2bits[0] = bits_per_level; + for (i = 1; i < height; i++) + ret->level2bits[i] = bits_per_level; + + ret->root = (void**)base_alloc(sizeof(void *) << ret->level2bits[0]); + if (ret->root == NULL) { + /* + * We leak the rtree here, since there's no generic base + * deallocation. + */ + return (NULL); + } + memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]); + + return (ret); +} diff --git a/deps/jemalloc.orig/src/stats.c b/deps/jemalloc.orig/src/stats.c new file mode 100644 index 00000000000..dc172e425c0 --- /dev/null +++ b/deps/jemalloc.orig/src/stats.c @@ -0,0 +1,790 @@ +#define JEMALLOC_STATS_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +#define CTL_GET(n, v, t) do { \ + size_t sz = sizeof(t); \ + xmallctl(n, v, &sz, NULL, 0); \ +} while (0) + +#define CTL_I_GET(n, v, t) do { \ + size_t mib[6]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[2] = i; \ + xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ +} while (0) + +#define CTL_J_GET(n, v, t) do { \ + size_t mib[6]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[2] = j; \ + xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ +} while (0) + +#define CTL_IJ_GET(n, v, t) do { \ + size_t mib[6]; \ + size_t miblen = sizeof(mib) / sizeof(size_t); \ + size_t sz = sizeof(t); \ + xmallctlnametomib(n, mib, &miblen); \ + mib[2] = i; \ + mib[4] = j; \ + xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ +} while (0) + +/******************************************************************************/ +/* Data. */ + +bool opt_stats_print = false; + +#ifdef JEMALLOC_STATS +size_t stats_cactive = 0; +#endif + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +#ifdef JEMALLOC_STATS +static void malloc_vcprintf(void (*write_cb)(void *, const char *), + void *cbopaque, const char *format, va_list ap); +static void stats_arena_bins_print(void (*write_cb)(void *, const char *), + void *cbopaque, unsigned i); +static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), + void *cbopaque, unsigned i); +static void stats_arena_print(void (*write_cb)(void *, const char *), + void *cbopaque, unsigned i); +#endif + +/******************************************************************************/ + +/* + * We don't want to depend on vsnprintf() for production builds, since that can + * cause unnecessary bloat for static binaries. u2s() provides minimal integer + * printing functionality, so that malloc_printf() use can be limited to + * JEMALLOC_STATS code. + */ +char * +u2s(uint64_t x, unsigned base, char *s) +{ + unsigned i; + + i = UMAX2S_BUFSIZE - 1; + s[i] = '\0'; + switch (base) { + case 10: + do { + i--; + s[i] = "0123456789"[x % (uint64_t)10]; + x /= (uint64_t)10; + } while (x > 0); + break; + case 16: + do { + i--; + s[i] = "0123456789abcdef"[x & 0xf]; + x >>= 4; + } while (x > 0); + break; + default: + do { + i--; + s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % + (uint64_t)base]; + x /= (uint64_t)base; + } while (x > 0); + } + + return (&s[i]); +} + +#ifdef JEMALLOC_STATS +static void +malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, va_list ap) +{ + char buf[4096]; + + if (write_cb == NULL) { + /* + * The caller did not provide an alternate write_cb callback + * function, so use the default one. malloc_write() is an + * inline function, so use malloc_message() directly here. + */ + write_cb = JEMALLOC_P(malloc_message); + cbopaque = NULL; + } + + vsnprintf(buf, sizeof(buf), format, ap); + write_cb(cbopaque, buf); +} + +/* + * Print to a callback function in such a way as to (hopefully) avoid memory + * allocation. + */ +JEMALLOC_ATTR(format(printf, 3, 4)) +void +malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(write_cb, cbopaque, format, ap); + va_end(ap); +} + +/* + * Print to stderr in such a way as to (hopefully) avoid memory allocation. + */ +JEMALLOC_ATTR(format(printf, 1, 2)) +void +malloc_printf(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); +} +#endif + +#ifdef JEMALLOC_STATS +static void +stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, + unsigned i) +{ + size_t pagesize; + bool config_tcache; + unsigned nbins, j, gap_start; + + CTL_GET("arenas.pagesize", &pagesize, size_t); + + CTL_GET("config.tcache", &config_tcache, bool); + if (config_tcache) { + malloc_cprintf(write_cb, cbopaque, + "bins: bin size regs pgs allocated nmalloc" + " ndalloc nrequests nfills nflushes" + " newruns reruns maxruns curruns\n"); + } else { + malloc_cprintf(write_cb, cbopaque, + "bins: bin size regs pgs allocated nmalloc" + " ndalloc newruns reruns maxruns" + " curruns\n"); + } + CTL_GET("arenas.nbins", &nbins, unsigned); + for (j = 0, gap_start = UINT_MAX; j < nbins; j++) { + uint64_t nruns; + + CTL_IJ_GET("stats.arenas.0.bins.0.nruns", &nruns, uint64_t); + if (nruns == 0) { + if (gap_start == UINT_MAX) + gap_start = j; + } else { + unsigned ntbins_, nqbins, ncbins, nsbins; + size_t reg_size, run_size, allocated; + uint32_t nregs; + uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; + uint64_t reruns; + size_t highruns, curruns; + + if (gap_start != UINT_MAX) { + if (j > gap_start + 1) { + /* Gap of more than one size class. */ + malloc_cprintf(write_cb, cbopaque, + "[%u..%u]\n", gap_start, + j - 1); + } else { + /* Gap of one size class. */ + malloc_cprintf(write_cb, cbopaque, + "[%u]\n", gap_start); + } + gap_start = UINT_MAX; + } + CTL_GET("arenas.ntbins", &ntbins_, unsigned); + CTL_GET("arenas.nqbins", &nqbins, unsigned); + CTL_GET("arenas.ncbins", &ncbins, unsigned); + CTL_GET("arenas.nsbins", &nsbins, unsigned); + CTL_J_GET("arenas.bin.0.size", ®_size, size_t); + CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); + CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); + CTL_IJ_GET("stats.arenas.0.bins.0.allocated", + &allocated, size_t); + CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc", + &nmalloc, uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc", + &ndalloc, uint64_t); + if (config_tcache) { + CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", + &nrequests, uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.nfills", + &nfills, uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.nflushes", + &nflushes, uint64_t); + } + CTL_IJ_GET("stats.arenas.0.bins.0.nreruns", &reruns, + uint64_t); + CTL_IJ_GET("stats.arenas.0.bins.0.highruns", &highruns, + size_t); + CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns, + size_t); + if (config_tcache) { + malloc_cprintf(write_cb, cbopaque, + "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 + " %12"PRIu64" %12"PRIu64" %12"PRIu64 + " %12"PRIu64" %12"PRIu64" %12"PRIu64 + " %12zu %12zu\n", + j, + j < ntbins_ ? "T" : j < ntbins_ + nqbins ? + "Q" : j < ntbins_ + nqbins + ncbins ? "C" : + "S", + reg_size, nregs, run_size / pagesize, + allocated, nmalloc, ndalloc, nrequests, + nfills, nflushes, nruns, reruns, highruns, + curruns); + } else { + malloc_cprintf(write_cb, cbopaque, + "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 + " %12"PRIu64" %12"PRIu64" %12"PRIu64 + " %12zu %12zu\n", + j, + j < ntbins_ ? "T" : j < ntbins_ + nqbins ? + "Q" : j < ntbins_ + nqbins + ncbins ? "C" : + "S", + reg_size, nregs, run_size / pagesize, + allocated, nmalloc, ndalloc, nruns, reruns, + highruns, curruns); + } + } + } + if (gap_start != UINT_MAX) { + if (j > gap_start + 1) { + /* Gap of more than one size class. */ + malloc_cprintf(write_cb, cbopaque, "[%u..%u]\n", + gap_start, j - 1); + } else { + /* Gap of one size class. */ + malloc_cprintf(write_cb, cbopaque, "[%u]\n", gap_start); + } + } +} + +static void +stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, + unsigned i) +{ + size_t pagesize, nlruns, j; + ssize_t gap_start; + + CTL_GET("arenas.pagesize", &pagesize, size_t); + + malloc_cprintf(write_cb, cbopaque, + "large: size pages nmalloc ndalloc nrequests" + " maxruns curruns\n"); + CTL_GET("arenas.nlruns", &nlruns, size_t); + for (j = 0, gap_start = -1; j < nlruns; j++) { + uint64_t nmalloc, ndalloc, nrequests; + size_t run_size, highruns, curruns; + + CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc, + uint64_t); + CTL_IJ_GET("stats.arenas.0.lruns.0.ndalloc", &ndalloc, + uint64_t); + CTL_IJ_GET("stats.arenas.0.lruns.0.nrequests", &nrequests, + uint64_t); + if (nrequests == 0) { + if (gap_start == -1) + gap_start = j; + } else { + CTL_J_GET("arenas.lrun.0.size", &run_size, size_t); + CTL_IJ_GET("stats.arenas.0.lruns.0.highruns", &highruns, + size_t); + CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns, + size_t); + if (gap_start != -1) { + malloc_cprintf(write_cb, cbopaque, "[%zu]\n", + j - gap_start); + gap_start = -1; + } + malloc_cprintf(write_cb, cbopaque, + "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64 + " %12zu %12zu\n", + run_size, run_size / pagesize, nmalloc, ndalloc, + nrequests, highruns, curruns); + } + } + if (gap_start != -1) + malloc_cprintf(write_cb, cbopaque, "[%zu]\n", j - gap_start); +} + +static void +stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, + unsigned i) +{ + unsigned nthreads; + size_t pagesize, pactive, pdirty, mapped; + uint64_t npurge, nmadvise, purged; + size_t small_allocated; + uint64_t small_nmalloc, small_ndalloc, small_nrequests; + size_t large_allocated; + uint64_t large_nmalloc, large_ndalloc, large_nrequests; + + CTL_GET("arenas.pagesize", &pagesize, size_t); + + CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned); + malloc_cprintf(write_cb, cbopaque, + "assigned threads: %u\n", nthreads); + CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t); + CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t); + CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t); + CTL_I_GET("stats.arenas.0.nmadvise", &nmadvise, uint64_t); + CTL_I_GET("stats.arenas.0.purged", &purged, uint64_t); + malloc_cprintf(write_cb, cbopaque, + "dirty pages: %zu:%zu active:dirty, %"PRIu64" sweep%s," + " %"PRIu64" madvise%s, %"PRIu64" purged\n", + pactive, pdirty, npurge, npurge == 1 ? "" : "s", + nmadvise, nmadvise == 1 ? "" : "s", purged); + + malloc_cprintf(write_cb, cbopaque, + " allocated nmalloc ndalloc nrequests\n"); + CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t); + CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t); + CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t); + CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t); + malloc_cprintf(write_cb, cbopaque, + "small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + small_allocated, small_nmalloc, small_ndalloc, small_nrequests); + CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t); + CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t); + CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t); + CTL_I_GET("stats.arenas.0.large.nrequests", &large_nrequests, uint64_t); + malloc_cprintf(write_cb, cbopaque, + "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + large_allocated, large_nmalloc, large_ndalloc, large_nrequests); + malloc_cprintf(write_cb, cbopaque, + "total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", + small_allocated + large_allocated, + small_nmalloc + large_nmalloc, + small_ndalloc + large_ndalloc, + small_nrequests + large_nrequests); + malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", + pactive * pagesize ); + CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); + malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); + + stats_arena_bins_print(write_cb, cbopaque, i); + stats_arena_lruns_print(write_cb, cbopaque, i); +} +#endif + +void +stats_print(void (*write_cb)(void *, const char *), void *cbopaque, + const char *opts) +{ + int err; + uint64_t epoch; + size_t u64sz; + char s[UMAX2S_BUFSIZE]; + bool general = true; + bool merged = true; + bool unmerged = true; + bool bins = true; + bool large = true; + + /* + * Refresh stats, in case mallctl() was called by the application. + * + * Check for OOM here, since refreshing the ctl cache can trigger + * allocation. In practice, none of the subsequent mallctl()-related + * calls in this function will cause OOM if this one succeeds. + * */ + epoch = 1; + u64sz = sizeof(uint64_t); + err = JEMALLOC_P(mallctl)("epoch", &epoch, &u64sz, &epoch, + sizeof(uint64_t)); + if (err != 0) { + if (err == EAGAIN) { + malloc_write(": Memory allocation failure in " + "mallctl(\"epoch\", ...)\n"); + return; + } + malloc_write(": Failure in mallctl(\"epoch\", " + "...)\n"); + abort(); + } + + if (write_cb == NULL) { + /* + * The caller did not provide an alternate write_cb callback + * function, so use the default one. malloc_write() is an + * inline function, so use malloc_message() directly here. + */ + write_cb = JEMALLOC_P(malloc_message); + cbopaque = NULL; + } + + if (opts != NULL) { + unsigned i; + + for (i = 0; opts[i] != '\0'; i++) { + switch (opts[i]) { + case 'g': + general = false; + break; + case 'm': + merged = false; + break; + case 'a': + unmerged = false; + break; + case 'b': + bins = false; + break; + case 'l': + large = false; + break; + default:; + } + } + } + + write_cb(cbopaque, "___ Begin jemalloc statistics ___\n"); + if (general) { + int err; + const char *cpv; + bool bv; + unsigned uv; + ssize_t ssv; + size_t sv, bsz, ssz, sssz, cpsz; + + bsz = sizeof(bool); + ssz = sizeof(size_t); + sssz = sizeof(ssize_t); + cpsz = sizeof(const char *); + + CTL_GET("version", &cpv, const char *); + write_cb(cbopaque, "Version: "); + write_cb(cbopaque, cpv); + write_cb(cbopaque, "\n"); + CTL_GET("config.debug", &bv, bool); + write_cb(cbopaque, "Assertions "); + write_cb(cbopaque, bv ? "enabled" : "disabled"); + write_cb(cbopaque, "\n"); + +#define OPT_WRITE_BOOL(n) \ + if ((err = JEMALLOC_P(mallctl)("opt."#n, &bv, &bsz, \ + NULL, 0)) == 0) { \ + write_cb(cbopaque, " opt."#n": "); \ + write_cb(cbopaque, bv ? "true" : "false"); \ + write_cb(cbopaque, "\n"); \ + } +#define OPT_WRITE_SIZE_T(n) \ + if ((err = JEMALLOC_P(mallctl)("opt."#n, &sv, &ssz, \ + NULL, 0)) == 0) { \ + write_cb(cbopaque, " opt."#n": "); \ + write_cb(cbopaque, u2s(sv, 10, s)); \ + write_cb(cbopaque, "\n"); \ + } +#define OPT_WRITE_SSIZE_T(n) \ + if ((err = JEMALLOC_P(mallctl)("opt."#n, &ssv, &sssz, \ + NULL, 0)) == 0) { \ + if (ssv >= 0) { \ + write_cb(cbopaque, " opt."#n": "); \ + write_cb(cbopaque, u2s(ssv, 10, s)); \ + } else { \ + write_cb(cbopaque, " opt."#n": -"); \ + write_cb(cbopaque, u2s(-ssv, 10, s)); \ + } \ + write_cb(cbopaque, "\n"); \ + } +#define OPT_WRITE_CHAR_P(n) \ + if ((err = JEMALLOC_P(mallctl)("opt."#n, &cpv, &cpsz, \ + NULL, 0)) == 0) { \ + write_cb(cbopaque, " opt."#n": \""); \ + write_cb(cbopaque, cpv); \ + write_cb(cbopaque, "\"\n"); \ + } + + write_cb(cbopaque, "Run-time option settings:\n"); + OPT_WRITE_BOOL(abort) + OPT_WRITE_SIZE_T(lg_qspace_max) + OPT_WRITE_SIZE_T(lg_cspace_max) + OPT_WRITE_SIZE_T(lg_chunk) + OPT_WRITE_SIZE_T(narenas) + OPT_WRITE_SSIZE_T(lg_dirty_mult) + OPT_WRITE_BOOL(stats_print) + OPT_WRITE_BOOL(junk) + OPT_WRITE_BOOL(zero) + OPT_WRITE_BOOL(sysv) + OPT_WRITE_BOOL(xmalloc) + OPT_WRITE_BOOL(tcache) + OPT_WRITE_SSIZE_T(lg_tcache_gc_sweep) + OPT_WRITE_SSIZE_T(lg_tcache_max) + OPT_WRITE_BOOL(prof) + OPT_WRITE_CHAR_P(prof_prefix) + OPT_WRITE_SIZE_T(lg_prof_bt_max) + OPT_WRITE_BOOL(prof_active) + OPT_WRITE_SSIZE_T(lg_prof_sample) + OPT_WRITE_BOOL(prof_accum) + OPT_WRITE_SSIZE_T(lg_prof_tcmax) + OPT_WRITE_SSIZE_T(lg_prof_interval) + OPT_WRITE_BOOL(prof_gdump) + OPT_WRITE_BOOL(prof_leak) + OPT_WRITE_BOOL(overcommit) + +#undef OPT_WRITE_BOOL +#undef OPT_WRITE_SIZE_T +#undef OPT_WRITE_SSIZE_T +#undef OPT_WRITE_CHAR_P + + write_cb(cbopaque, "CPUs: "); + write_cb(cbopaque, u2s(ncpus, 10, s)); + write_cb(cbopaque, "\n"); + + CTL_GET("arenas.narenas", &uv, unsigned); + write_cb(cbopaque, "Max arenas: "); + write_cb(cbopaque, u2s(uv, 10, s)); + write_cb(cbopaque, "\n"); + + write_cb(cbopaque, "Pointer size: "); + write_cb(cbopaque, u2s(sizeof(void *), 10, s)); + write_cb(cbopaque, "\n"); + + CTL_GET("arenas.quantum", &sv, size_t); + write_cb(cbopaque, "Quantum size: "); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "\n"); + + CTL_GET("arenas.cacheline", &sv, size_t); + write_cb(cbopaque, "Cacheline size (assumed): "); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "\n"); + + CTL_GET("arenas.subpage", &sv, size_t); + write_cb(cbopaque, "Subpage spacing: "); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "\n"); + + if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz, + NULL, 0)) == 0) { + write_cb(cbopaque, "Tiny 2^n-spaced sizes: ["); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, ".."); + + CTL_GET("arenas.tspace_max", &sv, size_t); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "]\n"); + } + + CTL_GET("arenas.qspace_min", &sv, size_t); + write_cb(cbopaque, "Quantum-spaced sizes: ["); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, ".."); + CTL_GET("arenas.qspace_max", &sv, size_t); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "]\n"); + + CTL_GET("arenas.cspace_min", &sv, size_t); + write_cb(cbopaque, "Cacheline-spaced sizes: ["); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, ".."); + CTL_GET("arenas.cspace_max", &sv, size_t); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "]\n"); + + CTL_GET("arenas.sspace_min", &sv, size_t); + write_cb(cbopaque, "Subpage-spaced sizes: ["); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, ".."); + CTL_GET("arenas.sspace_max", &sv, size_t); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "]\n"); + + CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t); + if (ssv >= 0) { + write_cb(cbopaque, + "Min active:dirty page ratio per arena: "); + write_cb(cbopaque, u2s((1U << ssv), 10, s)); + write_cb(cbopaque, ":1\n"); + } else { + write_cb(cbopaque, + "Min active:dirty page ratio per arena: N/A\n"); + } + if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv, + &ssz, NULL, 0)) == 0) { + write_cb(cbopaque, + "Maximum thread-cached size class: "); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, "\n"); + } + if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv, + &ssz, NULL, 0)) == 0) { + size_t tcache_gc_sweep = (1U << ssv); + bool tcache_enabled; + CTL_GET("opt.tcache", &tcache_enabled, bool); + write_cb(cbopaque, "Thread cache GC sweep interval: "); + write_cb(cbopaque, tcache_enabled && ssv >= 0 ? + u2s(tcache_gc_sweep, 10, s) : "N/A"); + write_cb(cbopaque, "\n"); + } + if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0)) + == 0 && bv) { + CTL_GET("opt.lg_prof_bt_max", &sv, size_t); + write_cb(cbopaque, "Maximum profile backtrace depth: "); + write_cb(cbopaque, u2s((1U << sv), 10, s)); + write_cb(cbopaque, "\n"); + + CTL_GET("opt.lg_prof_tcmax", &ssv, ssize_t); + write_cb(cbopaque, + "Maximum per thread backtrace cache: "); + if (ssv >= 0) { + write_cb(cbopaque, u2s((1U << ssv), 10, s)); + write_cb(cbopaque, " (2^"); + write_cb(cbopaque, u2s(ssv, 10, s)); + write_cb(cbopaque, ")\n"); + } else + write_cb(cbopaque, "N/A\n"); + + CTL_GET("opt.lg_prof_sample", &sv, size_t); + write_cb(cbopaque, "Average profile sample interval: "); + write_cb(cbopaque, u2s((((uint64_t)1U) << sv), 10, s)); + write_cb(cbopaque, " (2^"); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, ")\n"); + + CTL_GET("opt.lg_prof_interval", &ssv, ssize_t); + write_cb(cbopaque, "Average profile dump interval: "); + if (ssv >= 0) { + write_cb(cbopaque, u2s((((uint64_t)1U) << ssv), + 10, s)); + write_cb(cbopaque, " (2^"); + write_cb(cbopaque, u2s(ssv, 10, s)); + write_cb(cbopaque, ")\n"); + } else + write_cb(cbopaque, "N/A\n"); + } + CTL_GET("arenas.chunksize", &sv, size_t); + write_cb(cbopaque, "Chunk size: "); + write_cb(cbopaque, u2s(sv, 10, s)); + CTL_GET("opt.lg_chunk", &sv, size_t); + write_cb(cbopaque, " (2^"); + write_cb(cbopaque, u2s(sv, 10, s)); + write_cb(cbopaque, ")\n"); + } + +#ifdef JEMALLOC_STATS + { + int err; + size_t sszp, ssz; + size_t *cactive; + size_t allocated, active, mapped; + size_t chunks_current, chunks_high, swap_avail; + uint64_t chunks_total; + size_t huge_allocated; + uint64_t huge_nmalloc, huge_ndalloc; + + sszp = sizeof(size_t *); + ssz = sizeof(size_t); + + CTL_GET("stats.cactive", &cactive, size_t *); + CTL_GET("stats.allocated", &allocated, size_t); + CTL_GET("stats.active", &active, size_t); + CTL_GET("stats.mapped", &mapped, size_t); + malloc_cprintf(write_cb, cbopaque, + "Allocated: %zu, active: %zu, mapped: %zu\n", + allocated, active, mapped); + malloc_cprintf(write_cb, cbopaque, + "Current active ceiling: %zu\n", atomic_read_z(cactive)); + + /* Print chunk stats. */ + CTL_GET("stats.chunks.total", &chunks_total, uint64_t); + CTL_GET("stats.chunks.high", &chunks_high, size_t); + CTL_GET("stats.chunks.current", &chunks_current, size_t); + if ((err = JEMALLOC_P(mallctl)("swap.avail", &swap_avail, &ssz, + NULL, 0)) == 0) { + size_t lg_chunk; + + malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " + "highchunks curchunks swap_avail\n"); + CTL_GET("opt.lg_chunk", &lg_chunk, size_t); + malloc_cprintf(write_cb, cbopaque, + " %13"PRIu64"%13zu%13zu%13zu\n", + chunks_total, chunks_high, chunks_current, + swap_avail << lg_chunk); + } else { + malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " + "highchunks curchunks\n"); + malloc_cprintf(write_cb, cbopaque, + " %13"PRIu64"%13zu%13zu\n", + chunks_total, chunks_high, chunks_current); + } + + /* Print huge stats. */ + CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t); + CTL_GET("stats.huge.ndalloc", &huge_ndalloc, uint64_t); + CTL_GET("stats.huge.allocated", &huge_allocated, size_t); + malloc_cprintf(write_cb, cbopaque, + "huge: nmalloc ndalloc allocated\n"); + malloc_cprintf(write_cb, cbopaque, + " %12"PRIu64" %12"PRIu64" %12zu\n", + huge_nmalloc, huge_ndalloc, huge_allocated); + + if (merged) { + unsigned narenas; + + CTL_GET("arenas.narenas", &narenas, unsigned); + { + bool initialized[narenas]; + size_t isz; + unsigned i, ninitialized; + + isz = sizeof(initialized); + xmallctl("arenas.initialized", initialized, + &isz, NULL, 0); + for (i = ninitialized = 0; i < narenas; i++) { + if (initialized[i]) + ninitialized++; + } + + if (ninitialized > 1 || unmerged == false) { + /* Print merged arena stats. */ + malloc_cprintf(write_cb, cbopaque, + "\nMerged arenas stats:\n"); + stats_arena_print(write_cb, cbopaque, + narenas); + } + } + } + + if (unmerged) { + unsigned narenas; + + /* Print stats for each arena. */ + + CTL_GET("arenas.narenas", &narenas, unsigned); + { + bool initialized[narenas]; + size_t isz; + unsigned i; + + isz = sizeof(initialized); + xmallctl("arenas.initialized", initialized, + &isz, NULL, 0); + + for (i = 0; i < narenas; i++) { + if (initialized[i]) { + malloc_cprintf(write_cb, + cbopaque, + "\narenas[%u]:\n", i); + stats_arena_print(write_cb, + cbopaque, i); + } + } + } + } + } +#endif /* #ifdef JEMALLOC_STATS */ + write_cb(cbopaque, "--- End jemalloc statistics ---\n"); +} diff --git a/deps/jemalloc.orig/src/tcache.c b/deps/jemalloc.orig/src/tcache.c new file mode 100644 index 00000000000..31c329e1613 --- /dev/null +++ b/deps/jemalloc.orig/src/tcache.c @@ -0,0 +1,480 @@ +#define JEMALLOC_TCACHE_C_ +#include "jemalloc/internal/jemalloc_internal.h" +#ifdef JEMALLOC_TCACHE +/******************************************************************************/ +/* Data. */ + +bool opt_tcache = true; +ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; +ssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; + +tcache_bin_info_t *tcache_bin_info; +static unsigned stack_nelms; /* Total stack elms per tcache. */ + +/* Map of thread-specific caches. */ +#ifndef NO_TLS +__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); +#endif + +/* + * Same contents as tcache, but initialized such that the TSD destructor is + * called when a thread exits, so that the cache can be cleaned up. + */ +pthread_key_t tcache_tsd; + +size_t nhbins; +size_t tcache_maxclass; +unsigned tcache_gc_incr; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void tcache_thread_cleanup(void *arg); + +/******************************************************************************/ + +void * +tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) +{ + void *ret; + + arena_tcache_fill_small(tcache->arena, tbin, binind +#ifdef JEMALLOC_PROF + , tcache->prof_accumbytes +#endif + ); +#ifdef JEMALLOC_PROF + tcache->prof_accumbytes = 0; +#endif + ret = tcache_alloc_easy(tbin); + + return (ret); +} + +void +tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache_t *tcache +#endif + ) +{ + void *ptr; + unsigned i, nflush, ndeferred; +#ifdef JEMALLOC_STATS + bool merged_stats = false; +#endif + + assert(binind < nbins); + assert(rem <= tbin->ncached); + + for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { + /* Lock the arena bin associated with the first object. */ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( + tbin->avail[0]); + arena_t *arena = chunk->arena; + arena_bin_t *bin = &arena->bins[binind]; + +#ifdef JEMALLOC_PROF + if (arena == tcache->arena) { + malloc_mutex_lock(&arena->lock); + arena_prof_accum(arena, tcache->prof_accumbytes); + malloc_mutex_unlock(&arena->lock); + tcache->prof_accumbytes = 0; + } +#endif + + malloc_mutex_lock(&bin->lock); +#ifdef JEMALLOC_STATS + if (arena == tcache->arena) { + assert(merged_stats == false); + merged_stats = true; + bin->stats.nflushes++; + bin->stats.nrequests += tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; + } +#endif + ndeferred = 0; + for (i = 0; i < nflush; i++) { + ptr = tbin->avail[i]; + assert(ptr != NULL); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk->arena == arena) { + size_t pageind = ((uintptr_t)ptr - + (uintptr_t)chunk) >> PAGE_SHIFT; + arena_chunk_map_t *mapelm = + &chunk->map[pageind-map_bias]; + arena_dalloc_bin(arena, chunk, ptr, mapelm); + } else { + /* + * This object was allocated via a different + * arena bin than the one that is currently + * locked. Stash the object, so that it can be + * handled in a future pass. + */ + tbin->avail[ndeferred] = ptr; + ndeferred++; + } + } + malloc_mutex_unlock(&bin->lock); + } +#ifdef JEMALLOC_STATS + if (merged_stats == false) { + /* + * The flush loop didn't happen to flush to this thread's + * arena, so the stats didn't get merged. Manually do so now. + */ + arena_bin_t *bin = &tcache->arena->bins[binind]; + malloc_mutex_lock(&bin->lock); + bin->stats.nflushes++; + bin->stats.nrequests += tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; + malloc_mutex_unlock(&bin->lock); + } +#endif + + memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], + rem * sizeof(void *)); + tbin->ncached = rem; + if ((int)tbin->ncached < tbin->low_water) + tbin->low_water = tbin->ncached; +} + +void +tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache_t *tcache +#endif + ) +{ + void *ptr; + unsigned i, nflush, ndeferred; +#ifdef JEMALLOC_STATS + bool merged_stats = false; +#endif + + assert(binind < nhbins); + assert(rem <= tbin->ncached); + + for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { + /* Lock the arena associated with the first object. */ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( + tbin->avail[0]); + arena_t *arena = chunk->arena; + + malloc_mutex_lock(&arena->lock); +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + if (arena == tcache->arena) { +#endif +#ifdef JEMALLOC_PROF + arena_prof_accum(arena, tcache->prof_accumbytes); + tcache->prof_accumbytes = 0; +#endif +#ifdef JEMALLOC_STATS + merged_stats = true; + arena->stats.nrequests_large += tbin->tstats.nrequests; + arena->stats.lstats[binind - nbins].nrequests += + tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; +#endif +#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + } +#endif + ndeferred = 0; + for (i = 0; i < nflush; i++) { + ptr = tbin->avail[i]; + assert(ptr != NULL); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk->arena == arena) + arena_dalloc_large(arena, chunk, ptr); + else { + /* + * This object was allocated via a different + * arena than the one that is currently locked. + * Stash the object, so that it can be handled + * in a future pass. + */ + tbin->avail[ndeferred] = ptr; + ndeferred++; + } + } + malloc_mutex_unlock(&arena->lock); + } +#ifdef JEMALLOC_STATS + if (merged_stats == false) { + /* + * The flush loop didn't happen to flush to this thread's + * arena, so the stats didn't get merged. Manually do so now. + */ + arena_t *arena = tcache->arena; + malloc_mutex_lock(&arena->lock); + arena->stats.nrequests_large += tbin->tstats.nrequests; + arena->stats.lstats[binind - nbins].nrequests += + tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; + malloc_mutex_unlock(&arena->lock); + } +#endif + + memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], + rem * sizeof(void *)); + tbin->ncached = rem; + if ((int)tbin->ncached < tbin->low_water) + tbin->low_water = tbin->ncached; +} + +tcache_t * +tcache_create(arena_t *arena) +{ + tcache_t *tcache; + size_t size, stack_offset; + unsigned i; + + size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins); + /* Naturally align the pointer stacks. */ + size = PTR_CEILING(size); + stack_offset = size; + size += stack_nelms * sizeof(void *); + /* + * Round up to the nearest multiple of the cacheline size, in order to + * avoid the possibility of false cacheline sharing. + * + * That this works relies on the same logic as in ipalloc(), but we + * cannot directly call ipalloc() here due to tcache bootstrapping + * issues. + */ + size = (size + CACHELINE_MASK) & (-CACHELINE); + + if (size <= small_maxclass) + tcache = (tcache_t *)arena_malloc_small(arena, size, true); + else if (size <= tcache_maxclass) + tcache = (tcache_t *)arena_malloc_large(arena, size, true); + else + tcache = (tcache_t *)icalloc(size); + + if (tcache == NULL) + return (NULL); + +#ifdef JEMALLOC_STATS + /* Link into list of extant tcaches. */ + malloc_mutex_lock(&arena->lock); + ql_elm_new(tcache, link); + ql_tail_insert(&arena->tcache_ql, tcache, link); + malloc_mutex_unlock(&arena->lock); +#endif + + tcache->arena = arena; + assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); + for (i = 0; i < nhbins; i++) { + tcache->tbins[i].lg_fill_div = 1; + tcache->tbins[i].avail = (void **)((uintptr_t)tcache + + (uintptr_t)stack_offset); + stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); + } + + TCACHE_SET(tcache); + + return (tcache); +} + +void +tcache_destroy(tcache_t *tcache) +{ + unsigned i; + size_t tcache_size; + +#ifdef JEMALLOC_STATS + /* Unlink from list of extant tcaches. */ + malloc_mutex_lock(&tcache->arena->lock); + ql_remove(&tcache->arena->tcache_ql, tcache, link); + malloc_mutex_unlock(&tcache->arena->lock); + tcache_stats_merge(tcache, tcache->arena); +#endif + + for (i = 0; i < nbins; i++) { + tcache_bin_t *tbin = &tcache->tbins[i]; + tcache_bin_flush_small(tbin, i, 0 +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache +#endif + ); + +#ifdef JEMALLOC_STATS + if (tbin->tstats.nrequests != 0) { + arena_t *arena = tcache->arena; + arena_bin_t *bin = &arena->bins[i]; + malloc_mutex_lock(&bin->lock); + bin->stats.nrequests += tbin->tstats.nrequests; + malloc_mutex_unlock(&bin->lock); + } +#endif + } + + for (; i < nhbins; i++) { + tcache_bin_t *tbin = &tcache->tbins[i]; + tcache_bin_flush_large(tbin, i, 0 +#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) + , tcache +#endif + ); + +#ifdef JEMALLOC_STATS + if (tbin->tstats.nrequests != 0) { + arena_t *arena = tcache->arena; + malloc_mutex_lock(&arena->lock); + arena->stats.nrequests_large += tbin->tstats.nrequests; + arena->stats.lstats[i - nbins].nrequests += + tbin->tstats.nrequests; + malloc_mutex_unlock(&arena->lock); + } +#endif + } + +#ifdef JEMALLOC_PROF + if (tcache->prof_accumbytes > 0) { + malloc_mutex_lock(&tcache->arena->lock); + arena_prof_accum(tcache->arena, tcache->prof_accumbytes); + malloc_mutex_unlock(&tcache->arena->lock); + } +#endif + + tcache_size = arena_salloc(tcache); + if (tcache_size <= small_maxclass) { + arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); + arena_t *arena = chunk->arena; + size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> + PAGE_SHIFT; + arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; + arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + + (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << + PAGE_SHIFT)); + arena_bin_t *bin = run->bin; + + malloc_mutex_lock(&bin->lock); + arena_dalloc_bin(arena, chunk, tcache, mapelm); + malloc_mutex_unlock(&bin->lock); + } else if (tcache_size <= tcache_maxclass) { + arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); + arena_t *arena = chunk->arena; + + malloc_mutex_lock(&arena->lock); + arena_dalloc_large(arena, chunk, tcache); + malloc_mutex_unlock(&arena->lock); + } else + idalloc(tcache); +} + +static void +tcache_thread_cleanup(void *arg) +{ + tcache_t *tcache = (tcache_t *)arg; + + if (tcache == (void *)(uintptr_t)1) { + /* + * The previous time this destructor was called, we set the key + * to 1 so that other destructors wouldn't cause re-creation of + * the tcache. This time, do nothing, so that the destructor + * will not be called again. + */ + } else if (tcache == (void *)(uintptr_t)2) { + /* + * Another destructor called an allocator function after this + * destructor was called. Reset tcache to 1 in order to + * receive another callback. + */ + TCACHE_SET((uintptr_t)1); + } else if (tcache != NULL) { + assert(tcache != (void *)(uintptr_t)1); + tcache_destroy(tcache); + TCACHE_SET((uintptr_t)1); + } +} + +#ifdef JEMALLOC_STATS +void +tcache_stats_merge(tcache_t *tcache, arena_t *arena) +{ + unsigned i; + + /* Merge and reset tcache stats. */ + for (i = 0; i < nbins; i++) { + arena_bin_t *bin = &arena->bins[i]; + tcache_bin_t *tbin = &tcache->tbins[i]; + malloc_mutex_lock(&bin->lock); + bin->stats.nrequests += tbin->tstats.nrequests; + malloc_mutex_unlock(&bin->lock); + tbin->tstats.nrequests = 0; + } + + for (; i < nhbins; i++) { + malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; + tcache_bin_t *tbin = &tcache->tbins[i]; + arena->stats.nrequests_large += tbin->tstats.nrequests; + lstats->nrequests += tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; + } +} +#endif + +bool +tcache_boot(void) +{ + + if (opt_tcache) { + unsigned i; + + /* + * If necessary, clamp opt_lg_tcache_max, now that + * small_maxclass and arena_maxclass are known. + */ + if (opt_lg_tcache_max < 0 || (1U << + opt_lg_tcache_max) < small_maxclass) + tcache_maxclass = small_maxclass; + else if ((1U << opt_lg_tcache_max) > arena_maxclass) + tcache_maxclass = arena_maxclass; + else + tcache_maxclass = (1U << opt_lg_tcache_max); + + nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); + + /* Initialize tcache_bin_info. */ + tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * + sizeof(tcache_bin_info_t)); + if (tcache_bin_info == NULL) + return (true); + stack_nelms = 0; + for (i = 0; i < nbins; i++) { + if ((arena_bin_info[i].nregs << 1) <= + TCACHE_NSLOTS_SMALL_MAX) { + tcache_bin_info[i].ncached_max = + (arena_bin_info[i].nregs << 1); + } else { + tcache_bin_info[i].ncached_max = + TCACHE_NSLOTS_SMALL_MAX; + } + stack_nelms += tcache_bin_info[i].ncached_max; + } + for (; i < nhbins; i++) { + tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; + stack_nelms += tcache_bin_info[i].ncached_max; + } + + /* Compute incremental GC event threshold. */ + if (opt_lg_tcache_gc_sweep >= 0) { + tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / + nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == + 0) ? 0 : 1); + } else + tcache_gc_incr = 0; + + if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != + 0) { + malloc_write( + ": Error in pthread_key_create()\n"); + abort(); + } + } + + return (false); +} +/******************************************************************************/ +#endif /* JEMALLOC_TCACHE */ diff --git a/deps/jemalloc.orig/src/zone.c b/deps/jemalloc.orig/src/zone.c new file mode 100644 index 00000000000..2c1b2318ef4 --- /dev/null +++ b/deps/jemalloc.orig/src/zone.c @@ -0,0 +1,354 @@ +#include "jemalloc/internal/jemalloc_internal.h" +#ifndef JEMALLOC_ZONE +# error "This source file is for zones on Darwin (OS X)." +#endif + +/******************************************************************************/ +/* Data. */ + +static malloc_zone_t zone, szone; +static struct malloc_introspection_t zone_introspect, ozone_introspect; + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static size_t zone_size(malloc_zone_t *zone, void *ptr); +static void *zone_malloc(malloc_zone_t *zone, size_t size); +static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); +static void *zone_valloc(malloc_zone_t *zone, size_t size); +static void zone_free(malloc_zone_t *zone, void *ptr); +static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); +#if (JEMALLOC_ZONE_VERSION >= 6) +static void *zone_memalign(malloc_zone_t *zone, size_t alignment, + size_t size); +static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, + size_t size); +#endif +static void *zone_destroy(malloc_zone_t *zone); +static size_t zone_good_size(malloc_zone_t *zone, size_t size); +static void zone_force_lock(malloc_zone_t *zone); +static void zone_force_unlock(malloc_zone_t *zone); +static size_t ozone_size(malloc_zone_t *zone, void *ptr); +static void ozone_free(malloc_zone_t *zone, void *ptr); +static void *ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size); +static unsigned ozone_batch_malloc(malloc_zone_t *zone, size_t size, + void **results, unsigned num_requested); +static void ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, + unsigned num); +#if (JEMALLOC_ZONE_VERSION >= 6) +static void ozone_free_definite_size(malloc_zone_t *zone, void *ptr, + size_t size); +#endif +static void ozone_force_lock(malloc_zone_t *zone); +static void ozone_force_unlock(malloc_zone_t *zone); + +/******************************************************************************/ +/* + * Functions. + */ + +static size_t +zone_size(malloc_zone_t *zone, void *ptr) +{ + + /* + * There appear to be places within Darwin (such as setenv(3)) that + * cause calls to this function with pointers that *no* zone owns. If + * we knew that all pointers were owned by *some* zone, we could split + * our zone into two parts, and use one as the default allocator and + * the other as the default deallocator/reallocator. Since that will + * not work in practice, we must check all pointers to assure that they + * reside within a mapped chunk before determining size. + */ + return (ivsalloc(ptr)); +} + +static void * +zone_malloc(malloc_zone_t *zone, size_t size) +{ + + return (JEMALLOC_P(malloc)(size)); +} + +static void * +zone_calloc(malloc_zone_t *zone, size_t num, size_t size) +{ + + return (JEMALLOC_P(calloc)(num, size)); +} + +static void * +zone_valloc(malloc_zone_t *zone, size_t size) +{ + void *ret = NULL; /* Assignment avoids useless compiler warning. */ + + JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size); + + return (ret); +} + +static void +zone_free(malloc_zone_t *zone, void *ptr) +{ + + JEMALLOC_P(free)(ptr); +} + +static void * +zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) +{ + + return (JEMALLOC_P(realloc)(ptr, size)); +} + +#if (JEMALLOC_ZONE_VERSION >= 6) +static void * +zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) +{ + void *ret = NULL; /* Assignment avoids useless compiler warning. */ + + JEMALLOC_P(posix_memalign)(&ret, alignment, size); + + return (ret); +} + +static void +zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) +{ + + assert(ivsalloc(ptr) == size); + JEMALLOC_P(free)(ptr); +} +#endif + +static void * +zone_destroy(malloc_zone_t *zone) +{ + + /* This function should never be called. */ + assert(false); + return (NULL); +} + +static size_t +zone_good_size(malloc_zone_t *zone, size_t size) +{ + size_t ret; + void *p; + + /* + * Actually create an object of the appropriate size, then find out + * how large it could have been without moving up to the next size + * class. + */ + p = JEMALLOC_P(malloc)(size); + if (p != NULL) { + ret = isalloc(p); + JEMALLOC_P(free)(p); + } else + ret = size; + + return (ret); +} + +static void +zone_force_lock(malloc_zone_t *zone) +{ + + if (isthreaded) + jemalloc_prefork(); +} + +static void +zone_force_unlock(malloc_zone_t *zone) +{ + + if (isthreaded) + jemalloc_postfork(); +} + +malloc_zone_t * +create_zone(void) +{ + + zone.size = (void *)zone_size; + zone.malloc = (void *)zone_malloc; + zone.calloc = (void *)zone_calloc; + zone.valloc = (void *)zone_valloc; + zone.free = (void *)zone_free; + zone.realloc = (void *)zone_realloc; + zone.destroy = (void *)zone_destroy; + zone.zone_name = "jemalloc_zone"; + zone.batch_malloc = NULL; + zone.batch_free = NULL; + zone.introspect = &zone_introspect; + zone.version = JEMALLOC_ZONE_VERSION; +#if (JEMALLOC_ZONE_VERSION >= 6) + zone.memalign = zone_memalign; + zone.free_definite_size = zone_free_definite_size; +#endif + + zone_introspect.enumerator = NULL; + zone_introspect.good_size = (void *)zone_good_size; + zone_introspect.check = NULL; + zone_introspect.print = NULL; + zone_introspect.log = NULL; + zone_introspect.force_lock = (void *)zone_force_lock; + zone_introspect.force_unlock = (void *)zone_force_unlock; + zone_introspect.statistics = NULL; +#if (JEMALLOC_ZONE_VERSION >= 6) + zone_introspect.zone_locked = NULL; +#endif + + return (&zone); +} + +static size_t +ozone_size(malloc_zone_t *zone, void *ptr) +{ + size_t ret; + + ret = ivsalloc(ptr); + if (ret == 0) + ret = szone.size(zone, ptr); + + return (ret); +} + +static void +ozone_free(malloc_zone_t *zone, void *ptr) +{ + + if (ivsalloc(ptr) != 0) + JEMALLOC_P(free)(ptr); + else { + size_t size = szone.size(zone, ptr); + if (size != 0) + (szone.free)(zone, ptr); + } +} + +static void * +ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size) +{ + size_t oldsize; + + if (ptr == NULL) + return (JEMALLOC_P(malloc)(size)); + + oldsize = ivsalloc(ptr); + if (oldsize != 0) + return (JEMALLOC_P(realloc)(ptr, size)); + else { + oldsize = szone.size(zone, ptr); + if (oldsize == 0) + return (JEMALLOC_P(malloc)(size)); + else { + void *ret = JEMALLOC_P(malloc)(size); + if (ret != NULL) { + memcpy(ret, ptr, (oldsize < size) ? oldsize : + size); + (szone.free)(zone, ptr); + } + return (ret); + } + } +} + +static unsigned +ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, + unsigned num_requested) +{ + + /* Don't bother implementing this interface, since it isn't required. */ + return (0); +} + +static void +ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) +{ + unsigned i; + + for (i = 0; i < num; i++) + ozone_free(zone, to_be_freed[i]); +} + +#if (JEMALLOC_ZONE_VERSION >= 6) +static void +ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) +{ + + if (ivsalloc(ptr) != 0) { + assert(ivsalloc(ptr) == size); + JEMALLOC_P(free)(ptr); + } else { + assert(size == szone.size(zone, ptr)); + szone.free_definite_size(zone, ptr, size); + } +} +#endif + +static void +ozone_force_lock(malloc_zone_t *zone) +{ + + /* jemalloc locking is taken care of by the normal jemalloc zone. */ + szone.introspect->force_lock(zone); +} + +static void +ozone_force_unlock(malloc_zone_t *zone) +{ + + /* jemalloc locking is taken care of by the normal jemalloc zone. */ + szone.introspect->force_unlock(zone); +} + +/* + * Overlay the default scalable zone (szone) such that existing allocations are + * drained, and further allocations come from jemalloc. This is necessary + * because Core Foundation directly accesses and uses the szone before the + * jemalloc library is even loaded. + */ +void +szone2ozone(malloc_zone_t *zone) +{ + + /* + * Stash a copy of the original szone so that we can call its + * functions as needed. Note that the internally, the szone stores its + * bookkeeping data structures immediately following the malloc_zone_t + * header, so when calling szone functions, we need to pass a pointer + * to the original zone structure. + */ + memcpy(&szone, zone, sizeof(malloc_zone_t)); + + zone->size = (void *)ozone_size; + zone->malloc = (void *)zone_malloc; + zone->calloc = (void *)zone_calloc; + zone->valloc = (void *)zone_valloc; + zone->free = (void *)ozone_free; + zone->realloc = (void *)ozone_realloc; + zone->destroy = (void *)zone_destroy; + zone->zone_name = "jemalloc_ozone"; + zone->batch_malloc = ozone_batch_malloc; + zone->batch_free = ozone_batch_free; + zone->introspect = &ozone_introspect; + zone->version = JEMALLOC_ZONE_VERSION; +#if (JEMALLOC_ZONE_VERSION >= 6) + zone->memalign = zone_memalign; + zone->free_definite_size = ozone_free_definite_size; +#endif + + ozone_introspect.enumerator = NULL; + ozone_introspect.good_size = (void *)zone_good_size; + ozone_introspect.check = NULL; + ozone_introspect.print = NULL; + ozone_introspect.log = NULL; + ozone_introspect.force_lock = (void *)ozone_force_lock; + ozone_introspect.force_unlock = (void *)ozone_force_unlock; + ozone_introspect.statistics = NULL; +#if (JEMALLOC_ZONE_VERSION >= 6) + ozone_introspect.zone_locked = NULL; +#endif +} diff --git a/deps/jemalloc.orig/test/allocated.c b/deps/jemalloc.orig/test/allocated.c new file mode 100644 index 00000000000..b1e40e4720e --- /dev/null +++ b/deps/jemalloc.orig/test/allocated.c @@ -0,0 +1,142 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +void * +thread_start(void *arg) +{ + int err; + void *p; + uint64_t a0, a1, d0, d1; + uint64_t *ap0, *ap1, *dp0, *dp1; + size_t sz, usize; + + sz = sizeof(a0); + if ((err = JEMALLOC_P(mallctl)("thread.allocated", &a0, &sz, NULL, + 0))) { + if (err == ENOENT) { +#ifdef JEMALLOC_STATS + assert(false); +#endif + goto RETURN; + } + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + exit(1); + } + sz = sizeof(ap0); + if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL, + 0))) { + if (err == ENOENT) { +#ifdef JEMALLOC_STATS + assert(false); +#endif + goto RETURN; + } + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + exit(1); + } + assert(*ap0 == a0); + + sz = sizeof(d0); + if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL, + 0))) { + if (err == ENOENT) { +#ifdef JEMALLOC_STATS + assert(false); +#endif + goto RETURN; + } + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + exit(1); + } + sz = sizeof(dp0); + if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL, + 0))) { + if (err == ENOENT) { +#ifdef JEMALLOC_STATS + assert(false); +#endif + goto RETURN; + } + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + exit(1); + } + assert(*dp0 == d0); + + p = JEMALLOC_P(malloc)(1); + if (p == NULL) { + fprintf(stderr, "%s(): Error in malloc()\n", __func__); + exit(1); + } + + sz = sizeof(a1); + JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0); + sz = sizeof(ap1); + JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0); + assert(*ap1 == a1); + assert(ap0 == ap1); + + usize = JEMALLOC_P(malloc_usable_size)(p); + assert(a0 + usize <= a1); + + JEMALLOC_P(free)(p); + + sz = sizeof(d1); + JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0); + sz = sizeof(dp1); + JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0); + assert(*dp1 == d1); + assert(dp0 == dp1); + + assert(d0 + usize <= d1); + +RETURN: + return (NULL); +} + +int +main(void) +{ + int ret = 0; + pthread_t thread; + + fprintf(stderr, "Test begin\n"); + + thread_start(NULL); + + if (pthread_create(&thread, NULL, thread_start, NULL) + != 0) { + fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); + ret = 1; + goto RETURN; + } + pthread_join(thread, (void *)&ret); + + thread_start(NULL); + + if (pthread_create(&thread, NULL, thread_start, NULL) + != 0) { + fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); + ret = 1; + goto RETURN; + } + pthread_join(thread, (void *)&ret); + + thread_start(NULL); + +RETURN: + fprintf(stderr, "Test end\n"); + return (ret); +} diff --git a/deps/jemalloc.orig/test/allocated.exp b/deps/jemalloc.orig/test/allocated.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc.orig/test/allocated.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/deps/jemalloc.orig/test/allocm.c b/deps/jemalloc.orig/test/allocm.c new file mode 100644 index 00000000000..59d0002e7be --- /dev/null +++ b/deps/jemalloc.orig/test/allocm.c @@ -0,0 +1,133 @@ +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +#define CHUNK 0x400000 +/* #define MAXALIGN ((size_t)0x80000000000LLU) */ +#define MAXALIGN ((size_t)0x2000000LLU) +#define NITER 4 + +int +main(void) +{ + int r; + void *p; + size_t sz, alignment, total, tsz; + unsigned i; + void *ps[NITER]; + + fprintf(stderr, "Test begin\n"); + + sz = 0; + r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); + if (r != ALLOCM_SUCCESS) { + fprintf(stderr, "Unexpected allocm() error\n"); + abort(); + } + if (sz < 42) + fprintf(stderr, "Real size smaller than expected\n"); + if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected dallocm() error\n"); + + r = JEMALLOC_P(allocm)(&p, NULL, 42, 0); + if (r != ALLOCM_SUCCESS) { + fprintf(stderr, "Unexpected allocm() error\n"); + abort(); + } + if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected dallocm() error\n"); + + r = JEMALLOC_P(allocm)(&p, NULL, 42, ALLOCM_ZERO); + if (r != ALLOCM_SUCCESS) { + fprintf(stderr, "Unexpected allocm() error\n"); + abort(); + } + if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected dallocm() error\n"); + +#if LG_SIZEOF_PTR == 3 + alignment = 0x8000000000000000LLU; + sz = 0x8000000000000000LLU; +#else + alignment = 0x80000000LU; + sz = 0x80000000LU; +#endif + r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); + if (r == ALLOCM_SUCCESS) { + fprintf(stderr, + "Expected error for allocm(&p, %zu, 0x%x)\n", + sz, ALLOCM_ALIGN(alignment)); + } + +#if LG_SIZEOF_PTR == 3 + alignment = 0x4000000000000000LLU; + sz = 0x8400000000000001LLU; +#else + alignment = 0x40000000LU; + sz = 0x84000001LU; +#endif + r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); + if (r == ALLOCM_SUCCESS) { + fprintf(stderr, + "Expected error for allocm(&p, %zu, 0x%x)\n", + sz, ALLOCM_ALIGN(alignment)); + } + + alignment = 0x10LLU; +#if LG_SIZEOF_PTR == 3 + sz = 0xfffffffffffffff0LLU; +#else + sz = 0xfffffff0LU; +#endif + r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); + if (r == ALLOCM_SUCCESS) { + fprintf(stderr, + "Expected error for allocm(&p, %zu, 0x%x)\n", + sz, ALLOCM_ALIGN(alignment)); + } + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + fprintf(stderr, "Alignment: %zu\n", alignment); + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + r = JEMALLOC_P(allocm)(&ps[i], NULL, sz, + ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); + if (r != ALLOCM_SUCCESS) { + fprintf(stderr, + "Error for size %zu (0x%zx): %d\n", + sz, sz, r); + exit(1); + } + if ((uintptr_t)p & (alignment-1)) { + fprintf(stderr, + "%p inadequately aligned for" + " alignment: %zu\n", p, alignment); + } + JEMALLOC_P(sallocm)(ps[i], &tsz, 0); + total += tsz; + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + JEMALLOC_P(dallocm)(ps[i], 0); + ps[i] = NULL; + } + } + } + } + + fprintf(stderr, "Test end\n"); + return (0); +} diff --git a/deps/jemalloc.orig/test/allocm.exp b/deps/jemalloc.orig/test/allocm.exp new file mode 100644 index 00000000000..b5061c7277e --- /dev/null +++ b/deps/jemalloc.orig/test/allocm.exp @@ -0,0 +1,25 @@ +Test begin +Alignment: 8 +Alignment: 16 +Alignment: 32 +Alignment: 64 +Alignment: 128 +Alignment: 256 +Alignment: 512 +Alignment: 1024 +Alignment: 2048 +Alignment: 4096 +Alignment: 8192 +Alignment: 16384 +Alignment: 32768 +Alignment: 65536 +Alignment: 131072 +Alignment: 262144 +Alignment: 524288 +Alignment: 1048576 +Alignment: 2097152 +Alignment: 4194304 +Alignment: 8388608 +Alignment: 16777216 +Alignment: 33554432 +Test end diff --git a/deps/jemalloc.orig/test/bitmap.c b/deps/jemalloc.orig/test/bitmap.c new file mode 100644 index 00000000000..adfaacfe52b --- /dev/null +++ b/deps/jemalloc.orig/test/bitmap.c @@ -0,0 +1,157 @@ +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +/* + * Avoid using the assert() from jemalloc_internal.h, since it requires + * internal libjemalloc functionality. + * */ +#include + +/* + * Directly include the bitmap code, since it isn't exposed outside + * libjemalloc. + */ +#include "../src/bitmap.c" + +#if (LG_BITMAP_MAXBITS > 12) +# define MAXBITS 4500 +#else +# define MAXBITS (1U << LG_BITMAP_MAXBITS) +#endif + +static void +test_bitmap_size(void) +{ + size_t i, prev_size; + + prev_size = 0; + for (i = 1; i <= MAXBITS; i++) { + size_t size = bitmap_size(i); + assert(size >= prev_size); + prev_size = size; + } +} + +static void +test_bitmap_init(void) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + size_t j; + bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_init(bitmap, &binfo); + + for (j = 0; j < i; j++) + assert(bitmap_get(bitmap, &binfo, j) == false); + + } + } +} + +static void +test_bitmap_set(void) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + size_t j; + bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_init(bitmap, &binfo); + + for (j = 0; j < i; j++) + bitmap_set(bitmap, &binfo, j); + assert(bitmap_full(bitmap, &binfo)); + } + } +} + +static void +test_bitmap_unset(void) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + size_t j; + bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_init(bitmap, &binfo); + + for (j = 0; j < i; j++) + bitmap_set(bitmap, &binfo, j); + assert(bitmap_full(bitmap, &binfo)); + for (j = 0; j < i; j++) + bitmap_unset(bitmap, &binfo, j); + for (j = 0; j < i; j++) + bitmap_set(bitmap, &binfo, j); + assert(bitmap_full(bitmap, &binfo)); + } + } +} + +static void +test_bitmap_sfu(void) +{ + size_t i; + + for (i = 1; i <= MAXBITS; i++) { + bitmap_info_t binfo; + bitmap_info_init(&binfo, i); + { + ssize_t j; + bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_init(bitmap, &binfo); + + /* Iteratively set bits starting at the beginning. */ + for (j = 0; j < i; j++) + assert(bitmap_sfu(bitmap, &binfo) == j); + assert(bitmap_full(bitmap, &binfo)); + + /* + * Iteratively unset bits starting at the end, and + * verify that bitmap_sfu() reaches the unset bits. + */ + for (j = i - 1; j >= 0; j--) { + bitmap_unset(bitmap, &binfo, j); + assert(bitmap_sfu(bitmap, &binfo) == j); + bitmap_unset(bitmap, &binfo, j); + } + assert(bitmap_get(bitmap, &binfo, 0) == false); + + /* + * Iteratively set bits starting at the beginning, and + * verify that bitmap_sfu() looks past them. + */ + for (j = 1; j < i; j++) { + bitmap_set(bitmap, &binfo, j - 1); + assert(bitmap_sfu(bitmap, &binfo) == j); + bitmap_unset(bitmap, &binfo, j); + } + assert(bitmap_sfu(bitmap, &binfo) == i - 1); + assert(bitmap_full(bitmap, &binfo)); + } + } +} + +int +main(void) +{ + fprintf(stderr, "Test begin\n"); + + test_bitmap_size(); + test_bitmap_init(); + test_bitmap_set(); + test_bitmap_unset(); + test_bitmap_sfu(); + + fprintf(stderr, "Test end\n"); + return (0); +} diff --git a/deps/jemalloc.orig/test/bitmap.exp b/deps/jemalloc.orig/test/bitmap.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc.orig/test/bitmap.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/deps/jemalloc.orig/test/mremap.c b/deps/jemalloc.orig/test/mremap.c new file mode 100644 index 00000000000..146c66f4212 --- /dev/null +++ b/deps/jemalloc.orig/test/mremap.c @@ -0,0 +1,67 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +int +main(void) +{ + int ret, err; + size_t sz, lg_chunk, chunksize, i; + char *p, *q; + + fprintf(stderr, "Test begin\n"); + + sz = sizeof(lg_chunk); + if ((err = JEMALLOC_P(mallctl)("opt.lg_chunk", &lg_chunk, &sz, NULL, + 0))) { + assert(err != ENOENT); + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + ret = 1; + goto RETURN; + } + chunksize = ((size_t)1U) << lg_chunk; + + p = (char *)malloc(chunksize); + if (p == NULL) { + fprintf(stderr, "malloc(%zu) --> %p\n", chunksize, p); + ret = 1; + goto RETURN; + } + memset(p, 'a', chunksize); + + q = (char *)realloc(p, chunksize * 2); + if (q == NULL) { + fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize * 2, + q); + ret = 1; + goto RETURN; + } + for (i = 0; i < chunksize; i++) { + assert(q[i] == 'a'); + } + + p = q; + + q = (char *)realloc(p, chunksize); + if (q == NULL) { + fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize, q); + ret = 1; + goto RETURN; + } + for (i = 0; i < chunksize; i++) { + assert(q[i] == 'a'); + } + + free(q); + + ret = 0; +RETURN: + fprintf(stderr, "Test end\n"); + return (ret); +} diff --git a/deps/jemalloc.orig/test/mremap.exp b/deps/jemalloc.orig/test/mremap.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc.orig/test/mremap.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/deps/jemalloc.orig/test/posix_memalign.c b/deps/jemalloc.orig/test/posix_memalign.c new file mode 100644 index 00000000000..3e306c01dd2 --- /dev/null +++ b/deps/jemalloc.orig/test/posix_memalign.c @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +#define CHUNK 0x400000 +/* #define MAXALIGN ((size_t)0x80000000000LLU) */ +#define MAXALIGN ((size_t)0x2000000LLU) +#define NITER 4 + +int +main(void) +{ + size_t alignment, size, total; + unsigned i; + int err; + void *p, *ps[NITER]; + + fprintf(stderr, "Test begin\n"); + + /* Test error conditions. */ + for (alignment = 0; alignment < sizeof(void *); alignment++) { + err = JEMALLOC_P(posix_memalign)(&p, alignment, 1); + if (err != EINVAL) { + fprintf(stderr, + "Expected error for invalid alignment %zu\n", + alignment); + } + } + + for (alignment = sizeof(size_t); alignment < MAXALIGN; + alignment <<= 1) { + err = JEMALLOC_P(posix_memalign)(&p, alignment + 1, 1); + if (err == 0) { + fprintf(stderr, + "Expected error for invalid alignment %zu\n", + alignment + 1); + } + } + +#if LG_SIZEOF_PTR == 3 + alignment = 0x8000000000000000LLU; + size = 0x8000000000000000LLU; +#else + alignment = 0x80000000LU; + size = 0x80000000LU; +#endif + err = JEMALLOC_P(posix_memalign)(&p, alignment, size); + if (err == 0) { + fprintf(stderr, + "Expected error for posix_memalign(&p, %zu, %zu)\n", + alignment, size); + } + +#if LG_SIZEOF_PTR == 3 + alignment = 0x4000000000000000LLU; + size = 0x8400000000000001LLU; +#else + alignment = 0x40000000LU; + size = 0x84000001LU; +#endif + err = JEMALLOC_P(posix_memalign)(&p, alignment, size); + if (err == 0) { + fprintf(stderr, + "Expected error for posix_memalign(&p, %zu, %zu)\n", + alignment, size); + } + + alignment = 0x10LLU; +#if LG_SIZEOF_PTR == 3 + size = 0xfffffffffffffff0LLU; +#else + size = 0xfffffff0LU; +#endif + err = JEMALLOC_P(posix_memalign)(&p, alignment, size); + if (err == 0) { + fprintf(stderr, + "Expected error for posix_memalign(&p, %zu, %zu)\n", + alignment, size); + } + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + fprintf(stderr, "Alignment: %zu\n", alignment); + for (size = 1; + size < 3 * alignment && size < (1U << 31); + size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + err = JEMALLOC_P(posix_memalign)(&ps[i], + alignment, size); + if (err) { + fprintf(stderr, + "Error for size %zu (0x%zx): %s\n", + size, size, strerror(err)); + exit(1); + } + total += JEMALLOC_P(malloc_usable_size)(ps[i]); + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + JEMALLOC_P(free)(ps[i]); + ps[i] = NULL; + } + } + } + } + + fprintf(stderr, "Test end\n"); + return (0); +} diff --git a/deps/jemalloc.orig/test/posix_memalign.exp b/deps/jemalloc.orig/test/posix_memalign.exp new file mode 100644 index 00000000000..b5061c7277e --- /dev/null +++ b/deps/jemalloc.orig/test/posix_memalign.exp @@ -0,0 +1,25 @@ +Test begin +Alignment: 8 +Alignment: 16 +Alignment: 32 +Alignment: 64 +Alignment: 128 +Alignment: 256 +Alignment: 512 +Alignment: 1024 +Alignment: 2048 +Alignment: 4096 +Alignment: 8192 +Alignment: 16384 +Alignment: 32768 +Alignment: 65536 +Alignment: 131072 +Alignment: 262144 +Alignment: 524288 +Alignment: 1048576 +Alignment: 2097152 +Alignment: 4194304 +Alignment: 8388608 +Alignment: 16777216 +Alignment: 33554432 +Test end diff --git a/deps/jemalloc.orig/test/rallocm.c b/deps/jemalloc.orig/test/rallocm.c new file mode 100644 index 00000000000..ccf326bb523 --- /dev/null +++ b/deps/jemalloc.orig/test/rallocm.c @@ -0,0 +1,127 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +int +main(void) +{ + size_t pagesize; + void *p, *q; + size_t sz, tsz; + int r; + + fprintf(stderr, "Test begin\n"); + + /* Get page size. */ + { + long result = sysconf(_SC_PAGESIZE); + assert(result != -1); + pagesize = (size_t)result; + } + + r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); + if (r != ALLOCM_SUCCESS) { + fprintf(stderr, "Unexpected allocm() error\n"); + abort(); + } + + q = p; + r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 0, ALLOCM_NO_MOVE); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (q != p) + fprintf(stderr, "Unexpected object move\n"); + if (tsz != sz) { + fprintf(stderr, "Unexpected size change: %zu --> %zu\n", + sz, tsz); + } + + q = p; + r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 5, ALLOCM_NO_MOVE); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (q != p) + fprintf(stderr, "Unexpected object move\n"); + if (tsz != sz) { + fprintf(stderr, "Unexpected size change: %zu --> %zu\n", + sz, tsz); + } + + q = p; + r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE); + if (r != ALLOCM_ERR_NOT_MOVED) + fprintf(stderr, "Unexpected rallocm() result\n"); + if (q != p) + fprintf(stderr, "Unexpected object move\n"); + if (tsz != sz) { + fprintf(stderr, "Unexpected size change: %zu --> %zu\n", + sz, tsz); + } + + q = p; + r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, 0); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (q == p) + fprintf(stderr, "Expected object move\n"); + if (tsz == sz) { + fprintf(stderr, "Expected size change: %zu --> %zu\n", + sz, tsz); + } + p = q; + sz = tsz; + + r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, 0); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (q == p) + fprintf(stderr, "Expected object move\n"); + if (tsz == sz) { + fprintf(stderr, "Expected size change: %zu --> %zu\n", + sz, tsz); + } + p = q; + sz = tsz; + + r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, 0); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (tsz == sz) { + fprintf(stderr, "Expected size change: %zu --> %zu\n", + sz, tsz); + } + p = q; + sz = tsz; + + r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, ALLOCM_NO_MOVE); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (q != p) + fprintf(stderr, "Unexpected object move\n"); + if (tsz == sz) { + fprintf(stderr, "Expected size change: %zu --> %zu\n", + sz, tsz); + } + sz = tsz; + + r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, ALLOCM_NO_MOVE); + if (r != ALLOCM_SUCCESS) + fprintf(stderr, "Unexpected rallocm() error\n"); + if (q != p) + fprintf(stderr, "Unexpected object move\n"); + if (tsz == sz) { + fprintf(stderr, "Expected size change: %zu --> %zu\n", + sz, tsz); + } + sz = tsz; + + JEMALLOC_P(dallocm)(p, 0); + + fprintf(stderr, "Test end\n"); + return (0); +} diff --git a/deps/jemalloc.orig/test/rallocm.exp b/deps/jemalloc.orig/test/rallocm.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc.orig/test/rallocm.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/deps/jemalloc.orig/test/thread_arena.c b/deps/jemalloc.orig/test/thread_arena.c new file mode 100644 index 00000000000..ef8d6817500 --- /dev/null +++ b/deps/jemalloc.orig/test/thread_arena.c @@ -0,0 +1,92 @@ +#include +#include +#include +#include +#include + +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +#define NTHREADS 10 + +void * +thread_start(void *arg) +{ + unsigned main_arena_ind = *(unsigned *)arg; + void *p; + unsigned arena_ind; + size_t size; + int err; + + p = JEMALLOC_P(malloc)(1); + if (p == NULL) { + fprintf(stderr, "%s(): Error in malloc()\n", __func__); + return (void *)1; + } + + size = sizeof(arena_ind); + if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, + &main_arena_ind, sizeof(main_arena_ind)))) { + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + return (void *)1; + } + + size = sizeof(arena_ind); + if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, + 0))) { + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + return (void *)1; + } + assert(arena_ind == main_arena_ind); + + return (NULL); +} + +int +main(void) +{ + int ret = 0; + void *p; + unsigned arena_ind; + size_t size; + int err; + pthread_t threads[NTHREADS]; + unsigned i; + + fprintf(stderr, "Test begin\n"); + + p = JEMALLOC_P(malloc)(1); + if (p == NULL) { + fprintf(stderr, "%s(): Error in malloc()\n", __func__); + ret = 1; + goto RETURN; + } + + size = sizeof(arena_ind); + if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, + 0))) { + fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + strerror(err)); + ret = 1; + goto RETURN; + } + + for (i = 0; i < NTHREADS; i++) { + if (pthread_create(&threads[i], NULL, thread_start, + (void *)&arena_ind) != 0) { + fprintf(stderr, "%s(): Error in pthread_create()\n", + __func__); + ret = 1; + goto RETURN; + } + } + + for (i = 0; i < NTHREADS; i++) + pthread_join(threads[i], (void *)&ret); + +RETURN: + fprintf(stderr, "Test end\n"); + return (ret); +} diff --git a/deps/jemalloc.orig/test/thread_arena.exp b/deps/jemalloc.orig/test/thread_arena.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc.orig/test/thread_arena.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/deps/jemalloc/.gitignore b/deps/jemalloc/.gitignore index 32b4c424bde..e6e8bb00904 100644 --- a/deps/jemalloc/.gitignore +++ b/deps/jemalloc/.gitignore @@ -11,6 +11,7 @@ /lib/ /Makefile /include/jemalloc/internal/jemalloc_internal\.h +/include/jemalloc/internal/size_classes\.h /include/jemalloc/jemalloc\.h /include/jemalloc/jemalloc_defs\.h /test/jemalloc_test\.h @@ -21,3 +22,4 @@ !test/*.c !test/*.exp /VERSION +/bin/jemalloc.sh diff --git a/deps/jemalloc/COPYING b/deps/jemalloc/COPYING index 10ade120049..e27fc4d6cab 100644 --- a/deps/jemalloc/COPYING +++ b/deps/jemalloc/COPYING @@ -1,9 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are -subject to the following licenses: +subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2010 Jason Evans . +Copyright (C) 2002-2012 Jason Evans . All rights reserved. -Copyright (C) 2007-2010 Mozilla Foundation. All rights reserved. +Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. +Copyright (C) 2009-2012 Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -24,28 +25,3 @@ LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- -Copyright (C) 2009-2010 Facebook, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Facebook, Inc. nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- diff --git a/deps/jemalloc/ChangeLog b/deps/jemalloc/ChangeLog index 326ee7a97e1..231dd6da730 100644 --- a/deps/jemalloc/ChangeLog +++ b/deps/jemalloc/ChangeLog @@ -6,6 +6,95 @@ found in the git revision history: http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git git://canonware.com/jemalloc.git +* 3.0.0 (May 11, 2012) + + Although this version adds some major new features, the primary focus is on + internal code cleanup that facilitates maintainability and portability, most + of which is not reflected in the ChangeLog. This is the first release to + incorporate substantial contributions from numerous other developers, and the + result is a more broadly useful allocator (see the git revision history for + contribution details). Note that the license has been unified, thanks to + Facebook granting a license under the same terms as the other copyright + holders (see COPYING). + + New features: + - Implement Valgrind support, redzones, and quarantine. + - Add support for additional platforms: + + FreeBSD + + Mac OS X Lion + + MinGW + + Windows (no support yet for replacing the system malloc) + - Add support for additional architectures: + + MIPS + + SH4 + + Tilera + - Add support for cross compiling. + - Add nallocm(), which rounds a request size up to the nearest size class + without actually allocating. + - Implement aligned_alloc() (blame C11). + - Add the "thread.tcache.enabled" mallctl. + - Add the "opt.prof_final" mallctl. + - Update pprof (from gperftools 2.0). + - Add the --with-mangling option. + - Add the --disable-experimental option. + - Add the --disable-munmap option, and make it the default on Linux. + - Add the --enable-mremap option, which disables use of mremap(2) by default. + + Incompatible changes: + - Enable stats by default. + - Enable fill by default. + - Disable lazy locking by default. + - Rename the "tcache.flush" mallctl to "thread.tcache.flush". + - Rename the "arenas.pagesize" mallctl to "arenas.page". + - Change the "opt.lg_prof_sample" default from 0 to 19 (1 B to 512 KiB). + - Change the "opt.prof_accum" default from true to false. + + Removed features: + - Remove the swap feature, including the "config.swap", "swap.avail", + "swap.prezeroed", "swap.nfds", and "swap.fds" mallctls. + - Remove highruns statistics, including the + "stats.arenas..bins..highruns" and + "stats.arenas..lruns..highruns" mallctls. + - As part of small size class refactoring, remove the "opt.lg_[qc]space_max", + "arenas.cacheline", "arenas.subpage", "arenas.[tqcs]space_{min,max}", and + "arenas.[tqcs]bins" mallctls. + - Remove the "arenas.chunksize" mallctl. + - Remove the "opt.lg_prof_tcmax" option. + - Remove the "opt.lg_prof_bt_max" option. + - Remove the "opt.lg_tcache_gc_sweep" option. + - Remove the --disable-tiny option, including the "config.tiny" mallctl. + - Remove the --enable-dynamic-page-shift configure option. + - Remove the --enable-sysv configure option. + + Bug fixes: + - Fix a statistics-related bug in the "thread.arena" mallctl that could cause + invalid statistics and crashes. + - Work around TLS deallocation via free() on Linux. This bug could cause + write-after-free memory corruption. + - Fix a potential deadlock that could occur during interval- and + growth-triggered heap profile dumps. + - Fix large calloc() zeroing bugs due to dropping chunk map unzeroed flags. + - Fix chunk_alloc_dss() to stop claiming memory is zeroed. This bug could + cause memory corruption and crashes with --enable-dss specified. + - Fix fork-related bugs that could cause deadlock in children between fork + and exec. + - Fix malloc_stats_print() to honor 'b' and 'l' in the opts parameter. + - Fix realloc(p, 0) to act like free(p). + - Do not enforce minimum alignment in memalign(). + - Check for NULL pointer in malloc_usable_size(). + - Fix an off-by-one heap profile statistics bug that could be observed in + interval- and growth-triggered heap profiles. + - Fix the "epoch" mallctl to update cached stats even if the passed in epoch + is 0. + - Fix bin->runcur management to fix a layout policy bug. This bug did not + affect correctness. + - Fix a bug in choose_arena_hard() that potentially caused more arenas to be + initialized than necessary. + - Add missing "opt.lg_tcache_max" mallctl implementation. + - Use glibc allocator hooks to make mixed allocator usage less likely. + - Fix build issues for --disable-tcache. + - Don't mangle pthread_create() when --with-private-namespace is specified. + * 2.2.5 (November 14, 2011) Bug fixes: diff --git a/deps/jemalloc/INSTALL b/deps/jemalloc/INSTALL index 2a1e469cb8b..e40a7eddca2 100644 --- a/deps/jemalloc/INSTALL +++ b/deps/jemalloc/INSTALL @@ -26,6 +26,19 @@ any of the following arguments (not a definitive list) to 'configure': Embed one or more library paths, so that libjemalloc can find the libraries it is linked to. This works only on ELF-based systems. +--with-mangling= + Mangle public symbols specified in which is a comma-separated list of + name:mangled pairs. + + For example, to use ld's --wrap option as an alternative method for + overriding libc's malloc implementation, specify something like: + + --with-mangling=malloc:__wrap_malloc,free:__wrap_free[...] + + Note that mangling happens prior to application of the prefix specified by + --with-jemalloc-prefix, and mangled symbols are then ignored when applying + the prefix. + --with-jemalloc-prefix= Prefix all public APIs with . For example, if is "prefix_", API changes like the following occur: @@ -62,8 +75,8 @@ any of the following arguments (not a definitive list) to 'configure': Enable assertions and validation code. This incurs a substantial performance hit, but is very useful during application development. ---enable-stats - Enable statistics gathering functionality. See the "opt.stats_print" +--disable-stats + Disable statistics gathering functionality. See the "opt.stats_print" option documentation for usage details. --enable-prof @@ -90,51 +103,50 @@ any of the following arguments (not a definitive list) to 'configure': Statically link against the specified libunwind.a rather than dynamically linking with -lunwind. ---disable-tiny - Disable tiny (sub-quantum-sized) object support. Technically it is not - legal for a malloc implementation to allocate objects with less than - quantum alignment (8 or 16 bytes, depending on architecture), but in - practice it never causes any problems if, for example, 4-byte allocations - are 4-byte-aligned. - --disable-tcache Disable thread-specific caches for small objects. Objects are cached and released in bulk, thus reducing the total number of mutex operations. See the "opt.tcache" option for usage details. ---enable-swap - Enable mmap()ed swap file support. When this feature is built in, it is - possible to specify one or more files that act as backing store. This - effectively allows for per application swap files. +--enable-mremap + Enable huge realloc() via mremap(2). mremap() is disabled by default + because the flavor used is specific to Linux, which has a quirk in its + virtual memory allocation algorithm that causes semi-permanent VM map holes + under normal jemalloc operation. + +--disable-munmap + Disable virtual memory deallocation via munmap(2); instead keep track of + the virtual memory for later use. munmap() is disabled by default (i.e. + --disable-munmap is implied) on Linux, which has a quirk in its virtual + memory allocation algorithm that causes semi-permanent VM map holes under + normal jemalloc operation. --enable-dss Enable support for page allocation/deallocation via sbrk(2), in addition to mmap(2). ---enable-fill - Enable support for junk/zero filling of memory. See the "opt.junk"/ - "opt.zero" option documentation for usage details. +--disable-fill + Disable support for junk/zero filling of memory, quarantine, and redzones. + See the "opt.junk", "opt.zero", "opt.quarantine", and "opt.redzone" option + documentation for usage details. + +--disable-valgrind + Disable support for Valgrind. + +--disable-experimental + Disable support for the experimental API (*allocm()). + +--enable-utrace + Enable utrace(2)-based allocation tracing. This feature is not broadly + portable (FreeBSD has it, but Linux and OS X do not). --enable-xmalloc Enable support for optional immediate termination due to out-of-memory errors, as is commonly implemented by "xmalloc" wrapper function for malloc. See the "opt.xmalloc" option documentation for usage details. ---enable-sysv - Enable support for System V semantics, wherein malloc(0) returns NULL - rather than a minimal allocation. See the "opt.sysv" option documentation - for usage details. - ---enable-dynamic-page-shift - Under most conditions, the system page size never changes (usually 4KiB or - 8KiB, depending on architecture and configuration), and unless this option - is enabled, jemalloc assumes that page size can safely be determined during - configuration and hard-coded. Enabling dynamic page size determination has - a measurable impact on performance, since the compiler is forced to load - the page size from memory rather than embedding immediate values. - ---disable-lazy-lock - Disable code that wraps pthread_create() to detect when an application +--enable-lazy-lock + Enable code that wraps pthread_create() to detect when an application switches from single-threaded to multi-threaded mode, so that it can avoid mutex locking/unlocking operations while in single-threaded mode. In practice, this feature usually has little impact on performance unless @@ -181,11 +193,24 @@ PATH="?" === Advanced compilation ======================================================= +To build only parts of jemalloc, use the following targets: + + build_lib_shared + build_lib_static + build_lib + build_doc_html + build_doc_man + build_doc + To install only parts of jemalloc, use the following targets: install_bin install_include + install_lib_shared + install_lib_static install_lib + install_doc_html + install_doc_man install_doc To clean up build results to varying degrees, use the following make targets: @@ -248,10 +273,6 @@ directory, issue configuration and build commands: The manual page is generated in both html and roff formats. Any web browser can be used to view the html manual. The roff manual page can be formatted -prior to installation via any of the following commands: +prior to installation via the following command: nroff -man -t doc/jemalloc.3 - - groff -man -t -Tps doc/jemalloc.3 | ps2pdf - doc/jemalloc.3.pdf - - (cd doc; groff -man -man-ext -t -Thtml jemalloc.3 > jemalloc.3.html) diff --git a/deps/jemalloc/Makefile.in b/deps/jemalloc/Makefile.in index de7492f951a..6675b596661 100644 --- a/deps/jemalloc/Makefile.in +++ b/deps/jemalloc/Makefile.in @@ -17,129 +17,184 @@ INCLUDEDIR := $(DESTDIR)@INCLUDEDIR@ LIBDIR := $(DESTDIR)@LIBDIR@ DATADIR := $(DESTDIR)@DATADIR@ MANDIR := $(DESTDIR)@MANDIR@ +srcroot := @srcroot@ +objroot := @objroot@ +abs_srcroot := @abs_srcroot@ +abs_objroot := @abs_objroot@ # Build parameters. -CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include +CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include CFLAGS := @CFLAGS@ -ifeq (macho, @abi@) -CFLAGS += -dynamic -endif LDFLAGS := @LDFLAGS@ +EXTRA_LDFLAGS := @EXTRA_LDFLAGS@ LIBS := @LIBS@ RPATH_EXTRA := @RPATH_EXTRA@ -ifeq (macho, @abi@) -SO := dylib -WL_SONAME := dylib_install_name +SO := @so@ +IMPORTLIB := @importlib@ +O := @o@ +A := @a@ +EXE := @exe@ +LIBPREFIX := @libprefix@ +REV := @rev@ +install_suffix := @install_suffix@ +ABI := @abi@ +XSLTPROC := @XSLTPROC@ +AUTOCONF := @AUTOCONF@ +_RPATH = @RPATH@ +RPATH = $(if $(1),$(call _RPATH,$(1))) +cfghdrs_in := @cfghdrs_in@ +cfghdrs_out := @cfghdrs_out@ +cfgoutputs_in := @cfgoutputs_in@ +cfgoutputs_out := @cfgoutputs_out@ +enable_autogen := @enable_autogen@ +enable_experimental := @enable_experimental@ +DSO_LDFLAGS = @DSO_LDFLAGS@ +SOREV = @SOREV@ +PIC_CFLAGS = @PIC_CFLAGS@ +CTARGET = @CTARGET@ +LDTARGET = @LDTARGET@ +MKLIB = @MKLIB@ +CC_MM = @CC_MM@ + +ifeq (macho, $(ABI)) +TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH="$(objroot)lib" else -SO := so -WL_SONAME := soname -endif -REV := 1 -ifeq (macho, @abi@) -TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH=@objroot@lib +ifeq (pecoff, $(ABI)) +TEST_LIBRARY_PATH := PATH="$(PATH):$(objroot)lib" else TEST_LIBRARY_PATH := endif +endif + +LIBJEMALLOC := $(LIBPREFIX)jemalloc$(install_suffix) # Lists of files. -BINS := @srcroot@bin/pprof -CHDRS := @objroot@include/jemalloc/jemalloc@install_suffix@.h \ - @objroot@include/jemalloc/jemalloc_defs@install_suffix@.h -CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \ - @srcroot@src/base.c @srcroot@src/bitmap.c @srcroot@src/chunk.c \ - @srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \ - @srcroot@src/chunk_swap.c @srcroot@src/ckh.c @srcroot@src/ctl.c \ - @srcroot@src/extent.c @srcroot@src/hash.c @srcroot@src/huge.c \ - @srcroot@src/mb.c @srcroot@src/mutex.c @srcroot@src/prof.c \ - @srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c -ifeq (macho, @abi@) -CSRCS += @srcroot@src/zone.c +BINS := $(srcroot)bin/pprof $(objroot)bin/jemalloc.sh +CHDRS := $(objroot)include/jemalloc/jemalloc$(install_suffix).h \ + $(objroot)include/jemalloc/jemalloc_defs$(install_suffix).h +CSRCS := $(srcroot)src/jemalloc.c $(srcroot)src/arena.c $(srcroot)src/atomic.c \ + $(srcroot)src/base.c $(srcroot)src/bitmap.c $(srcroot)src/chunk.c \ + $(srcroot)src/chunk_dss.c $(srcroot)src/chunk_mmap.c \ + $(srcroot)src/ckh.c $(srcroot)src/ctl.c $(srcroot)src/extent.c \ + $(srcroot)src/hash.c $(srcroot)src/huge.c $(srcroot)src/mb.c \ + $(srcroot)src/mutex.c $(srcroot)src/prof.c $(srcroot)src/quarantine.c \ + $(srcroot)src/rtree.c $(srcroot)src/stats.c $(srcroot)src/tcache.c \ + $(srcroot)src/util.c $(srcroot)src/tsd.c +ifeq (macho, $(ABI)) +CSRCS += $(srcroot)src/zone.c +endif +ifeq ($(IMPORTLIB),$(SO)) +STATIC_LIBS := $(objroot)lib/$(LIBJEMALLOC).$(A) +endif +ifdef PIC_CFLAGS +STATIC_LIBS += $(objroot)lib/$(LIBJEMALLOC)_pic.$(A) +else +STATIC_LIBS += $(objroot)lib/$(LIBJEMALLOC)_s.$(A) endif -STATIC_LIBS := @objroot@lib/libjemalloc@install_suffix@.a -DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \ - @objroot@lib/libjemalloc@install_suffix@.$(SO) \ - @objroot@lib/libjemalloc@install_suffix@_pic.a -MAN3 := @objroot@doc/jemalloc@install_suffix@.3 -DOCS_XML := @objroot@doc/jemalloc@install_suffix@.xml -DOCS_HTML := $(DOCS_XML:@objroot@%.xml=@srcroot@%.html) -DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3) +DSOS := $(objroot)lib/$(LIBJEMALLOC).$(SOREV) +ifneq ($(SOREV),$(SO)) +DSOS += $(objroot)lib/$(LIBJEMALLOC).$(SO) +endif +MAN3 := $(objroot)doc/jemalloc$(install_suffix).3 +DOCS_XML := $(objroot)doc/jemalloc$(install_suffix).xml +DOCS_HTML := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.html) +DOCS_MAN3 := $(DOCS_XML:$(objroot)%.xml=$(srcroot)%.3) DOCS := $(DOCS_HTML) $(DOCS_MAN3) -CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \ - @srcroot@test/bitmap.c @srcroot@test/mremap.c \ - @srcroot@test/posix_memalign.c @srcroot@test/rallocm.c \ - @srcroot@test/thread_arena.c +CTESTS := $(srcroot)test/aligned_alloc.c $(srcroot)test/allocated.c \ + $(srcroot)test/bitmap.c $(srcroot)test/mremap.c \ + $(srcroot)test/posix_memalign.c $(srcroot)test/thread_arena.c \ + $(srcroot)test/thread_tcache_enabled.c +ifeq ($(enable_experimental), 1) +CTESTS += $(srcroot)test/allocm.c $(srcroot)test/rallocm.c +endif + +COBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.$(O)) +CPICOBJS := $(CSRCS:$(srcroot)%.c=$(objroot)%.pic.$(O)) +CTESTOBJS := $(CTESTS:$(srcroot)%.c=$(objroot)%.$(O)) .PHONY: all dist doc_html doc_man doc .PHONY: install_bin install_include install_lib .PHONY: install_html install_man install_doc install .PHONY: tests check clean distclean relclean -.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o) +.SECONDARY : $(CTESTOBJS) # Default target. -all: $(DSOS) $(STATIC_LIBS) +all: build -dist: doc +dist: build_doc -@srcroot@doc/%.html : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/html.xsl - @XSLTPROC@ -o $@ @objroot@doc/html.xsl $< +$(srcroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl + $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< -@srcroot@doc/%.3 : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/manpages.xsl - @XSLTPROC@ -o $@ @objroot@doc/manpages.xsl $< +$(srcroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl + $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< -doc_html: $(DOCS_HTML) -doc_man: $(DOCS_MAN3) -doc: $(DOCS) +build_doc_html: $(DOCS_HTML) +build_doc_man: $(DOCS_MAN3) +build_doc: $(DOCS) # # Include generated dependency files. # --include $(CSRCS:@srcroot@%.c=@objroot@%.d) --include $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) --include $(CTESTS:@srcroot@%.c=@objroot@%.d) +ifdef CC_MM +-include $(COBJS:%.$(O)=%.d) +-include $(CPICOBJS:%.$(O)=%.d) +-include $(CTESTOBJS:%.$(O)=%.d) +endif -@objroot@src/%.o: @srcroot@src/%.c - @mkdir -p $(@D) - $(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $< - @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)" +$(COBJS): $(objroot)src/%.$(O): $(srcroot)src/%.c +$(CPICOBJS): $(objroot)src/%.pic.$(O): $(srcroot)src/%.c +$(CPICOBJS): CFLAGS += $(PIC_CFLAGS) +$(CTESTOBJS): $(objroot)test/%.$(O): $(srcroot)test/%.c +$(CTESTOBJS): CPPFLAGS += -I$(objroot)test +ifneq ($(IMPORTLIB),$(SO)) +$(COBJS): CPPFLAGS += -DDLLEXPORT +endif -@objroot@src/%.pic.o: @srcroot@src/%.c - @mkdir -p $(@D) - $(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $< - @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $(basename $@))))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.pic.o \2/g\" > $(@:%.o=%.d)" +ifndef CC_MM +# Dependencies +HEADER_DIRS = $(srcroot)include/jemalloc/internal \ + $(objroot)include/jemalloc $(objroot)include/jemalloc/internal +HEADERS = $(wildcard $(foreach dir,$(HEADER_DIRS),$(dir)/*.h)) +$(COBJS) $(CPICOBJS) $(CTESTOBJS): $(HEADERS) +$(CTESTOBJS): $(objroot)test/jemalloc_test.h +endif -%.$(SO) : %.$(SO).$(REV) +$(COBJS) $(CPICOBJS) $(CTESTOBJS): %.$(O): @mkdir -p $(@D) - ln -sf $( $(@:%.o=%.d)" + $(MKLIB) $+ -# Automatic dependency generation misses #include "*.c". -@objroot@test/bitmap.o : @objroot@src/bitmap.o +$(objroot)test/bitmap$(EXE): $(objroot)src/bitmap.$(O) -@objroot@test/%: @objroot@test/%.o \ - @objroot@lib/libjemalloc@install_suffix@.$(SO) +$(objroot)test/%$(EXE): $(objroot)test/%.$(O) $(objroot)src/util.$(O) $(DSOS) @mkdir -p $(@D) -ifneq (@RPATH@, ) - $(CC) -o $@ $< @RPATH@@objroot@lib -L@objroot@lib -ljemalloc@install_suffix@ -lpthread -else - $(CC) -o $@ $< -L@objroot@lib -ljemalloc@install_suffix@ -lpthread -endif + $(CC) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) $(filter -lpthread,$(LIBS)) $(EXTRA_LDFLAGS) + +build_lib_shared: $(DSOS) +build_lib_static: $(STATIC_LIBS) +build: build_lib_shared build_lib_static install_bin: install -d $(BINDIR) @@ -155,46 +210,55 @@ install_include: install -m 644 $$h $(INCLUDEDIR)/jemalloc; \ done -install_lib: $(DSOS) $(STATIC_LIBS) +install_lib_shared: $(DSOS) install -d $(LIBDIR) - install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR) - ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO) - install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR) - install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR) + install -m 755 $(objroot)lib/$(LIBJEMALLOC).$(SOREV) $(LIBDIR) +ifneq ($(SOREV),$(SO)) + ln -sf $(LIBJEMALLOC).$(SOREV) $(LIBDIR)/$(LIBJEMALLOC).$(SO) +endif + +install_lib_static: $(STATIC_LIBS) + install -d $(LIBDIR) + @for l in $(STATIC_LIBS); do \ + echo "install -m 755 $$l $(LIBDIR)"; \ + install -m 755 $$l $(LIBDIR); \ +done + +install_lib: install_lib_shared install_lib_static -install_html: - install -d $(DATADIR)/doc/jemalloc@install_suffix@ +install_doc_html: + install -d $(DATADIR)/doc/jemalloc$(install_suffix) @for d in $(DOCS_HTML); do \ - echo "install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@"; \ - install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@; \ + echo "install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix)"; \ + install -m 644 $$d $(DATADIR)/doc/jemalloc$(install_suffix); \ done -install_man: +install_doc_man: install -d $(MANDIR)/man3 @for d in $(DOCS_MAN3); do \ echo "install -m 644 $$d $(MANDIR)/man3"; \ install -m 644 $$d $(MANDIR)/man3; \ done -install_doc: install_html install_man +install_doc: install_doc_html install_doc_man install: install_bin install_include install_lib install_doc -tests: $(CTESTS:@srcroot@%.c=@objroot@%) +tests: $(CTESTS:$(srcroot)%.c=$(objroot)%$(EXE)) check: tests - @mkdir -p @objroot@test + @mkdir -p $(objroot)test @$(SHELL) -c 'total=0; \ failures=0; \ echo "========================================="; \ - for t in $(CTESTS:@srcroot@%.c=@objroot@%); do \ + for t in $(CTESTS:$(srcroot)%.c=$(objroot)%); do \ total=`expr $$total + 1`; \ /bin/echo -n "$${t} ... "; \ - $(TEST_LIBRARY_PATH) $${t} @abs_srcroot@ @abs_objroot@ \ - > @objroot@$${t}.out 2>&1; \ - if test -e "@srcroot@$${t}.exp"; then \ - diff -u @srcroot@$${t}.exp \ - @objroot@$${t}.out >/dev/null 2>&1; \ + $(TEST_LIBRARY_PATH) $${t}$(EXE) $(abs_srcroot) \ + $(abs_objroot) > $(objroot)$${t}.out 2>&1; \ + if test -e "$(srcroot)$${t}.exp"; then \ + diff -w -u $(srcroot)$${t}.exp \ + $(objroot)$${t}.out >/dev/null 2>&1; \ fail=$$?; \ if test "$${fail}" -eq "1" ; then \ failures=`expr $${failures} + 1`; \ @@ -211,49 +275,49 @@ check: tests echo "Failures: $${failures}/$${total}"' clean: - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out) + rm -f $(COBJS) + rm -f $(CPICOBJS) + rm -f $(COBJS:%.$(O)=%.d) + rm -f $(CPICOBJS:%.$(O)=%.d) + rm -f $(CTESTOBJS:%.$(O)=%$(EXE)) + rm -f $(CTESTOBJS) + rm -f $(CTESTOBJS:%.$(O)=%.d) + rm -f $(CTESTOBJS:%.$(O)=%.out) rm -f $(DSOS) $(STATIC_LIBS) distclean: clean - rm -rf @objroot@autom4te.cache - rm -f @objroot@config.log - rm -f @objroot@config.status - rm -f @objroot@config.stamp - rm -f @cfghdrs_out@ - rm -f @cfgoutputs_out@ + rm -rf $(objroot)autom4te.cache + rm -f $(objroot)config.log + rm -f $(objroot)config.status + rm -f $(objroot)config.stamp + rm -f $(cfghdrs_out) + rm -f $(cfgoutputs_out) relclean: distclean - rm -f @objroot@configure - rm -f @srcroot@VERSION + rm -f $(objroot)configure + rm -f $(srcroot)VERSION rm -f $(DOCS_HTML) rm -f $(DOCS_MAN3) #=============================================================================== # Re-configuration rules. -ifeq (@enable_autogen@, 1) -@srcroot@configure : @srcroot@configure.ac - cd ./@srcroot@ && @AUTOCONF@ +ifeq ($(enable_autogen), 1) +$(srcroot)configure : $(srcroot)configure.ac + cd ./$(srcroot) && $(AUTOCONF) -@objroot@config.status : @srcroot@configure - ./@objroot@config.status --recheck +$(objroot)config.status : $(srcroot)configure + ./$(objroot)config.status --recheck -@srcroot@config.stamp.in : @srcroot@configure.ac - echo stamp > @srcroot@config.stamp.in +$(srcroot)config.stamp.in : $(srcroot)configure.ac + echo stamp > $(srcroot)config.stamp.in -@objroot@config.stamp : @cfgoutputs_in@ @cfghdrs_in@ @srcroot@configure - ./@objroot@config.status +$(objroot)config.stamp : $(cfgoutputs_in) $(cfghdrs_in) $(srcroot)configure + ./$(objroot)config.status @touch $@ # There must be some action in order for make to re-read Makefile when it is # out of date. -@cfgoutputs_out@ @cfghdrs_out@ : @objroot@config.stamp +$(cfgoutputs_out) $(cfghdrs_out) : $(objroot)config.stamp @true endif diff --git a/deps/jemalloc/README b/deps/jemalloc/README index 4d7b552bf3d..7661683bae7 100644 --- a/deps/jemalloc/README +++ b/deps/jemalloc/README @@ -1,10 +1,10 @@ jemalloc is a general-purpose scalable concurrent malloc(3) implementation. -This distribution is a stand-alone "portable" implementation that currently -targets Linux and Apple OS X. jemalloc is included as the default allocator in -the FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox -web browser on Microsoft Windows-related platforms. Depending on your needs, -one of the other divergent versions may suit your needs better than this -distribution. +This distribution is a "portable" implementation that currently targets +FreeBSD, Linux, Apple OS X, and MinGW. jemalloc is included as the default +allocator in the FreeBSD and NetBSD operating systems, and it is used by the +Mozilla Firefox web browser on Microsoft Windows-related platforms. Depending +on your needs, one of the other divergent versions may suit your needs better +than this distribution. The COPYING file contains copyright and licensing information. diff --git a/deps/jemalloc/VERSION b/deps/jemalloc/VERSION index aa85f5a2acf..c0f4e740940 100644 --- a/deps/jemalloc/VERSION +++ b/deps/jemalloc/VERSION @@ -1 +1 @@ -2.2.5-0-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760 +3.0.0-0-gfc9b1dbf69f59d7ecfc4ac68da9847e017e1d046 diff --git a/deps/jemalloc/bin/jemalloc.sh.in b/deps/jemalloc/bin/jemalloc.sh.in new file mode 100644 index 00000000000..cdf36737591 --- /dev/null +++ b/deps/jemalloc/bin/jemalloc.sh.in @@ -0,0 +1,9 @@ +#!/bin/sh + +prefix=@prefix@ +exec_prefix=@exec_prefix@ +libdir=@libdir@ + +@LD_PRELOAD_VAR@=${libdir}/libjemalloc.@SOREV@ +export @LD_PRELOAD_VAR@ +exec "$@" diff --git a/deps/jemalloc/bin/pprof b/deps/jemalloc/bin/pprof index 280ddcc8329..727eb43704f 100755 --- a/deps/jemalloc/bin/pprof +++ b/deps/jemalloc/bin/pprof @@ -72,7 +72,7 @@ use strict; use warnings; use Getopt::Long; -my $PPROF_VERSION = "1.7"; +my $PPROF_VERSION = "2.0"; # These are the object tools we use which can come from a # user-specified location using --tools, from the PPROF_TOOLS @@ -87,13 +87,14 @@ my %obj_tool_map = ( #"addr2line_pdb" => "addr2line-pdb", # ditto #"otool" => "otool", # equivalent of objdump on OS X ); -my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local -my $GV = "gv"; -my $EVINCE = "evince"; # could also be xpdf or perhaps acroread -my $KCACHEGRIND = "kcachegrind"; -my $PS2PDF = "ps2pdf"; +# NOTE: these are lists, so you can put in commandline flags if you want. +my @DOT = ("dot"); # leave non-absolute, since it may be in /usr/local +my @GV = ("gv"); +my @EVINCE = ("evince"); # could also be xpdf or perhaps acroread +my @KCACHEGRIND = ("kcachegrind"); +my @PS2PDF = ("ps2pdf"); # These are used for dynamic profiles -my $URL_FETCHER = "curl -s"; +my @URL_FETCHER = ("curl", "-s"); # These are the web pages that servers need to support for dynamic profiles my $HEAP_PAGE = "/pprof/heap"; @@ -104,7 +105,10 @@ my $GROWTH_PAGE = "/pprof/growth"; my $CONTENTION_PAGE = "/pprof/contention"; my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; -my $CENSUSPROFILE_PAGE = "/pprof/censusprofile"; # must support "?seconds=#" +my $CENSUSPROFILE_PAGE = "/pprof/censusprofile(?:\\?.*)?"; # must support cgi-param + # "?seconds=#", + # "?tags_regexp=#" and + # "?type=#". my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; @@ -122,6 +126,11 @@ my $UNKNOWN_BINARY = "(unknown)"; # 64-bit profiles. To err on the safe size, default to 64-bit here: my $address_length = 16; +my $dev_null = "/dev/null"; +if (! -e $dev_null && $^O =~ /MSWin/) { # $^O is the OS perl was built for + $dev_null = "nul"; +} + # A list of paths to search for shared object files my @prefix_list = (); @@ -151,7 +160,8 @@ pprof [options] The / can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. - For instance: "pprof http://myserver.com:80$HEAP_PAGE". + For instance: + pprof http://myserver.com:80$HEAP_PAGE If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). pprof --symbols Maps addresses to symbol names. In this mode, stdin should be a @@ -162,7 +172,7 @@ pprof --symbols For more help with querying remote servers, including how to add the necessary server-side support code, see this filename (or one like it): - /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html + /usr/doc/gperftools-$PPROF_VERSION/pprof_remote_servers.html Options: --cum Sort by cumulative data @@ -260,7 +270,7 @@ EOF sub version_string { return < 0) { + if (IsProfileURL($ARGV[0])) { + $main::use_symbol_page = 1; + } elsif (IsSymbolizedProfileFile($ARGV[0])) { + $main::use_symbolized_profile = 1; + $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file + } } if ($main::use_symbol_page || $main::use_symbolized_profile) { @@ -540,7 +552,7 @@ sub Init() { ConfigureObjTools($main::prog) } - # Break the opt_list_prefix into the prefix_list array + # Break the opt_lib_prefix into the prefix_list array @prefix_list = split (',', $main::opt_lib_prefix); # Remove trailing / from the prefixes, in the list to prevent @@ -636,9 +648,9 @@ sub Main() { # Print if (!$main::opt_interactive) { if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); + PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm); } elsif ($main::opt_list) { - PrintListing($libs, $flat, $cumulative, $main::opt_list); + PrintListing($total, $libs, $flat, $cumulative, $main::opt_list, 0); } elsif ($main::opt_text) { # Make sure the output is empty when have nothing to report # (only matters when --heapcheck is given but we must be @@ -646,7 +658,7 @@ sub Main() { if ($total != 0) { printf("Total: %s %s\n", Unparse($total), Units()); } - PrintText($symbols, $flat, $cumulative, $total, -1); + PrintText($symbols, $flat, $cumulative, -1); } elsif ($main::opt_raw) { PrintSymbolizedProfile($symbols, $profile, $main::prog); } elsif ($main::opt_callgrind) { @@ -656,7 +668,7 @@ sub Main() { if ($main::opt_gv) { RunGV(TempName($main::next_tmpfile, "ps"), ""); } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), ""); + RunEvince(TempName($main::next_tmpfile, "pdf"), ""); } elsif ($main::opt_web) { my $tmp = TempName($main::next_tmpfile, "svg"); RunWeb($tmp); @@ -705,24 +717,25 @@ sub ReadlineMightFail { sub RunGV { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - if (!system("$GV --version >/dev/null 2>&1")) { + if (!system(ShellEscape(@GV, "--version") . " >$dev_null 2>&1")) { # Options using double dash are supported by this gv version. # Also, turn on noantialias to better handle bug in gv for # postscript files with large dimensions. # TODO: Maybe we should not pass the --noantialias flag # if the gv version is known to work properly without the flag. - system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); + system(ShellEscape(@GV, "--scale=$main::opt_scale", "--noantialias", $fname) + . $bg); } else { # Old gv version - only supports options that use single dash. - print STDERR "$GV -scale $main::opt_scale\n"; - system("$GV -scale $main::opt_scale " . $fname . $bg); + print STDERR ShellEscape(@GV, "-scale", $main::opt_scale) . "\n"; + system(ShellEscape(@GV, "-scale", "$main::opt_scale", $fname) . $bg); } } sub RunEvince { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - system("$EVINCE " . $fname . $bg); + system(ShellEscape(@EVINCE, $fname) . $bg); } sub RunWeb { @@ -756,8 +769,8 @@ sub RunWeb { sub RunKcachegrind { my $fname = shift; my $bg = shift; # "" or " &" if we should run in background - print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; - system("$KCACHEGRIND " . $fname . $bg); + print STDERR "Starting '@KCACHEGRIND " . $fname . $bg . "'\n"; + system(ShellEscape(@KCACHEGRIND, $fname) . $bg); } @@ -834,14 +847,14 @@ sub InteractiveCommand { my $ignore; ($routine, $ignore) = ParseInteractiveArgs($3); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintText($symbols, $flat, $cumulative, $total, $line_limit); + PrintText($symbols, $flat, $cumulative, $line_limit); return 1; } if (m/^\s*callgrind\s*([^ \n]*)/) { @@ -861,21 +874,22 @@ sub InteractiveCommand { return 1; } - if (m/^\s*list\s*(.+)/) { + if (m/^\s*(web)?list\s*(.+)/) { + my $html = (defined($1) && ($1 eq "web")); $main::opt_list = 1; my $routine; my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); + ($routine, $ignore) = ParseInteractiveArgs($2); - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintListing($libs, $flat, $cumulative, $routine); + PrintListing($total, $libs, $flat, $cumulative, $routine, $html); return 1; } if (m/^\s*disasm\s*(.+)/) { @@ -886,14 +900,14 @@ sub InteractiveCommand { ($routine, $ignore) = ParseInteractiveArgs($1); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, "", $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles my $flat = FlatProfile($reduced); my $cumulative = CumulativeProfile($reduced); - PrintDisassembly($libs, $flat, $cumulative, $routine, $total); + PrintDisassembly($libs, $flat, $cumulative, $routine); return 1; } if (m/^\s*(gv|web|evince)\s*(.*)/) { @@ -913,7 +927,8 @@ sub InteractiveCommand { ($focus, $ignore) = ParseInteractiveArgs($2); # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, $focus, $ignore); + my $profile = ProcessProfile($total, $orig_profile, $symbols, + $focus, $ignore); my $reduced = ReduceProfile($symbols, $profile); # Get derived profiles @@ -941,6 +956,7 @@ sub InteractiveCommand { sub ProcessProfile { + my $total_count = shift; my $orig_profile = shift; my $symbols = shift; my $focus = shift; @@ -948,7 +964,6 @@ sub ProcessProfile { # Process current profile to account for various settings my $profile = $orig_profile; - my $total_count = TotalProfile($profile); printf("Total: %s %s\n", Unparse($total_count), Units()); if ($focus ne '') { $profile = FocusProfile($symbols, $profile, $focus); @@ -995,6 +1010,11 @@ Commands: list [routine_regexp] [-ignore1] [-ignore2] Show source listing of routines whose names match "routine_regexp" + weblist [routine_regexp] [-ignore1] [-ignore2] + Displays a source listing of routines whose names match "routine_regexp" + in a web browser. You can click on source lines to view the + corresponding disassembly. + top [--cum] [-ignore1] [-ignore2] top20 [--cum] [-ignore1] [-ignore2] top37 [--cum] [-ignore1] [-ignore2] @@ -1019,8 +1039,8 @@ parameters will be ignored. Further pprof details are available at this location (or one similar): - /usr/doc/google-perftools-$PPROF_VERSION/cpu_profiler.html - /usr/doc/google-perftools-$PPROF_VERSION/heap_profiler.html + /usr/doc/gperftools-$PPROF_VERSION/cpu_profiler.html + /usr/doc/gperftools-$PPROF_VERSION/heap_profiler.html ENDOFHELP } @@ -1137,9 +1157,10 @@ sub PrintText { my $symbols = shift; my $flat = shift; my $cumulative = shift; - my $total = shift; my $line_limit = shift; + my $total = TotalProfile($flat); + # Which profile to sort by? my $s = $main::opt_cum ? $cumulative : $flat; @@ -1169,7 +1190,29 @@ sub PrintText { $sym); } $lines++; - last if ($line_limit >= 0 && $lines > $line_limit); + last if ($line_limit >= 0 && $lines >= $line_limit); + } +} + +# Callgrind format has a compression for repeated function and file +# names. You show the name the first time, and just use its number +# subsequently. This can cut down the file to about a third or a +# quarter of its uncompressed size. $key and $val are the key/value +# pair that would normally be printed by callgrind; $map is a map from +# value to number. +sub CompressedCGName { + my($key, $val, $map) = @_; + my $idx = $map->{$val}; + # For very short keys, providing an index hurts rather than helps. + if (length($val) <= 3) { + return "$key=$val\n"; + } elsif (defined($idx)) { + return "$key=($idx)\n"; + } else { + # scalar(keys $map) gives the number of items in the map. + $idx = scalar(keys(%{$map})) + 1; + $map->{$val} = $idx; + return "$key=($idx) $val\n"; } } @@ -1177,13 +1220,16 @@ sub PrintText { sub PrintCallgrind { my $calls = shift; my $filename; + my %filename_to_index_map; + my %fnname_to_index_map; + if ($main::opt_interactive) { $filename = shift; print STDERR "Writing callgrind file to '$filename'.\n" } else { $filename = "&STDOUT"; } - open(CG, ">".$filename ); + open(CG, ">$filename"); printf CG ("events: Hits\n\n"); foreach my $call ( map { $_->[0] } sort { $a->[1] cmp $b ->[1] || @@ -1197,11 +1243,14 @@ sub PrintCallgrind { $callee_file, $callee_line, $callee_function ) = ( $1, $2, $3, $5, $6, $7 ); - - printf CG ("fl=$caller_file\nfn=$caller_function\n"); + # TODO(csilvers): for better compression, collect all the + # caller/callee_files and functions first, before printing + # anything, and only compress those referenced more than once. + printf CG CompressedCGName("fl", $caller_file, \%filename_to_index_map); + printf CG CompressedCGName("fn", $caller_function, \%fnname_to_index_map); if (defined $6) { - printf CG ("cfl=$callee_file\n"); - printf CG ("cfn=$callee_function\n"); + printf CG CompressedCGName("cfl", $callee_file, \%filename_to_index_map); + printf CG CompressedCGName("cfn", $callee_function, \%fnname_to_index_map); printf CG ("calls=$count $callee_line\n"); } printf CG ("$caller_line $count\n\n"); @@ -1214,7 +1263,8 @@ sub PrintDisassembly { my $flat = shift; my $cumulative = shift; my $disasm_opts = shift; - my $total = shift; + + my $total = TotalProfile($flat); foreach my $lib (@{$libs}) { my $symbol_table = GetProcedureBoundaries($lib->[0], $disasm_opts); @@ -1249,10 +1299,10 @@ sub Disassemble { my $end_addr = shift; my $objdump = $obj_tool_map{"objdump"}; - my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . - "--start-address=0x$start_addr " . - "--stop-address=0x$end_addr $prog"); - open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); + my $cmd = ShellEscape($objdump, "-C", "-d", "-l", "--no-show-raw-insn", + "--start-address=0x$start_addr", + "--stop-address=0x$end_addr", $prog); + open(OBJDUMP, "$cmd |") || error("$cmd: $!\n"); my @result = (); my $filename = ""; my $linenumber = -1; @@ -1315,13 +1365,33 @@ sub ByName { return ShortFunctionName($a) cmp ShortFunctionName($b); } -# Print source-listing for all all routines that match $main::opt_list +# Print source-listing for all all routines that match $list_opts sub PrintListing { + my $total = shift; my $libs = shift; my $flat = shift; my $cumulative = shift; my $list_opts = shift; + my $html = shift; + + my $output = \*STDOUT; + my $fname = ""; + if ($html) { + # Arrange to write the output to a temporary file + $fname = TempName($main::next_tmpfile, "html"); + $main::next_tmpfile++; + if (!open(TEMP, ">$fname")) { + print STDERR "$fname: $!\n"; + return; + } + $output = \*TEMP; + print $output HtmlListingHeader(); + printf $output ("
%s
Total: %s %s
\n", + $main::prog, Unparse($total), Units()); + } + + my $listed = 0; foreach my $lib (@{$libs}) { my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); my $offset = AddressSub($lib->[1], $lib->[3]); @@ -1333,15 +1403,113 @@ sub PrintListing { my $addr = AddressAdd($start_addr, $offset); for (my $i = 0; $i < $length; $i++) { if (defined($cumulative->{$addr})) { - PrintSource($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr); + $listed += PrintSource( + $lib->[0], $offset, + $routine, $flat, $cumulative, + $start_addr, $end_addr, + $html, + $output); last; } $addr = AddressInc($addr); } } } + + if ($html) { + if ($listed > 0) { + print $output HtmlListingFooter(); + close($output); + RunWeb($fname); + } else { + close($output); + unlink($fname); + } + } +} + +sub HtmlListingHeader { + return <<'EOF'; + + + +Pprof listing + + + + +EOF +} + +sub HtmlListingFooter { + return <<'EOF'; + + +EOF +} + +sub HtmlEscape { + my $text = shift; + $text =~ s/&/&/g; + $text =~ s//>/g; + return $text; } # Returns the indentation of the line, if it has any non-whitespace @@ -1355,6 +1523,45 @@ sub Indentation { } } +# If the symbol table contains inlining info, Disassemble() may tag an +# instruction with a location inside an inlined function. But for +# source listings, we prefer to use the location in the function we +# are listing. So use MapToSymbols() to fetch full location +# information for each instruction and then pick out the first +# location from a location list (location list contains callers before +# callees in case of inlining). +# +# After this routine has run, each entry in $instructions contains: +# [0] start address +# [1] filename for function we are listing +# [2] line number for function we are listing +# [3] disassembly +# [4] limit address +# [5] most specific filename (may be different from [1] due to inlining) +# [6] most specific line number (may be different from [2] due to inlining) +sub GetTopLevelLineNumbers { + my ($lib, $offset, $instructions) = @_; + my $pcs = []; + for (my $i = 0; $i <= $#{$instructions}; $i++) { + push(@{$pcs}, $instructions->[$i]->[0]); + } + my $symbols = {}; + MapToSymbols($lib, $offset, $pcs, $symbols); + for (my $i = 0; $i <= $#{$instructions}; $i++) { + my $e = $instructions->[$i]; + push(@{$e}, $e->[1]); + push(@{$e}, $e->[2]); + my $addr = $e->[0]; + my $sym = $symbols->{$addr}; + if (defined($sym)) { + if ($#{$sym} >= 2 && $sym->[1] =~ m/^(.*):(\d+)$/) { + $e->[1] = $1; # File name + $e->[2] = $2; # Line number + } + } + } +} + # Print source-listing for one routine sub PrintSource { my $prog = shift; @@ -1364,9 +1571,12 @@ sub PrintSource { my $cumulative = shift; my $start_addr = shift; my $end_addr = shift; + my $html = shift; + my $output = shift; # Disassemble all instructions (just to get line numbers) my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); + GetTopLevelLineNumbers($prog, $offset, \@instructions); # Hack 1: assume that the first source file encountered in the # disassembly contains the routine @@ -1379,7 +1589,7 @@ sub PrintSource { } if (!defined($filename)) { print STDERR "no filename found in $routine\n"; - return; + return 0; } # Hack 2: assume that the largest line number from $filename is the @@ -1412,7 +1622,7 @@ sub PrintSource { { if (!open(FILE, "<$filename")) { print STDERR "$filename: $!\n"; - return; + return 0; } my $l = 0; my $first_indentation = -1; @@ -1440,12 +1650,24 @@ sub PrintSource { # Assign all samples to the range $firstline,$lastline, # Hack 4: If an instruction does not occur in the range, its samples # are moved to the next instruction that occurs in the range. - my $samples1 = {}; - my $samples2 = {}; - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts + my $samples1 = {}; # Map from line number to flat count + my $samples2 = {}; # Map from line number to cumulative count + my $running1 = 0; # Unassigned flat counts + my $running2 = 0; # Unassigned cumulative counts + my $total1 = 0; # Total flat counts + my $total2 = 0; # Total cumulative counts + my %disasm = (); # Map from line number to disassembly + my $running_disasm = ""; # Unassigned disassembly + my $skip_marker = "---\n"; + if ($html) { + $skip_marker = ""; + for (my $l = $firstline; $l <= $lastline; $l++) { + $disasm{$l} = ""; + } + } + my $last_dis_filename = ''; + my $last_dis_linenum = -1; + my $last_touched_line = -1; # To detect gaps in disassembly for a line foreach my $e (@instructions) { # Add up counts for all address that fall inside this instruction my $c1 = 0; @@ -1454,6 +1676,38 @@ sub PrintSource { $c1 += GetEntry($flat, $a); $c2 += GetEntry($cumulative, $a); } + + if ($html) { + my $dis = sprintf(" %6s %6s \t\t%8s: %s ", + HtmlPrintNumber($c1), + HtmlPrintNumber($c2), + UnparseAddress($offset, $e->[0]), + CleanDisassembly($e->[3])); + + # Append the most specific source line associated with this instruction + if (length($dis) < 80) { $dis .= (' ' x (80 - length($dis))) }; + $dis = HtmlEscape($dis); + my $f = $e->[5]; + my $l = $e->[6]; + if ($f ne $last_dis_filename) { + $dis .= sprintf("%s:%d", + HtmlEscape(CleanFileName($f)), $l); + } elsif ($l ne $last_dis_linenum) { + # De-emphasize the unchanged file name portion + $dis .= sprintf("%s" . + ":%d", + HtmlEscape(CleanFileName($f)), $l); + } else { + # De-emphasize the entire location + $dis .= sprintf("%s:%d", + HtmlEscape(CleanFileName($f)), $l); + } + $last_dis_filename = $f; + $last_dis_linenum = $l; + $running_disasm .= $dis; + $running_disasm .= "\n"; + } + $running1 += $c1; $running2 += $c2; $total1 += $c1; @@ -1468,23 +1722,49 @@ sub PrintSource { AddEntry($samples2, $line, $running2); $running1 = 0; $running2 = 0; + if ($html) { + if ($line != $last_touched_line && $disasm{$line} ne '') { + $disasm{$line} .= "\n"; + } + $disasm{$line} .= $running_disasm; + $running_disasm = ''; + $last_touched_line = $line; + } } } # Assign any leftover samples to $lastline AddEntry($samples1, $lastline, $running1); AddEntry($samples2, $lastline, $running2); - - printf("ROUTINE ====================== %s in %s\n" . - "%6s %6s Total %s (flat / cumulative)\n", - ShortFunctionName($routine), - $filename, - Units(), - Unparse($total1), - Unparse($total2)); + if ($html) { + if ($lastline != $last_touched_line && $disasm{$lastline} ne '') { + $disasm{$lastline} .= "\n"; + } + $disasm{$lastline} .= $running_disasm; + } + + if ($html) { + printf $output ( + "

%s

%s\n
\n" .
+      "Total:%6s %6s (flat / cumulative %s)\n",
+      HtmlEscape(ShortFunctionName($routine)),
+      HtmlEscape(CleanFileName($filename)),
+      Unparse($total1),
+      Unparse($total2),
+      Units());
+  } else {
+    printf $output (
+      "ROUTINE ====================== %s in %s\n" .
+      "%6s %6s Total %s (flat / cumulative)\n",
+      ShortFunctionName($routine),
+      CleanFileName($filename),
+      Unparse($total1),
+      Unparse($total2),
+      Units());
+  }
   if (!open(FILE, "<$filename")) {
     print STDERR "$filename: $!\n";
-    return;
+    return 0;
   }
   my $l = 0;
   while () {
@@ -1494,16 +1774,47 @@ sub PrintSource {
         (($l <= $oldlastline + 5) || ($l <= $lastline))) {
       chop;
       my $text = $_;
-      if ($l == $firstline) { printf("---\n"); }
-      printf("%6s %6s %4d: %s\n",
-             UnparseAlt(GetEntry($samples1, $l)),
-             UnparseAlt(GetEntry($samples2, $l)),
-             $l,
-             $text);
-      if ($l == $lastline)  { printf("---\n"); }
+      if ($l == $firstline) { print $output $skip_marker; }
+      my $n1 = GetEntry($samples1, $l);
+      my $n2 = GetEntry($samples2, $l);
+      if ($html) {
+        # Emit a span that has one of the following classes:
+        #    livesrc -- has samples
+        #    deadsrc -- has disassembly, but with no samples
+        #    nop     -- has no matching disasembly
+        # Also emit an optional span containing disassembly.
+        my $dis = $disasm{$l};
+        my $asm = "";
+        if (defined($dis) && $dis ne '') {
+          $asm = "" . $dis . "";
+        }
+        my $source_class = (($n1 + $n2 > 0) 
+                            ? "livesrc" 
+                            : (($asm ne "") ? "deadsrc" : "nop"));
+        printf $output (
+          "%5d " .
+          "%6s %6s %s%s\n",
+          $l, $source_class,
+          HtmlPrintNumber($n1),
+          HtmlPrintNumber($n2),
+          HtmlEscape($text),
+          $asm);
+      } else {
+        printf $output(
+          "%6s %6s %4d: %s\n",
+          UnparseAlt($n1),
+          UnparseAlt($n2),
+          $l,
+          $text);
+      }
+      if ($l == $lastline)  { print $output $skip_marker; }
     };
   }
   close(FILE);
+  if ($html) {
+    print $output "
\n"; + } + return 1; } # Return the source line for the specified file/linenumber. @@ -1646,21 +1957,11 @@ sub PrintDisassembledFunction { # Print disassembly for (my $x = $first_inst; $x <= $last_inst; $x++) { my $e = $instructions[$x]; - my $address = $e->[0]; - $address = AddressSub($address, $offset); # Make relative to section - $address =~ s/^0x//; - $address =~ s/^0*//; - - # Trim symbols - my $d = $e->[3]; - while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) - while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments - printf("%6s %6s %8s: %6s\n", UnparseAlt($flat_count[$x]), UnparseAlt($cum_count[$x]), - $address, - $d); + UnparseAddress($offset, $e->[0]), + CleanDisassembly($e->[3])); } } } @@ -1706,19 +2007,24 @@ sub PrintDot { # Open DOT output file my $output; + my $escaped_dot = ShellEscape(@DOT); + my $escaped_ps2pdf = ShellEscape(@PS2PDF); if ($main::opt_gv) { - $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "ps")); + $output = "| $escaped_dot -Tps2 >$escaped_outfile"; } elsif ($main::opt_evince) { - $output = "| $DOT -Tps2 | $PS2PDF - " . TempName($main::next_tmpfile, "pdf"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "pdf")); + $output = "| $escaped_dot -Tps2 | $escaped_ps2pdf - $escaped_outfile"; } elsif ($main::opt_ps) { - $output = "| $DOT -Tps2"; + $output = "| $escaped_dot -Tps2"; } elsif ($main::opt_pdf) { - $output = "| $DOT -Tps2 | $PS2PDF - -"; + $output = "| $escaped_dot -Tps2 | $escaped_ps2pdf - -"; } elsif ($main::opt_web || $main::opt_svg) { # We need to post-process the SVG, so write to a temporary file always. - $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); + my $escaped_outfile = ShellEscape(TempName($main::next_tmpfile, "svg")); + $output = "| $escaped_dot -Tsvg >$escaped_outfile"; } elsif ($main::opt_gif) { - $output = "| $DOT -Tgif"; + $output = "| $escaped_dot -Tgif"; } else { $output = ">&STDOUT"; } @@ -1770,7 +2076,7 @@ sub PrintDot { if ($f != $c) { $extra = sprintf("\\rof %s (%s)", Unparse($c), - Percent($c, $overall_total)); + Percent($c, $local_total)); } my $style = ""; if ($main::opt_heapcheck) { @@ -1789,7 +2095,7 @@ sub PrintDot { $node{$a}, $sym, Unparse($f), - Percent($f, $overall_total), + Percent($f, $local_total), $extra, $fs, $style, @@ -1799,10 +2105,12 @@ sub PrintDot { # Get edges and counts per edge my %edge = (); my $n; + my $fullname_to_shortname_map = {}; + FillFullnameToShortnameMap($symbols, $fullname_to_shortname_map); foreach my $k (keys(%{$raw})) { # TODO: omit low %age edges $n = $raw->{$k}; - my @translated = TranslateStack($symbols, $k); + my @translated = TranslateStack($symbols, $fullname_to_shortname_map, $k); for (my $i = 1; $i <= $#translated; $i++) { my $src = $translated[$i]; my $dst = $translated[$i-1]; @@ -2186,6 +2494,50 @@ function handleMouseUp(evt) { EOF } +# Provides a map from fullname to shortname for cases where the +# shortname is ambiguous. The symlist has both the fullname and +# shortname for all symbols, which is usually fine, but sometimes -- +# such as overloaded functions -- two different fullnames can map to +# the same shortname. In that case, we use the address of the +# function to disambiguate the two. This function fills in a map that +# maps fullnames to modified shortnames in such cases. If a fullname +# is not present in the map, the 'normal' shortname provided by the +# symlist is the appropriate one to use. +sub FillFullnameToShortnameMap { + my $symbols = shift; + my $fullname_to_shortname_map = shift; + my $shortnames_seen_once = {}; + my $shortnames_seen_more_than_once = {}; + + foreach my $symlist (values(%{$symbols})) { + # TODO(csilvers): deal with inlined symbols too. + my $shortname = $symlist->[0]; + my $fullname = $symlist->[2]; + if ($fullname !~ /<[0-9a-fA-F]+>$/) { # fullname doesn't end in an address + next; # the only collisions we care about are when addresses differ + } + if (defined($shortnames_seen_once->{$shortname}) && + $shortnames_seen_once->{$shortname} ne $fullname) { + $shortnames_seen_more_than_once->{$shortname} = 1; + } else { + $shortnames_seen_once->{$shortname} = $fullname; + } + } + + foreach my $symlist (values(%{$symbols})) { + my $shortname = $symlist->[0]; + my $fullname = $symlist->[2]; + # TODO(csilvers): take in a list of addresses we care about, and only + # store in the map if $symlist->[1] is in that list. Saves space. + next if defined($fullname_to_shortname_map->{$fullname}); + if (defined($shortnames_seen_more_than_once->{$shortname})) { + if ($fullname =~ /<0*([^>]*)>$/) { # fullname has address at end of it + $fullname_to_shortname_map->{$fullname} = "$shortname\@$1"; + } + } + } +} + # Return a small number that identifies the argument. # Multiple calls with the same argument will return the same number. # Calls with different arguments will return different numbers. @@ -2202,6 +2554,7 @@ sub ShortIdFor { # Translate a stack of addresses into a stack of symbols sub TranslateStack { my $symbols = shift; + my $fullname_to_shortname_map = shift; my $k = shift; my @addrs = split(/\n/, $k); @@ -2233,6 +2586,9 @@ sub TranslateStack { my $func = $symlist->[$j-2]; my $fileline = $symlist->[$j-1]; my $fullfunc = $symlist->[$j]; + if (defined($fullname_to_shortname_map->{$fullfunc})) { + $func = $fullname_to_shortname_map->{$fullfunc}; + } if ($j > 2) { $func = "$func (inline)"; } @@ -2319,6 +2675,16 @@ sub UnparseAlt { } } +# Alternate pretty-printed form: 0 maps to "" +sub HtmlPrintNumber { + my $num = shift; + if ($num == 0) { + return ""; + } else { + return Unparse($num); + } +} + # Return output units sub Units { if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { @@ -2475,6 +2841,13 @@ sub RemoveUninterestingFrames { '__builtin_vec_new', 'operator new', 'operator new[]', + # The entry to our memory-allocation routines on OS X + 'malloc_zone_malloc', + 'malloc_zone_calloc', + 'malloc_zone_valloc', + 'malloc_zone_realloc', + 'malloc_zone_memalign', + 'malloc_zone_free', # These mark the beginning/end of our custom sections '__start_google_malloc', '__stop_google_malloc', @@ -2566,9 +2939,11 @@ sub ReduceProfile { my $symbols = shift; my $profile = shift; my $result = {}; + my $fullname_to_shortname_map = {}; + FillFullnameToShortnameMap($symbols, $fullname_to_shortname_map); foreach my $k (keys(%{$profile})) { my $count = $profile->{$k}; - my @translated = TranslateStack($symbols, $k); + my @translated = TranslateStack($symbols, $fullname_to_shortname_map, $k); my @path = (); my %seen = (); $seen{''} = 1; # So that empty keys are skipped @@ -2775,7 +3150,8 @@ sub AddEntries { sub CheckSymbolPage { my $url = SymbolPageURL(); - open(SYMBOL, "$URL_FETCHER '$url' |"); + my $command = ShellEscape(@URL_FETCHER, $url); + open(SYMBOL, "$command |") or error($command); my $line = ; $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines close(SYMBOL); @@ -2832,7 +3208,7 @@ sub SymbolPageURL { sub FetchProgramName() { my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); my $url = "$baseURL$PROGRAM_NAME_PAGE"; - my $command_line = "$URL_FETCHER '$url'"; + my $command_line = ShellEscape(@URL_FETCHER, $url); open(CMDLINE, "$command_line |") or error($command_line); my $cmdline = ; $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines @@ -2849,7 +3225,7 @@ sub FetchProgramName() { # curl. Redirection happens on borg hosts. sub ResolveRedirectionForCurl { my $url = shift; - my $command_line = "$URL_FETCHER --head '$url'"; + my $command_line = ShellEscape(@URL_FETCHER, "--head", $url); open(CMDLINE, "$command_line |") or error($command_line); while () { s/\r//g; # turn windows-looking lines into unix-looking lines @@ -2861,18 +3237,18 @@ sub ResolveRedirectionForCurl { return $url; } -# Add a timeout flat to URL_FETCHER +# Add a timeout flat to URL_FETCHER. Returns a new list. sub AddFetchTimeout { - my $fetcher = shift; my $timeout = shift; + my @fetcher = shift; if (defined($timeout)) { - if ($fetcher =~ m/\bcurl -s/) { - $fetcher .= sprintf(" --max-time %d", $timeout); - } elsif ($fetcher =~ m/\brpcget\b/) { - $fetcher .= sprintf(" --deadline=%d", $timeout); + if (join(" ", @fetcher) =~ m/\bcurl -s/) { + push(@fetcher, "--max-time", sprintf("%d", $timeout)); + } elsif (join(" ", @fetcher) =~ m/\brpcget\b/) { + push(@fetcher, sprintf("--deadline=%d", $timeout)); } } - return $fetcher; + return @fetcher; } # Reads a symbol map from the file handle name given as $1, returning @@ -2932,15 +3308,17 @@ sub FetchSymbols { my $url = SymbolPageURL(); my $command_line; - if ($URL_FETCHER =~ m/\bcurl -s/) { + if (join(" ", @URL_FETCHER) =~ m/\bcurl -s/) { $url = ResolveRedirectionForCurl($url); - $command_line = "$URL_FETCHER -d '\@$main::tmpfile_sym' '$url'"; + $command_line = ShellEscape(@URL_FETCHER, "-d", "\@$main::tmpfile_sym", + $url); } else { - $command_line = "$URL_FETCHER --post '$url' < '$main::tmpfile_sym'"; + $command_line = (ShellEscape(@URL_FETCHER, "--post", $url) + . " < " . ShellEscape($main::tmpfile_sym)); } # We use c++filt in case $SYMBOL_PAGE gives us mangled symbols. - my $cppfilt = $obj_tool_map{"c++filt"}; - open(SYMBOL, "$command_line | $cppfilt |") or error($command_line); + my $escaped_cppfilt = ShellEscape($obj_tool_map{"c++filt"}); + open(SYMBOL, "$command_line | $escaped_cppfilt |") or error($command_line); $symbol_map = ReadSymbols(*SYMBOL{IO}); close(SYMBOL); } @@ -2956,8 +3334,8 @@ sub FetchSymbols { my $shortpc = $pc; $shortpc =~ s/^0*//; # Each line may have a list of names, which includes the function - # and also other functions it has inlined. They are separated - # (in PrintSymbolizedFile), by --, which is illegal in function names. + # and also other functions it has inlined. They are separated (in + # PrintSymbolizedProfile), by --, which is illegal in function names. my $fullnames; if (defined($symbol_map->{$shortpc})) { $fullnames = $symbol_map->{$shortpc}; @@ -3035,8 +3413,8 @@ sub FetchDynamicProfile { return $real_profile; } - my $fetcher = AddFetchTimeout($URL_FETCHER, $fetch_timeout); - my $cmd = "$fetcher '$url' > '$tmp_profile'"; + my @fetcher = AddFetchTimeout($fetch_timeout, @URL_FETCHER); + my $cmd = ShellEscape(@fetcher, $url) . " > " . ShellEscape($tmp_profile); if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE|$CENSUSPROFILE_PAGE/){ print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; if ($encourage_patience) { @@ -3047,7 +3425,7 @@ sub FetchDynamicProfile { } (system($cmd) == 0) || error("Failed to get profile: $cmd: $!\n"); - (system("mv $tmp_profile $real_profile") == 0) || error("Unable to rename profile\n"); + (system("mv", $tmp_profile, $real_profile) == 0) || error("Unable to rename profile\n"); print STDERR "Wrote profile to $real_profile\n"; $main::collected_profile = $real_profile; return $main::collected_profile; @@ -3161,7 +3539,7 @@ BEGIN { my $has_q = 0; eval { $has_q = pack("Q", "1") ? 1 : 1; }; if (!$has_q) { - $self->{perl_is_64bit} = 0; + $self->{perl_is_64bit} = 0; } read($self->{file}, $str, 8); if (substr($str, 4, 4) eq chr(0)x4) { @@ -3197,17 +3575,17 @@ BEGIN { # TODO(csilvers): if this is a 32-bit perl, the math below # could end up in a too-large int, which perl will promote # to a double, losing necessary precision. Deal with that. - # Right now, we just die. - my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); + # Right now, we just die. + my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); if ($self->{unpack_code} eq 'N') { # big-endian - ($lo, $hi) = ($hi, $lo); - } - my $value = $lo + $hi * (2**32); - if (!$self->{perl_is_64bit} && # check value is exactly represented - (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { - ::error("Need a 64-bit perl to process this 64-bit profile.\n"); - } - push(@b64_values, $value); + ($lo, $hi) = ($hi, $lo); + } + my $value = $lo + $hi * (2**32); + if (!$self->{perl_is_64bit} && # check value is exactly represented + (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { + ::error("Need a 64-bit perl to process this 64-bit profile.\n"); + } + push(@b64_values, $value); } @$slots = @b64_values; } @@ -3335,7 +3713,7 @@ sub ReadProfile { if (!$main::use_symbolized_profile) { # we have both a binary and symbolized profiles, abort error("FATAL ERROR: Symbolized profile\n $fname\ncannot be used with " . - "a binary arg. Try again without passing\n $prog\n"); + "a binary arg. Try again without passing\n $prog\n"); } # Read the symbol section of the symbolized profile file. $symbols = ReadSymbols(*PROFILE{IO}); @@ -3636,18 +4014,18 @@ sub ReadHeapProfile { # The sampling frequency is the rate of a Poisson process. # This means that the probability of sampling an allocation of # size X with sampling rate Y is 1 - exp(-X/Y) - if ($n1 != 0) { - my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - } - if ($n2 != 0) { - my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } + if ($n1 != 0) { + my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n1 *= $scale_factor; + $s1 *= $scale_factor; + } + if ($n2 != 0) { + my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); + my $scale_factor = 1/(1 - exp(-$ratio)); + $n2 *= $scale_factor; + $s2 *= $scale_factor; + } } else { # Remote-heap version 1 my $ratio; @@ -3771,19 +4149,19 @@ sub ReadSynchProfile { return $r; } -# Given a hex value in the form "0x1abcd" return "0001abcd" or -# "000000000001abcd", depending on the current address length. -# There's probably a more idiomatic (or faster) way to do this... +# Given a hex value in the form "0x1abcd" or "1abcd", return either +# "0001abcd" or "000000000001abcd", depending on the current (global) +# address length. sub HexExtend { my $addr = shift; - $addr =~ s/^0x//; - - if (length $addr > $address_length) { - printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + $addr =~ s/^(0x)?0*//; + my $zeros_needed = $address_length - length($addr); + if ($zeros_needed < 0) { + printf STDERR "Warning: address $addr is longer than address length $address_length\n"; + return $addr; } - - return substr("000000000000000".$addr, -$address_length); + return ("0" x $zeros_needed) . $addr; } ##### Symbol extraction ##### @@ -3834,9 +4212,8 @@ sub ParseTextSectionHeaderFromObjdump { my $file_offset; # Get objdump output from the library file to figure out how to # map between mapped addresses and addresses in the library. - my $objdump = $obj_tool_map{"objdump"}; - open(OBJDUMP, "$objdump -h $lib |") - || error("$objdump $lib: $!\n"); + my $cmd = ShellEscape($obj_tool_map{"objdump"}, "-h", $lib); + open(OBJDUMP, "$cmd |") || error("$cmd: $!\n"); while () { s/\r//g; # turn windows-looking lines into unix-looking lines # Idx Name Size VMA LMA File off Algn @@ -3874,9 +4251,8 @@ sub ParseTextSectionHeaderFromOtool { my $file_offset = undef; # Get otool output from the library file to figure out how to # map between mapped addresses and addresses in the library. - my $otool = $obj_tool_map{"otool"}; - open(OTOOL, "$otool -l $lib |") - || error("$otool $lib: $!\n"); + my $command = ShellEscape($obj_tool_map{"otool"}, "-l", $lib); + open(OTOOL, "$command |") || error("$command: $!\n"); my $cmd = ""; my $sectname = ""; my $segname = ""; @@ -4218,18 +4594,18 @@ sub ExtractSymbols { my ($start_pc_index, $finish_pc_index); # Find smallest finish_pc_index such that $finish < $pc[$finish_pc_index]. for ($finish_pc_index = $#pcs + 1; $finish_pc_index > 0; - $finish_pc_index--) { + $finish_pc_index--) { last if $pcs[$finish_pc_index - 1] le $finish; } # Find smallest start_pc_index such that $start <= $pc[$start_pc_index]. for ($start_pc_index = $finish_pc_index; $start_pc_index > 0; - $start_pc_index--) { + $start_pc_index--) { last if $pcs[$start_pc_index - 1] lt $start; } # This keeps PC values higher than $pc[$finish_pc_index] in @pcs, # in case there are overlaps in libraries and the main binary. @{$contained} = splice(@pcs, $start_pc_index, - $finish_pc_index - $start_pc_index); + $finish_pc_index - $start_pc_index); # Map to symbols MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); } @@ -4251,15 +4627,15 @@ sub MapToSymbols { # Figure out the addr2line command to use my $addr2line = $obj_tool_map{"addr2line"}; - my $cmd = "$addr2line -f -C -e $image"; + my $cmd = ShellEscape($addr2line, "-f", "-C", "-e", $image); if (exists $obj_tool_map{"addr2line_pdb"}) { $addr2line = $obj_tool_map{"addr2line_pdb"}; - $cmd = "$addr2line --demangle -f -C -e $image"; + $cmd = ShellEscape($addr2line, "--demangle", "-f", "-C", "-e", $image); } # If "addr2line" isn't installed on the system at all, just use # nm to get what info we can (function names, but not line numbers). - if (system("$addr2line --help >/dev/null 2>&1") != 0) { + if (system(ShellEscape($addr2line, "--help") . " >$dev_null 2>&1") != 0) { MapSymbolsWithNM($image, $offset, $pclist, $symbols); return; } @@ -4273,11 +4649,10 @@ sub MapToSymbols { $sep_address = undef; # May be filled in by MapSymbolsWithNM() my $nm_symbols = {}; MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); - # TODO(csilvers): only add '-i' if addr2line supports it. if (defined($sep_address)) { # Only add " -i" to addr2line if the binary supports it. # addr2line --help returns 0, but not if it sees an unknown flag first. - if (system("$cmd -i --help >/dev/null 2>&1") == 0) { + if (system("$cmd -i --help >$dev_null 2>&1") == 0) { $cmd .= " -i"; } else { $sep_address = undef; # no need for sep_address if we don't support -i @@ -4299,13 +4674,14 @@ sub MapToSymbols { close(ADDRESSES); if ($debug) { print("----\n"); - system("cat $main::tmpfile_sym"); + system("cat", $main::tmpfile_sym); print("----\n"); - system("$cmd <$main::tmpfile_sym"); + system("$cmd < " . ShellEscape($main::tmpfile_sym)); print("----\n"); } - open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); + open(SYMBOLS, "$cmd <" . ShellEscape($main::tmpfile_sym) . " |") + || error("$cmd: $!\n"); my $count = 0; # Index in pclist while () { # Read fullfunction and filelineinfo from next pair of lines @@ -4325,15 +4701,29 @@ sub MapToSymbols { my $pcstr = $pclist->[$count]; my $function = ShortFunctionName($fullfunction); - if ($fullfunction eq '??') { - # See if nm found a symbol - my $nms = $nm_symbols->{$pcstr}; - if (defined($nms)) { + my $nms = $nm_symbols->{$pcstr}; + if (defined($nms)) { + if ($fullfunction eq '??') { + # nm found a symbol for us. $function = $nms->[0]; $fullfunction = $nms->[2]; + } else { + # MapSymbolsWithNM tags each routine with its starting address, + # useful in case the image has multiple occurrences of this + # routine. (It uses a syntax that resembles template paramters, + # that are automatically stripped out by ShortFunctionName().) + # addr2line does not provide the same information. So we check + # if nm disambiguated our symbol, and if so take the annotated + # (nm) version of the routine-name. TODO(csilvers): this won't + # catch overloaded, inlined symbols, which nm doesn't see. + # Better would be to do a check similar to nm's, in this fn. + if ($nms->[2] =~ m/^\Q$function\E/) { # sanity check it's the right fn + $function = $nms->[0]; + $fullfunction = $nms->[2]; + } } } - + # Prepend to accumulated symbols for pcstr # (so that caller comes before callee) my $sym = $symbols->{$pcstr}; @@ -4344,7 +4734,7 @@ sub MapToSymbols { unshift(@{$sym}, $function, $filelinenum, $fullfunction); if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } if (!defined($sep_address)) { - # Inlining is off, se this entry ends immediately + # Inlining is off, so this entry ends immediately $count++; } } @@ -4407,6 +4797,31 @@ sub ShortFunctionName { return $function; } +# Trim overly long symbols found in disassembler output +sub CleanDisassembly { + my $d = shift; + while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) + while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments + return $d; +} + +# Clean file name for display +sub CleanFileName { + my ($f) = @_; + $f =~ s|^/proc/self/cwd/||; + $f =~ s|^\./||; + return $f; +} + +# Make address relative to section and clean up for display +sub UnparseAddress { + my ($offset, $address) = @_; + $address = AddressSub($address, $offset); + $address =~ s/^0x//; + $address =~ s/^0*//; + return $address; +} + ##### Miscellaneous ##### # Find the right versions of the above object tools to use. The @@ -4423,8 +4838,18 @@ sub ConfigureObjTools { # predictably return error status in prod. (-e $prog_file) || error("$prog_file does not exist.\n"); - # Follow symlinks (at least for systems where "file" supports that) - my $file_type = `/usr/bin/file -L $prog_file 2>/dev/null || /usr/bin/file $prog_file`; + my $file_type = undef; + if (-e "/usr/bin/file") { + # Follow symlinks (at least for systems where "file" supports that). + my $escaped_prog_file = ShellEscape($prog_file); + $file_type = `/usr/bin/file -L $escaped_prog_file 2>$dev_null || + /usr/bin/file $escaped_prog_file`; + } elsif ($^O == "MSWin32") { + $file_type = "MS Windows"; + } else { + print STDERR "WARNING: Can't determine the file type of $prog_file"; + } + if ($file_type =~ /64-bit/) { # Change $address_length to 16 if the program file is ELF 64-bit. # We can't detect this from many (most?) heap or lock contention @@ -4500,6 +4925,19 @@ sub ConfigureTool { return $path; } +sub ShellEscape { + my @escaped_words = (); + foreach my $word (@_) { + my $escaped_word = $word; + if ($word =~ m![^a-zA-Z0-9/.,_=-]!) { # check for anything not in whitelist + $escaped_word =~ s/'/'\\''/; + $escaped_word = "'$escaped_word'"; + } + push(@escaped_words, $escaped_word); + } + return join(" ", @escaped_words); +} + sub cleanup { unlink($main::tmpfile_sym); unlink(keys %main::tempnames); @@ -4537,11 +4975,11 @@ sub error { # names match "$regexp" and returns them in a hashtable mapping from # procedure name to a two-element vector of [start address, end address] sub GetProcedureBoundariesViaNm { - my $nm_command = shift; + my $escaped_nm_command = shift; # shell-escaped my $regexp = shift; my $symbol_table = {}; - open(NM, "$nm_command |") || error("$nm_command: $!\n"); + open(NM, "$escaped_nm_command |") || error("$escaped_nm_command: $!\n"); my $last_start = "0"; my $routine = ""; while () { @@ -4619,6 +5057,21 @@ sub GetProcedureBoundaries { my $image = shift; my $regexp = shift; + # If $image doesn't start with /, then put ./ in front of it. This works + # around an obnoxious bug in our probing of nm -f behavior. + # "nm -f $image" is supposed to fail on GNU nm, but if: + # + # a. $image starts with [BbSsPp] (for example, bin/foo/bar), AND + # b. you have a.out in your current directory (a not uncommon occurence) + # + # then "nm -f $image" succeeds because -f only looks at the first letter of + # the argument, which looks valid because it's [BbSsPp], and then since + # there's no image provided, it looks for a.out and finds it. + # + # This regex makes sure that $image starts with . or /, forcing the -f + # parsing to fail since . and / are not valid formats. + $image =~ s#^[^/]#./$&#; + # For libc libraries, the copy in /usr/lib/debug contains debugging symbols my $debugging = DebuggingLibrary($image); if ($debugging) { @@ -4636,28 +5089,29 @@ sub GetProcedureBoundaries { # --demangle and -f. my $demangle_flag = ""; my $cppfilt_flag = ""; - if (system("$nm --demangle $image >/dev/null 2>&1") == 0) { + my $to_devnull = ">$dev_null 2>&1"; + if (system(ShellEscape($nm, "--demangle", "image") . $to_devnull) == 0) { # In this mode, we do "nm --demangle " $demangle_flag = "--demangle"; $cppfilt_flag = ""; - } elsif (system("$cppfilt $image >/dev/null 2>&1") == 0) { + } elsif (system(ShellEscape($cppfilt, $image) . $to_devnull) == 0) { # In this mode, we do "nm | c++filt" - $cppfilt_flag = " | $cppfilt"; + $cppfilt_flag = " | " . ShellEscape($cppfilt); }; my $flatten_flag = ""; - if (system("$nm -f $image >/dev/null 2>&1") == 0) { + if (system(ShellEscape($nm, "-f", $image) . $to_devnull) == 0) { $flatten_flag = "-f"; } # Finally, in the case $imagie isn't a debug library, we try again with # -D to at least get *exported* symbols. If we can't use --demangle, # we use c++filt instead, if it exists on this system. - my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . - " $image 2>/dev/null $cppfilt_flag", - "$nm -D -n $flatten_flag $demangle_flag" . - " $image 2>/dev/null $cppfilt_flag", + my @nm_commands = (ShellEscape($nm, "-n", $flatten_flag, $demangle_flag, + $image) . " 2>$dev_null $cppfilt_flag", + ShellEscape($nm, "-D", "-n", $flatten_flag, $demangle_flag, + $image) . " 2>$dev_null $cppfilt_flag", # 6nm is for Go binaries - "6nm $image 2>/dev/null | sort", + ShellEscape("6nm", "$image") . " 2>$dev_null | sort", ); # If the executable is an MS Windows PDB-format executable, we'll @@ -4665,8 +5119,9 @@ sub GetProcedureBoundaries { # want to use both unix nm and windows-specific nm_pdb, since # PDB-format executables can apparently include dwarf .o files. if (exists $obj_tool_map{"nm_pdb"}) { - my $nm_pdb = $obj_tool_map{"nm_pdb"}; - push(@nm_commands, "$nm_pdb --demangle $image 2>/dev/null"); + push(@nm_commands, + ShellEscape($obj_tool_map{"nm_pdb"}, "--demangle", $image) + . " 2>$dev_null"); } foreach my $nm_command (@nm_commands) { diff --git a/deps/jemalloc/config.guess b/deps/jemalloc/config.guess index 0773d0f6312..d622a44e551 100755 --- a/deps/jemalloc/config.guess +++ b/deps/jemalloc/config.guess @@ -1,9 +1,10 @@ #! /bin/sh # Attempt to guess a canonical system name. # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012 Free Software Foundation, Inc. -timestamp='2004-03-03' +timestamp='2012-02-10' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -16,24 +17,24 @@ timestamp='2004-03-03' # General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# along with this program; if not, see . # # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. -# Originally written by Per Bothner . -# Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. + +# Originally written by Per Bothner. Please send patches (context +# diff format) to and include a ChangeLog +# entry. # # This script attempts to guess a canonical system name similar to # config.sub. If it succeeds, it prints the system name on stdout, and # exits with 0. Otherwise, it exits with 1. # -# The plan is that this can be called by configure scripts if you -# don't specify an explicit build system type. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD me=`echo "$0" | sed -e 's,.*/,,'` @@ -53,7 +54,8 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, +2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO @@ -66,11 +68,11 @@ Try \`$me --help' for more information." while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; + echo "$timestamp" ; exit ;; --version | -v ) - echo "$version" ; exit 0 ;; + echo "$version" ; exit ;; --help | --h* | -h ) - echo "$usage"; exit 0 ;; + echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. @@ -104,7 +106,7 @@ set_cc_for_build=' trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; : ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d -q "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || + { tmp=`(umask 077 && mktemp -d "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; @@ -123,7 +125,7 @@ case $CC_FOR_BUILD,$HOST_CC,$CC in ;; ,,*) CC_FOR_BUILD=$CC ;; ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ;' +esac ; set_cc_for_build= ;' # This is needed to find uname on a Pyramid OSx when run in the BSD universe. # (ghazi@noc.rutgers.edu 1994-08-24) @@ -141,7 +143,7 @@ UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:NetBSD:*:*) # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, + # more of the tuples: *-*-netbsdelf*, *-*-netbsdaout*, # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently # switched to ELF, *-*-netbsd* would select the old # object file format. This provides both forward @@ -158,6 +160,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; + sh5el) machine=sh5le-unknown ;; *) machine=${UNAME_MACHINE_ARCH}-unknown ;; esac # The Operating System including object format, if it has switched @@ -166,7 +169,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep __ELF__ >/dev/null + | grep -q __ELF__ then # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). # Return netbsd for either. FIX? @@ -176,7 +179,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in fi ;; *) - os=netbsd + os=netbsd ;; esac # The OS release @@ -196,71 +199,30 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. echo "${machine}-${os}${release}" - exit 0 ;; - amd64:OpenBSD:*:*) - echo x86_64-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - amiga:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - arc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - cats:OpenBSD:*:*) - echo arm-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - hp300:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mac68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - macppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme88k:OpenBSD:*:*) - echo m88k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvmeppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pegasos:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pmax:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sgi:OpenBSD:*:*) - echo mipseb-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sun3:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - wgrisc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; + exit ;; *:OpenBSD:*:*) - echo ${UNAME_MACHINE}-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; + UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` + echo ${UNAME_MACHINE_ARCH}-unknown-openbsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} - exit 0 ;; + exit ;; + *:SolidBSD:*:*) + echo ${UNAME_MACHINE}-unknown-solidbsd${UNAME_RELEASE} + exit ;; macppc:MirBSD:*:*) - echo powerppc-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; + echo powerpc-unknown-mirbsd${UNAME_RELEASE} + exit ;; *:MirBSD:*:*) echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` ;; *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` + UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` ;; esac # According to Compaq, /usr/sbin/psrinfo has been available on @@ -306,40 +268,46 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - exit 0 ;; - Alpha*:OpenVMS:*:*) - echo alpha-hp-vms - exit 0 ;; + # Reset EXIT trap before exiting to avoid spurious non-zero exit code. + exitcode=$? + trap '' 0 + exit $exitcode ;; Alpha\ *:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # Should we change UNAME_MACHINE based on the output of uname instead # of the specific Alpha model? echo alpha-pc-interix - exit 0 ;; + exit ;; 21064:Windows_NT:50:3) echo alpha-dec-winnt3.5 - exit 0 ;; + exit ;; Amiga*:UNIX_System_V:4.0:*) echo m68k-unknown-sysv4 - exit 0;; + exit ;; *:[Aa]miga[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-amigaos - exit 0 ;; + exit ;; *:[Mm]orph[Oo][Ss]:*:*) echo ${UNAME_MACHINE}-unknown-morphos - exit 0 ;; + exit ;; *:OS/390:*:*) echo i370-ibm-openedition - exit 0 ;; + exit ;; + *:z/VM:*:*) + echo s390-ibm-zvmoe + exit ;; *:OS400:*:*) - echo powerpc-ibm-os400 - exit 0 ;; + echo powerpc-ibm-os400 + exit ;; arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) echo arm-acorn-riscix${UNAME_RELEASE} - exit 0;; + exit ;; + arm:riscos:*:*|arm:RISCOS:*:*) + echo arm-unknown-riscos + exit ;; SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) echo hppa1.1-hitachi-hiuxmpp - exit 0;; + exit ;; Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. if test "`(/bin/universe) 2>/dev/null`" = att ; then @@ -347,32 +315,51 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in else echo pyramid-pyramid-bsd fi - exit 0 ;; + exit ;; NILE*:*:*:dcosx) echo pyramid-pyramid-svr4 - exit 0 ;; + exit ;; DRS?6000:unix:4.0:6*) echo sparc-icl-nx6 - exit 0 ;; - DRS?6000:UNIX_SV:4.2*:7*) + exit ;; + DRS?6000:UNIX_SV:4.2*:7* | DRS?6000:isis:4.2*:7*) case `/usr/bin/uname -p` in - sparc) echo sparc-icl-nx7 && exit 0 ;; + sparc) echo sparc-icl-nx7; exit ;; esac ;; + s390x:SunOS:*:*) + echo ${UNAME_MACHINE}-ibm-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; sun4H:SunOS:5.*:*) echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - i86pc:SunOS:5.*:*) - echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; + i86pc:AuroraUX:5.*:* | i86xen:AuroraUX:5.*:*) + echo i386-pc-auroraux${UNAME_RELEASE} + exit ;; + i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) + eval $set_cc_for_build + SUN_ARCH="i386" + # If there is a compiler, see if it is configured for 64-bit objects. + # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. + # This test works for both compilers. + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + SUN_ARCH="x86_64" + fi + fi + echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` + exit ;; sun4*:SunOS:6*:*) # According to config.sub, this is the proper way to canonicalize # SunOS6. Hard to guess exactly what SunOS6 will be like, but # it's likely to be more like Solaris than SunOS4. echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; sun4*:SunOS:*:*) case "`/usr/bin/arch -k`" in Series*|S4*) @@ -381,10 +368,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in esac # Japanese Language versions have a version number like `4.1.3-JL'. echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` - exit 0 ;; + exit ;; sun3*:SunOS:*:*) echo m68k-sun-sunos${UNAME_RELEASE} - exit 0 ;; + exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 @@ -396,10 +383,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in echo sparc-sun-sunos${UNAME_RELEASE} ;; esac - exit 0 ;; + exit ;; aushp:SunOS:*:*) echo sparc-auspex-sunos${UNAME_RELEASE} - exit 0 ;; + exit ;; # The situation for MiNT is a little confusing. The machine name # can be virtually everything (everything which is not # "atarist" or "atariste" at least should have a processor @@ -409,41 +396,41 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # MiNT. But MiNT is downward compatible to TOS, so this should # be no problem. atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + exit ;; *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-atari-mint${UNAME_RELEASE} + exit ;; milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-milan-mint${UNAME_RELEASE} + exit ;; hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-hades-mint${UNAME_RELEASE} + exit ;; *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} - exit 0 ;; + echo m68k-unknown-mint${UNAME_RELEASE} + exit ;; m68k:machten:*:*) echo m68k-apple-machten${UNAME_RELEASE} - exit 0 ;; + exit ;; powerpc:machten:*:*) echo powerpc-apple-machten${UNAME_RELEASE} - exit 0 ;; + exit ;; RISC*:Mach:*:*) echo mips-dec-mach_bsd4.3 - exit 0 ;; + exit ;; RISC*:ULTRIX:*:*) echo mips-dec-ultrix${UNAME_RELEASE} - exit 0 ;; + exit ;; VAX*:ULTRIX*:*:*) echo vax-dec-ultrix${UNAME_RELEASE} - exit 0 ;; + exit ;; 2020:CLIX:*:* | 2430:CLIX:*:*) echo clipper-intergraph-clix${UNAME_RELEASE} - exit 0 ;; + exit ;; mips:*:*:UMIPS | mips:*:*:RISCos) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c @@ -467,35 +454,36 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit (-1); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c \ - && $dummy `echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` \ - && exit 0 + $CC_FOR_BUILD -o $dummy $dummy.c && + dummyarg=`echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` && + SYSTEM_NAME=`$dummy $dummyarg` && + { echo "$SYSTEM_NAME"; exit; } echo mips-mips-riscos${UNAME_RELEASE} - exit 0 ;; + exit ;; Motorola:PowerMAX_OS:*:*) echo powerpc-motorola-powermax - exit 0 ;; + exit ;; Motorola:*:4.3:PL8-*) echo powerpc-harris-powermax - exit 0 ;; + exit ;; Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) echo powerpc-harris-powermax - exit 0 ;; + exit ;; Night_Hawk:Power_UNIX:*:*) echo powerpc-harris-powerunix - exit 0 ;; + exit ;; m88k:CX/UX:7*:*) echo m88k-harris-cxux7 - exit 0 ;; + exit ;; m88k:*:4*:R4*) echo m88k-motorola-sysv4 - exit 0 ;; + exit ;; m88k:*:3*:R3*) echo m88k-motorola-sysv3 - exit 0 ;; + exit ;; AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` + # DG/UX returns AViiON for all architectures + UNAME_PROCESSOR=`/usr/bin/uname -p` if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] then if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ @@ -508,29 +496,29 @@ EOF else echo i586-dg-dgux${UNAME_RELEASE} fi - exit 0 ;; + exit ;; M88*:DolphinOS:*:*) # DolphinOS (SVR3) echo m88k-dolphin-sysv3 - exit 0 ;; + exit ;; M88*:*:R3*:*) # Delta 88k system running SVR3 echo m88k-motorola-sysv3 - exit 0 ;; + exit ;; XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) echo m88k-tektronix-sysv3 - exit 0 ;; + exit ;; Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) echo m68k-tektronix-bsd - exit 0 ;; + exit ;; *:IRIX*:*:*) echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` - exit 0 ;; + exit ;; ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit 0 ;; # Note that: echo "'`uname -s`'" gives 'AIX ' + echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id + exit ;; # Note that: echo "'`uname -s`'" gives 'AIX ' i*86:AIX:*:*) echo i386-ibm-aix - exit 0 ;; + exit ;; ia64:AIX:*:*) if [ -x /usr/bin/oslevel ] ; then IBM_REV=`/usr/bin/oslevel` @@ -538,7 +526,7 @@ EOF IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} - exit 0 ;; + exit ;; *:AIX:2:3) if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then eval $set_cc_for_build @@ -553,15 +541,19 @@ EOF exit(0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 - echo rs6000-ibm-aix3.2.5 + if $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` + then + echo "$SYSTEM_NAME" + else + echo rs6000-ibm-aix3.2.5 + fi elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then echo rs6000-ibm-aix3.2.4 else echo rs6000-ibm-aix3.2 fi - exit 0 ;; - *:AIX:*:[45]) + exit ;; + *:AIX:*:[4567]) IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then IBM_ARCH=rs6000 @@ -574,28 +566,28 @@ EOF IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} fi echo ${IBM_ARCH}-ibm-aix${IBM_REV} - exit 0 ;; + exit ;; *:AIX:*:*) echo rs6000-ibm-aix - exit 0 ;; + exit ;; ibmrt:4.4BSD:*|romp-ibm:BSD:*) echo romp-ibm-bsd4.4 - exit 0 ;; + exit ;; ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to - exit 0 ;; # report: romp-ibm BSD 4.3 + exit ;; # report: romp-ibm BSD 4.3 *:BOSX:*:*) echo rs6000-bull-bosx - exit 0 ;; + exit ;; DPX/2?00:B.O.S.:*:*) echo m68k-bull-sysv3 - exit 0 ;; + exit ;; 9000/[34]??:4.3bsd:1.*:*) echo m68k-hp-bsd - exit 0 ;; + exit ;; hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) echo m68k-hp-bsd4.4 - exit 0 ;; + exit ;; 9000/[34678]??:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` case "${UNAME_MACHINE}" in @@ -604,52 +596,52 @@ EOF 9000/[678][0-9][0-9]) if [ -x /usr/bin/getconf ]; then sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; + sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` + case "${sc_cpu_version}" in + 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 + 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 532) # CPU_PA_RISC2_0 + case "${sc_kernel_bits}" in + 32) HP_ARCH="hppa2.0n" ;; + 64) HP_ARCH="hppa2.0w" ;; '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 - esac ;; - esac + esac ;; + esac fi if [ "${HP_ARCH}" = "" ]; then eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c + sed 's/^ //' << EOF >$dummy.c - #define _HPUX_SOURCE - #include - #include + #define _HPUX_SOURCE + #include + #include - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); + int main () + { + #if defined(_SC_KERNEL_BITS) + long bits = sysconf(_SC_KERNEL_BITS); + #endif + long cpu = sysconf (_SC_CPU_VERSION); - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } + switch (cpu) + { + case CPU_PA_RISC1_0: puts ("hppa1.0"); break; + case CPU_PA_RISC1_1: puts ("hppa1.1"); break; + case CPU_PA_RISC2_0: + #if defined(_SC_KERNEL_BITS) + switch (bits) + { + case 64: puts ("hppa2.0w"); break; + case 32: puts ("hppa2.0n"); break; + default: puts ("hppa2.0"); break; + } break; + #else /* !defined(_SC_KERNEL_BITS) */ + puts ("hppa2.0"); break; + #endif + default: puts ("hppa1.0"); break; + } + exit (0); + } EOF (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa @@ -657,9 +649,19 @@ EOF esac if [ ${HP_ARCH} = "hppa2.0w" ] then - # avoid double evaluation of $set_cc_for_build - test -n "$CC_FOR_BUILD" || eval $set_cc_for_build - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E -) | grep __LP64__ >/dev/null + eval $set_cc_for_build + + # hppa2.0w-hp-hpux* has a 64-bit kernel and a compiler generating + # 32-bit code. hppa64-hp-hpux* has the same kernel and a compiler + # generating 64-bit code. GNU and HP use different nomenclature: + # + # $ CC_FOR_BUILD=cc ./config.guess + # => hppa2.0w-hp-hpux11.23 + # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess + # => hppa64-hp-hpux11.23 + + if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + grep -q __LP64__ then HP_ARCH="hppa2.0w" else @@ -667,11 +669,11 @@ EOF fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} - exit 0 ;; + exit ;; ia64:HP-UX:*:*) HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` echo ia64-hp-hpux${HPUX_REV} - exit 0 ;; + exit ;; 3050*:HI-UX:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c @@ -699,221 +701,266 @@ EOF exit (0); } EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 + $CC_FOR_BUILD -o $dummy $dummy.c && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } echo unknown-hitachi-hiuxwe2 - exit 0 ;; + exit ;; 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) echo hppa1.1-hp-bsd - exit 0 ;; + exit ;; 9000/8??:4.3bsd:*:*) echo hppa1.0-hp-bsd - exit 0 ;; + exit ;; *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) echo hppa1.0-hp-mpeix - exit 0 ;; + exit ;; hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) echo hppa1.1-hp-osf - exit 0 ;; + exit ;; hp8??:OSF1:*:*) echo hppa1.0-hp-osf - exit 0 ;; + exit ;; i*86:OSF1:*:*) if [ -x /usr/sbin/sysversion ] ; then echo ${UNAME_MACHINE}-unknown-osf1mk else echo ${UNAME_MACHINE}-unknown-osf1 fi - exit 0 ;; + exit ;; parisc*:Lites*:*:*) echo hppa1.1-hp-lites - exit 0 ;; + exit ;; C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) echo c1-convex-bsd - exit 0 ;; + exit ;; C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit 0 ;; + exit ;; C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) echo c34-convex-bsd - exit 0 ;; + exit ;; C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) echo c38-convex-bsd - exit 0 ;; + exit ;; C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) echo c4-convex-bsd - exit 0 ;; + exit ;; CRAY*Y-MP:*:*:*) echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*[A-Z]90:*:*:*) echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*TS:*:*:*) echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*T3E:*:*:*) echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; CRAY*SV1:*:*:*) echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + exit ;; *:UNICOS/mp:*:*) - echo nv1-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; + echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' + exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` + echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; + FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" + exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; sparc*:BSD/OS:*:*) echo sparc-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; *:BSD/OS:*:*) echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; + exit ;; *:FreeBSD:*:*) - # Determine whether the default compiler uses glibc. - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #if __GLIBC__ >= 2 - LIBC=gnu - #else - LIBC= - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - # GNU/KFreeBSD systems have a "k" prefix to indicate we are using - # FreeBSD's kernel, but not the complete OS. - case ${LIBC} in gnu) kernel_only='k' ;; esac - echo ${UNAME_MACHINE}-unknown-${kernel_only}freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`${LIBC:+-$LIBC} - exit 0 ;; + UNAME_PROCESSOR=`/usr/bin/uname -p` + case ${UNAME_PROCESSOR} in + amd64) + echo x86_64-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + *) + echo ${UNAME_PROCESSOR}-unknown-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + esac + exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin - exit 0 ;; - i*:MINGW*:*) + exit ;; + *:MINGW*:*) echo ${UNAME_MACHINE}-pc-mingw32 - exit 0 ;; + exit ;; + i*:MSYS*:*) + echo ${UNAME_MACHINE}-pc-msys + exit ;; + i*:windows32*:*) + # uname -m includes "-pc" on this system. + echo ${UNAME_MACHINE}-mingw32 + exit ;; i*:PW*:*) echo ${UNAME_MACHINE}-pc-pw32 - exit 0 ;; - x86:Interix*:[34]*) - echo i586-pc-interix${UNAME_RELEASE}|sed -e 's/\..*//' - exit 0 ;; + exit ;; + *:Interix*:*) + case ${UNAME_MACHINE} in + x86) + echo i586-pc-interix${UNAME_RELEASE} + exit ;; + authenticamd | genuineintel | EM64T) + echo x86_64-unknown-interix${UNAME_RELEASE} + exit ;; + IA64) + echo ia64-unknown-interix${UNAME_RELEASE} + exit ;; + esac ;; [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) echo i${UNAME_MACHINE}-pc-mks - exit 0 ;; + exit ;; + 8664:Windows_NT:*) + echo x86_64-pc-mks + exit ;; i*:Windows_NT*:* | Pentium*:Windows_NT*:*) # How do we know it's Interix rather than the generic POSIX subsystem? # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we # UNAME_MACHINE based on the output of uname instead of i386? echo i586-pc-interix - exit 0 ;; + exit ;; i*:UWIN*:*) echo ${UNAME_MACHINE}-pc-uwin - exit 0 ;; + exit ;; + amd64:CYGWIN*:*:* | x86_64:CYGWIN*:*:*) + echo x86_64-unknown-cygwin + exit ;; p*:CYGWIN*:*) echo powerpcle-unknown-cygwin - exit 0 ;; + exit ;; prep*:SunOS:5.*:*) echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; + exit ;; *:GNU:*:*) # the GNU system echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` - exit 0 ;; + exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu - exit 0 ;; + exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix - exit 0 ;; + exit ;; + aarch64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + aarch64_be:Linux:*:*) + UNAME_MACHINE=aarch64_be + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + alpha:Linux:*:*) + case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in + EV5) UNAME_MACHINE=alphaev5 ;; + EV56) UNAME_MACHINE=alphaev56 ;; + PCA56) UNAME_MACHINE=alphapca56 ;; + PCA57) UNAME_MACHINE=alphapca56 ;; + EV6) UNAME_MACHINE=alphaev6 ;; + EV67) UNAME_MACHINE=alphaev67 ;; + EV68*) UNAME_MACHINE=alphaev68 ;; + esac + objdump --private-headers /bin/sh | grep -q ld.so.1 + if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi + echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} + exit ;; arm*:Linux:*:*) + eval $set_cc_for_build + if echo __ARM_EABI__ | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_EABI__ + then + echo ${UNAME_MACHINE}-unknown-linux-gnu + else + if echo __ARM_PCS_VFP | $CC_FOR_BUILD -E - 2>/dev/null \ + | grep -q __ARM_PCS_VFP + then + echo ${UNAME_MACHINE}-unknown-linux-gnueabi + else + echo ${UNAME_MACHINE}-unknown-linux-gnueabihf + fi + fi + exit ;; + avr32*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + exit ;; cris:Linux:*:*) - echo cris-axis-linux-gnu - exit 0 ;; - ia64:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-gnu + exit ;; + crisv32:Linux:*:*) + echo ${UNAME_MACHINE}-axis-linux-gnu + exit ;; + frv:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - m68*:Linux:*:*) + exit ;; + hexagon:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - mips:Linux:*:*) + exit ;; + i*86:Linux:*:*) + LIBC=gnu eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c - #undef CPU - #undef mips - #undef mipsel - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mipsel - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips - #else - CPU= - #endif + #ifdef __dietlibc__ + LIBC=dietlibc #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 - ;; - mips64:Linux:*:*) + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^LIBC'` + echo "${UNAME_MACHINE}-pc-linux-${LIBC}" + exit ;; + ia64:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + m32r*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + m68*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + mips:Linux:*:* | mips64:Linux:*:*) eval $set_cc_for_build sed 's/^ //' << EOF >$dummy.c #undef CPU - #undef mips64 - #undef mips64el + #undef ${UNAME_MACHINE} + #undef ${UNAME_MACHINE}el #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mips64el + CPU=${UNAME_MACHINE}el #else #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips64 + CPU=${UNAME_MACHINE} #else CPU= #endif #endif EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 + eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` + test x"${CPU}" != x && { echo "${CPU}-unknown-linux-gnu"; exit; } ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu - exit 0 ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu - exit 0 ;; - alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} - exit 0 ;; + or32:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + padre:Linux:*:*) + echo sparc-unknown-linux-gnu + exit ;; + parisc64:Linux:*:* | hppa64:Linux:*:*) + echo hppa64-unknown-linux-gnu + exit ;; parisc:Linux:*:* | hppa:Linux:*:*) # Look for CPU level case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in @@ -921,115 +968,71 @@ EOF PA8*) echo hppa2.0-unknown-linux-gnu ;; *) echo hppa-unknown-linux-gnu ;; esac - exit 0 ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu - exit 0 ;; + exit ;; + ppc64:Linux:*:*) + echo powerpc64-unknown-linux-gnu + exit ;; + ppc:Linux:*:*) + echo powerpc-unknown-linux-gnu + exit ;; s390:Linux:*:* | s390x:Linux:*:*) echo ${UNAME_MACHINE}-ibm-linux - exit 0 ;; + exit ;; sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; sh*:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + exit ;; sparc:Linux:*:* | sparc64:Linux:*:*) echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; + exit ;; + tile*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + vax:Linux:*:*) + echo ${UNAME_MACHINE}-dec-linux-gnu + exit ;; x86_64:Linux:*:*) - echo x86_64-unknown-linux-gnu - exit 0 ;; - i*86:Linux:*:*) - # The BFD linker knows what the default object file format is, so - # first see if it will tell us. cd to the root directory to prevent - # problems with other programs or directories called `ld' in the path. - # Set LC_ALL=C to ensure ld outputs messages in English. - ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ - | sed -ne '/supported targets:/!d - s/[ ][ ]*/ /g - s/.*supported targets: *// - s/ .*// - p'` - case "$ld_supported_targets" in - elf32-i386) - TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu" - ;; - a.out-i386-linux) - echo "${UNAME_MACHINE}-pc-linux-gnuaout" - exit 0 ;; - coff-i386) - echo "${UNAME_MACHINE}-pc-linux-gnucoff" - exit 0 ;; - "") - # Either a pre-BFD a.out linker (linux-gnuoldld) or - # one that does not give us useful --help. - echo "${UNAME_MACHINE}-pc-linux-gnuoldld" - exit 0 ;; - esac - # Determine whether the default compiler is a.out or elf - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #ifdef __ELF__ - # ifdef __GLIBC__ - # if __GLIBC__ >= 2 - LIBC=gnu - # else - LIBC=gnulibc1 - # endif - # else - LIBC=gnulibc1 - # endif - #else - #ifdef __INTEL_COMPILER - LIBC=gnu - #else - LIBC=gnuaout - #endif - #endif - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - test x"${LIBC}" != x && echo "${UNAME_MACHINE}-pc-linux-${LIBC}" && exit 0 - test x"${TENTATIVE}" != x && echo "${TENTATIVE}" && exit 0 - ;; + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; + xtensa*:Linux:*:*) + echo ${UNAME_MACHINE}-unknown-linux-gnu + exit ;; i*86:DYNIX/ptx:4*:*) # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. # earlier versions are messed up and put the nodename in both # sysname and nodename. echo i386-sequent-sysv4 - exit 0 ;; + exit ;; i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, + # Unixware is an offshoot of SVR4, but it has its own version + # number series starting with 2... + # I am not positive that other SVR4 systems won't match this, # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. + # Use sysv4.2uw... so that sysv4* matches it. echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} - exit 0 ;; + exit ;; i*86:OS/2:*:*) # If we were able to find `uname', then EMX Unix compatibility # is probably installed. echo ${UNAME_MACHINE}-pc-os2-emx - exit 0 ;; + exit ;; i*86:XTS-300:*:STOP) echo ${UNAME_MACHINE}-unknown-stop - exit 0 ;; + exit ;; i*86:atheos:*:*) echo ${UNAME_MACHINE}-unknown-atheos - exit 0 ;; - i*86:syllable:*:*) + exit ;; + i*86:syllable:*:*) echo ${UNAME_MACHINE}-pc-syllable - exit 0 ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) + exit ;; + i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.[02]*:*) echo i386-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; i*86:*DOS:*:*) echo ${UNAME_MACHINE}-pc-msdosdjgpp - exit 0 ;; + exit ;; i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then @@ -1037,15 +1040,16 @@ EOF else echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} fi - exit 0 ;; - i*86:*:5:[78]*) + exit ;; + i*86:*:5:[678]*) + # UnixWare 7.x, OpenUNIX and OpenServer 6. case `/bin/uname -X | grep "^Machine"` in *486*) UNAME_MACHINE=i486 ;; *Pentium) UNAME_MACHINE=i586 ;; *Pent*|*Celeron) UNAME_MACHINE=i686 ;; esac echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} - exit 0 ;; + exit ;; i*86:*:3.2:*) if test -f /usr/options/cb.name; then UNAME_REL=`sed -n 's/.*Version //p' /dev/null 2>&1 ; then echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 else # Add other i860-SVR4 vendors below as they are discovered. echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 fi - exit 0 ;; + exit ;; mini*:CTIX:SYS*5:*) # "miniframe" echo m68010-convergent-sysv - exit 0 ;; + exit ;; mc68k:UNIX:SYSTEM5:3.51m) echo m68k-convergent-sysv - exit 0 ;; + exit ;; M680?0:D-NIX:5.3:*) echo m68k-diab-dnix - exit 0 ;; - M68*:*:R3V[567]*:*) - test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0) + exit ;; + M68*:*:R3V[5678]*:*) + test -r /sysV68 && { echo 'm68k-motorola-sysv'; exit; } ;; + 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0 | S7501*:*:4.0:3.0) OS_REL='' test -r /etc/.relid \ && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4.3${OS_REL} && exit 0 + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && echo i586-ncr-sysv4.3${OS_REL} && exit 0 ;; + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4 && exit 0 ;; + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4; exit; } ;; + NCR*:*:4.2:* | MPRAS*:*:4.2:*) + OS_REL='.3' + test -r /etc/.relid \ + && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` + /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ + && { echo i486-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } + /bin/uname -p 2>/dev/null | /bin/grep pteron >/dev/null \ + && { echo i586-ncr-sysv4.3${OS_REL}; exit; } ;; m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) echo m68k-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; mc68030:UNIX_System_V:4.*:*) echo m68k-atari-sysv4 - exit 0 ;; + exit ;; TSUNAMI:LynxOS:2.*:*) echo sparc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; rs6000:LynxOS:2.*:*) echo rs6000-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) + exit ;; + PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.[02]*:*) echo powerpc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; + exit ;; SM[BE]S:UNIX_SV:*:*) echo mips-dde-sysv${UNAME_RELEASE} - exit 0 ;; + exit ;; RM*:ReliantUNIX-*:*:*) echo mips-sni-sysv4 - exit 0 ;; + exit ;; RM*:SINIX-*:*:*) echo mips-sni-sysv4 - exit 0 ;; + exit ;; *:SINIX-*:*:*) if uname -p 2>/dev/null >/dev/null ; then UNAME_MACHINE=`(uname -p) 2>/dev/null` @@ -1137,68 +1154,94 @@ EOF else echo ns32k-sni-sysv fi - exit 0 ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - echo i586-unisys-sysv4 - exit 0 ;; + exit ;; + PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort + # says + echo i586-unisys-sysv4 + exit ;; *:UNIX_System_V:4*:FTX*) # From Gerald Hewes . # How about differentiating between stratus architectures? -djm echo hppa1.1-stratus-sysv4 - exit 0 ;; + exit ;; *:*:*:FTX*) # From seanf@swdc.stratus.com. echo i860-stratus-sysv4 - exit 0 ;; + exit ;; + i*86:VOS:*:*) + # From Paul.Green@stratus.com. + echo ${UNAME_MACHINE}-stratus-vos + exit ;; *:VOS:*:*) # From Paul.Green@stratus.com. echo hppa1.1-stratus-vos - exit 0 ;; + exit ;; mc68*:A/UX:*:*) echo m68k-apple-aux${UNAME_RELEASE} - exit 0 ;; + exit ;; news*:NEWS-OS:6*:*) echo mips-sony-newsos6 - exit 0 ;; + exit ;; R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} + echo mips-nec-sysv${UNAME_RELEASE} else - echo mips-unknown-sysv${UNAME_RELEASE} + echo mips-unknown-sysv${UNAME_RELEASE} fi - exit 0 ;; + exit ;; BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. echo powerpc-be-beos - exit 0 ;; + exit ;; BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. echo powerpc-apple-beos - exit 0 ;; + exit ;; BePC:BeOS:*:*) # BeOS running on Intel PC compatible. echo i586-pc-beos - exit 0 ;; + exit ;; + BePC:Haiku:*:*) # Haiku running on Intel PC compatible. + echo i586-pc-haiku + exit ;; SX-4:SUPER-UX:*:*) echo sx4-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; SX-5:SUPER-UX:*:*) echo sx5-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; SX-6:SUPER-UX:*:*) echo sx6-nec-superux${UNAME_RELEASE} - exit 0 ;; + exit ;; + SX-7:SUPER-UX:*:*) + echo sx7-nec-superux${UNAME_RELEASE} + exit ;; + SX-8:SUPER-UX:*:*) + echo sx8-nec-superux${UNAME_RELEASE} + exit ;; + SX-8R:SUPER-UX:*:*) + echo sx8r-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Rhapsody:*:*) echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Darwin:*:*) - case `uname -p` in - *86) UNAME_PROCESSOR=i686 ;; - powerpc) UNAME_PROCESSOR=powerpc ;; + UNAME_PROCESSOR=`uname -p` || UNAME_PROCESSOR=unknown + case $UNAME_PROCESSOR in + i386) + eval $set_cc_for_build + if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ + (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null + then + UNAME_PROCESSOR="x86_64" + fi + fi ;; + unknown) UNAME_PROCESSOR=powerpc ;; esac echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} - exit 0 ;; + exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` if test "$UNAME_PROCESSOR" = "x86"; then @@ -1206,22 +1249,28 @@ EOF UNAME_MACHINE=pc fi echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} - exit 0 ;; + exit ;; *:QNX:*:4*) echo i386-pc-qnx - exit 0 ;; + exit ;; + NEO-?:NONSTOP_KERNEL:*:*) + echo neo-tandem-nsk${UNAME_RELEASE} + exit ;; + NSE-?:NONSTOP_KERNEL:*:*) + echo nse-tandem-nsk${UNAME_RELEASE} + exit ;; NSR-?:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} - exit 0 ;; + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux - exit 0 ;; + exit ;; BS2000:POSIX*:*:*) echo bs2000-siemens-sysv - exit 0 ;; + exit ;; DS/*:UNIX_System_V:*:*) echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} - exit 0 ;; + exit ;; *:Plan9:*:*) # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 @@ -1232,31 +1281,53 @@ EOF UNAME_MACHINE="$cputype" fi echo ${UNAME_MACHINE}-unknown-plan9 - exit 0 ;; + exit ;; *:TOPS-10:*:*) echo pdp10-unknown-tops10 - exit 0 ;; + exit ;; *:TENEX:*:*) echo pdp10-unknown-tenex - exit 0 ;; + exit ;; KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) echo pdp10-dec-tops20 - exit 0 ;; + exit ;; XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) echo pdp10-xkl-tops20 - exit 0 ;; + exit ;; *:TOPS-20:*:*) echo pdp10-unknown-tops20 - exit 0 ;; + exit ;; *:ITS:*:*) echo pdp10-unknown-its - exit 0 ;; + exit ;; SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} - exit 0 ;; + echo mips-sei-seiux${UNAME_RELEASE} + exit ;; *:DragonFly:*:*) echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` - exit 0 ;; + exit ;; + *:*VMS:*:*) + UNAME_MACHINE=`(uname -p) 2>/dev/null` + case "${UNAME_MACHINE}" in + A*) echo alpha-dec-vms ; exit ;; + I*) echo ia64-dec-vms ; exit ;; + V*) echo vax-dec-vms ; exit ;; + esac ;; + *:XENIX:*:SysV) + echo i386-pc-xenix + exit ;; + i*86:skyos:*:*) + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + exit ;; + i*86:rdos:*:*) + echo ${UNAME_MACHINE}-pc-rdos + exit ;; + i*86:AROS:*:*) + echo ${UNAME_MACHINE}-pc-aros + exit ;; + x86_64:VMkernel:*:*) + echo ${UNAME_MACHINE}-unknown-esx + exit ;; esac #echo '(No uname command or uname output not recognized.)' 1>&2 @@ -1279,16 +1350,16 @@ main () #include printf ("m68k-sony-newsos%s\n", #ifdef NEWSOS4 - "4" + "4" #else - "" + "" #endif - ); exit (0); + ); exit (0); #endif #endif #if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix"); exit (0); + printf ("arm-acorn-riscix\n"); exit (0); #endif #if defined (hp300) && !defined (hpux) @@ -1377,11 +1448,12 @@ main () } EOF -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && $dummy && exit 0 +$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && SYSTEM_NAME=`$dummy` && + { echo "$SYSTEM_NAME"; exit; } # Apollos put the system type in the environment. -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; } +test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit; } # Convex versions that predate uname can use getsysinfo(1) @@ -1390,22 +1462,22 @@ then case `getsysinfo -f cpu_type` in c1*) echo c1-convex-bsd - exit 0 ;; + exit ;; c2*) if getsysinfo -f scalar_acc then echo c32-convex-bsd else echo c2-convex-bsd fi - exit 0 ;; + exit ;; c34*) echo c34-convex-bsd - exit 0 ;; + exit ;; c38*) echo c38-convex-bsd - exit 0 ;; + exit ;; c4*) echo c4-convex-bsd - exit 0 ;; + exit ;; esac fi @@ -1416,7 +1488,9 @@ This script, last modified $timestamp, has failed to recognize the operating system you are using. It is advised that you download the most up to date version of the config scripts from - ftp://ftp.gnu.org/pub/gnu/config/ + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +and + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD If the version you run ($0) is already up to date, please send the following data and any information you think might be diff --git a/deps/jemalloc/config.sub b/deps/jemalloc/config.sub index 264f820aa55..c894da45500 100755 --- a/deps/jemalloc/config.sub +++ b/deps/jemalloc/config.sub @@ -1,9 +1,10 @@ #! /bin/sh # Configuration validation subroutine script. # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. +# 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +# 2011, 2012 Free Software Foundation, Inc. -timestamp='2004-02-23' +timestamp='2012-02-10' # This file is (in principle) common to ALL GNU software. # The presence of a machine in this file suggests that SOME GNU software @@ -20,23 +21,25 @@ timestamp='2004-02-23' # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - +# along with this program; if not, see . +# # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. + # Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. +# diff and a properly formatted GNU ChangeLog entry. # # Configuration subroutine to validate and canonicalize a configuration type. # Supply the specified configuration type as an argument. # If it is invalid, we print an error message on stderr and exit with code 1. # Otherwise, we print the canonical config type on stdout and succeed. +# You can get the latest version of this script from: +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases # that are meaningful with *any* GNU software. @@ -70,7 +73,8 @@ Report bugs and patches to ." version="\ GNU config.sub ($timestamp) -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 +Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, +2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO @@ -83,11 +87,11 @@ Try \`$me --help' for more information." while test $# -gt 0 ; do case $1 in --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; + echo "$timestamp" ; exit ;; --version | -v ) - echo "$version" ; exit 0 ;; + echo "$version" ; exit ;; --help | --h* | -h ) - echo "$usage"; exit 0 ;; + echo "$usage"; exit ;; -- ) # Stop option processing shift; break ;; - ) # Use stdin as input. @@ -99,7 +103,7 @@ while test $# -gt 0 ; do *local*) # First pass through any local machine types. echo $1 - exit 0;; + exit ;; * ) break ;; @@ -118,11 +122,18 @@ esac # Here we must recognize all the valid KERNEL-OS combinations. maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in - nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \ - kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*) + nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ + linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | \ + kopensolaris*-gnu* | \ + storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` ;; + android-linux) + os=-linux-android + basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'`-unknown + ;; *) basic_machine=`echo $1 | sed 's/-[^-]*$//'` if [ $basic_machine != $1 ] @@ -145,10 +156,13 @@ case $os in -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis) + -apple | -axis | -knuth | -cray | -microblaze) os= basic_machine=$1 ;; + -bluegene*) + os=-cnk + ;; -sim | -cisco | -oki | -wec | -winbond) os= basic_machine=$1 @@ -163,13 +177,17 @@ case $os in os=-chorusos basic_machine=$1 ;; - -chorusrdb) - os=-chorusrdb + -chorusrdb) + os=-chorusrdb basic_machine=$1 - ;; + ;; -hiux*) os=-hiuxwe2 ;; + -sco6) + os=-sco5v6 + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; -sco5) os=-sco3.2v5 basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` @@ -186,6 +204,10 @@ case $os in # Don't forget version if it is 3.2v4 or newer. basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` ;; + -sco5v6*) + # Don't forget version if it is 3.2v4 or newer. + basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` + ;; -sco*) os=-sco3.2v2 basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` @@ -227,25 +249,36 @@ case $basic_machine in # Some are omitted here because they have special meanings below. 1750a | 580 \ | a29k \ + | aarch64 | aarch64_be \ | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \ + | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr | avr32 \ + | be32 | be64 \ + | bfin \ | c4x | clipper \ | d10v | d30v | dlx | dsp16xx \ - | fr30 | frv \ + | epiphany \ + | fido | fr30 | frv \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ + | hexagon \ | i370 | i860 | i960 | ia64 \ | ip2k | iq2000 \ - | m32r | m68000 | m68k | m88k | mcore \ + | le32 | le64 \ + | lm32 \ + | m32c | m32r | m32rle | m68000 | m68k | m88k \ + | maxq | mb | microblaze | mcore | mep | metag \ | mips | mipsbe | mipseb | mipsel | mipsle \ | mips16 \ | mips64 | mips64el \ - | mips64vr | mips64vrel \ + | mips64octeon | mips64octeonel \ | mips64orion | mips64orionel \ + | mips64r5900 | mips64r5900el \ + | mips64vr | mips64vrel \ | mips64vr4100 | mips64vr4100el \ | mips64vr4300 | mips64vr4300el \ | mips64vr5000 | mips64vr5000el \ + | mips64vr5900 | mips64vr5900el \ | mipsisa32 | mipsisa32el \ | mipsisa32r2 | mipsisa32r2el \ | mipsisa64 | mipsisa64el \ @@ -254,30 +287,65 @@ case $basic_machine in | mipsisa64sr71k | mipsisa64sr71kel \ | mipstx39 | mipstx39el \ | mn10200 | mn10300 \ + | moxie \ + | mt \ | msp430 \ + | nds32 | nds32le | nds32be \ + | nios | nios2 \ | ns16k | ns32k \ - | openrisc | or32 \ + | open8 \ + | or32 \ | pdp10 | pdp11 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \ + | powerpc | powerpc64 | powerpc64le | powerpcle \ | pyramid \ - | sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \ + | rl78 | rx \ + | score \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ - | sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \ - | strongarm \ - | tahoe | thumb | tic4x | tic80 | tron \ - | v850 | v850e \ + | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ + | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ + | spu \ + | tahoe | tic4x | tic54x | tic55x | tic6x | tic80 | tron \ + | ubicom32 \ + | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | we32k \ - | x86 | xscale | xstormy16 | xtensa \ - | z8k) + | x86 | xc16x | xstormy16 | xtensa \ + | z8k | z80) basic_machine=$basic_machine-unknown ;; - m6811 | m68hc11 | m6812 | m68hc12) - # Motorola 68HC11/12. + c54x) + basic_machine=tic54x-unknown + ;; + c55x) + basic_machine=tic55x-unknown + ;; + c6x) + basic_machine=tic6x-unknown + ;; + m6811 | m68hc11 | m6812 | m68hc12 | m68hcs12x | picochip) basic_machine=$basic_machine-unknown os=-none ;; m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) ;; + ms1) + basic_machine=mt-unknown + ;; + + strongarm | thumb | xscale) + basic_machine=arm-unknown + ;; + xgate) + basic_machine=$basic_machine-unknown + os=-none + ;; + xscaleeb) + basic_machine=armeb-unknown + ;; + + xscaleel) + basic_machine=armel-unknown + ;; # We use `pc' rather than `unknown' # because (1) that's what they normally are, and @@ -293,32 +361,40 @@ case $basic_machine in # Recognize the basic CPU types with company name. 580-* \ | a29k-* \ + | aarch64-* | aarch64_be-* \ | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* \ - | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \ - | clipper-* | cydra-* \ + | avr-* | avr32-* \ + | be32-* | be64-* \ + | bfin-* | bs2000-* \ + | c[123]* | c30-* | [cjt]90-* | c4x-* \ + | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ | elxsi-* \ - | f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \ + | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ + | hexagon-* \ | i*86-* | i860-* | i960-* | ia64-* \ | ip2k-* | iq2000-* \ - | m32r-* \ + | le32-* | le64-* \ + | lm32-* \ + | m32c-* | m32r-* | m32rle-* \ | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | mcore-* \ + | m88110-* | m88k-* | maxq-* | mcore-* | metag-* | microblaze-* \ | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ | mips16-* \ | mips64-* | mips64el-* \ - | mips64vr-* | mips64vrel-* \ + | mips64octeon-* | mips64octeonel-* \ | mips64orion-* | mips64orionel-* \ + | mips64r5900-* | mips64r5900el-* \ + | mips64vr-* | mips64vrel-* \ | mips64vr4100-* | mips64vr4100el-* \ | mips64vr4300-* | mips64vr4300el-* \ | mips64vr5000-* | mips64vr5000el-* \ + | mips64vr5900-* | mips64vr5900el-* \ | mipsisa32-* | mipsisa32el-* \ | mipsisa32r2-* | mipsisa32r2el-* \ | mipsisa64-* | mipsisa64el-* \ @@ -326,26 +402,39 @@ case $basic_machine in | mipsisa64sb1-* | mipsisa64sb1el-* \ | mipsisa64sr71k-* | mipsisa64sr71kel-* \ | mipstx39-* | mipstx39el-* \ + | mmix-* \ + | mt-* \ | msp430-* \ - | none-* | np1-* | nv1-* | ns16k-* | ns32k-* \ + | nds32-* | nds32le-* | nds32be-* \ + | nios-* | nios2-* \ + | none-* | np1-* | ns16k-* | ns32k-* \ + | open8-* \ | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \ + | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ | pyramid-* \ - | romp-* | rs6000-* \ - | sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \ + | rl78-* | romp-* | rs6000-* | rx-* \ + | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \ - | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \ - | tahoe-* | thumb-* \ + | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ + | sparclite-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ + | tile*-* \ | tron-* \ - | v850-* | v850e-* | vax-* \ + | ubicom32-* \ + | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ + | vax-* \ | we32k-* \ - | x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \ - | xtensa-* \ + | x86-* | x86_64-* | xc16x-* | xps100-* \ + | xstormy16-* | xtensa*-* \ | ymp-* \ - | z8k-*) + | z8k-* | z80-*) + ;; + # Recognize the basic CPU types without company name, with glob match. + xtensa*) + basic_machine=$basic_machine-unknown ;; # Recognize the various machine names and aliases which stand # for a CPU type and a company and sometimes even an OS. @@ -363,7 +452,7 @@ case $basic_machine in basic_machine=a29k-amd os=-udi ;; - abacus) + abacus) basic_machine=abacus-unknown ;; adobe68k) @@ -409,6 +498,10 @@ case $basic_machine in basic_machine=m68k-apollo os=-bsd ;; + aros) + basic_machine=i386-pc + os=-aros + ;; aux) basic_machine=m68k-apple os=-aux @@ -417,10 +510,35 @@ case $basic_machine in basic_machine=ns32k-sequent os=-dynix ;; + blackfin) + basic_machine=bfin-unknown + os=-linux + ;; + blackfin-*) + basic_machine=bfin-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; + bluegene*) + basic_machine=powerpc-ibm + os=-cnk + ;; + c54x-*) + basic_machine=tic54x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c55x-*) + basic_machine=tic55x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; + c6x-*) + basic_machine=tic6x-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; c90) basic_machine=c90-cray os=-unicos ;; + cegcc) + basic_machine=arm-unknown + os=-cegcc + ;; convex-c1) basic_machine=c1-convex os=-bsd @@ -445,13 +563,20 @@ case $basic_machine in basic_machine=j90-cray os=-unicos ;; - cr16c) - basic_machine=cr16c-unknown + craynv) + basic_machine=craynv-cray + os=-unicosmp + ;; + cr16 | cr16-*) + basic_machine=cr16-unknown os=-elf ;; crds | unos) basic_machine=m68k-crds ;; + crisv32 | crisv32-* | etraxfs*) + basic_machine=crisv32-axis + ;; cris | cris-* | etrax*) basic_machine=cris-axis ;; @@ -481,6 +606,14 @@ case $basic_machine in basic_machine=m88k-motorola os=-sysv3 ;; + dicos) + basic_machine=i686-pc + os=-dicos + ;; + djgpp) + basic_machine=i586-pc + os=-msdosdjgpp + ;; dpx20 | dpx20-*) basic_machine=rs6000-bull os=-bosx @@ -592,7 +725,6 @@ case $basic_machine in i370-ibm* | ibm*) basic_machine=i370-ibm ;; -# I'm not sure what "Sysv32" means. Should this be sysv3.2? i*86v32) basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` os=-sysv32 @@ -631,6 +763,14 @@ case $basic_machine in basic_machine=m68k-isi os=-sysv ;; + m68knommu) + basic_machine=m68k-unknown + os=-linux + ;; + m68knommu-*) + basic_machine=m68k-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; m88k-omron*) basic_machine=m88k-omron ;; @@ -642,10 +782,17 @@ case $basic_machine in basic_machine=ns32k-utek os=-sysv ;; + microblaze) + basic_machine=microblaze-xilinx + ;; mingw32) basic_machine=i386-pc os=-mingw32 ;; + mingw32ce) + basic_machine=arm-unknown + os=-mingw32ce + ;; miniframe) basic_machine=m68000-convergent ;; @@ -659,10 +806,6 @@ case $basic_machine in mips3*) basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown ;; - mmix*) - basic_machine=mmix-knuth - os=-mmixware - ;; monitor) basic_machine=m68k-rom68k os=-coff @@ -675,10 +818,21 @@ case $basic_machine in basic_machine=i386-pc os=-msdos ;; + ms1-*) + basic_machine=`echo $basic_machine | sed -e 's/ms1-/mt-/'` + ;; + msys) + basic_machine=i386-pc + os=-msys + ;; mvs) basic_machine=i370-ibm os=-mvs ;; + nacl) + basic_machine=le32-unknown + os=-nacl + ;; ncr3000) basic_machine=i486-ncr os=-sysv4 @@ -743,9 +897,11 @@ case $basic_machine in np1) basic_machine=np1-gould ;; - nv1) - basic_machine=nv1-cray - os=-unicosmp + neo-tandem) + basic_machine=neo-tandem + ;; + nse-tandem) + basic_machine=nse-tandem ;; nsr-tandem) basic_machine=nsr-tandem @@ -754,9 +910,8 @@ case $basic_machine in basic_machine=hppa1.1-oki os=-proelf ;; - or32 | or32-*) + openrisc | openrisc-*) basic_machine=or32-unknown - os=-coff ;; os400) basic_machine=powerpc-ibm @@ -778,6 +933,14 @@ case $basic_machine in basic_machine=i860-intel os=-osf ;; + parisc) + basic_machine=hppa-unknown + os=-linux + ;; + parisc-*) + basic_machine=hppa-`echo $basic_machine | sed 's/^[^-]*-//'` + os=-linux + ;; pbd) basic_machine=sparc-tti ;; @@ -787,6 +950,12 @@ case $basic_machine in pc532 | pc532-*) basic_machine=ns32k-pc532 ;; + pc98) + basic_machine=i386-pc + ;; + pc98-*) + basic_machine=i386-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; pentium | p5 | k5 | k6 | nexgen | viac3) basic_machine=i586-pc ;; @@ -816,9 +985,10 @@ case $basic_machine in ;; power) basic_machine=power-ibm ;; - ppc) basic_machine=powerpc-unknown + ppc | ppcbe) basic_machine=powerpc-unknown ;; - ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + ppc-* | ppcbe-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` ;; ppcle | powerpclittle | ppc-le | powerpc-little) basic_machine=powerpcle-unknown @@ -843,6 +1013,10 @@ case $basic_machine in basic_machine=i586-unknown os=-pw32 ;; + rdos) + basic_machine=i386-pc + os=-rdos + ;; rom68k) basic_machine=m68k-rom68k os=-coff @@ -869,6 +1043,10 @@ case $basic_machine in sb1el) basic_machine=mipsisa64sb1el-unknown ;; + sde) + basic_machine=mipsisa32-sde + os=-elf + ;; sei) basic_machine=mips-sei os=-seiux @@ -880,6 +1058,9 @@ case $basic_machine in basic_machine=sh-hitachi os=-hms ;; + sh5el) + basic_machine=sh5le-unknown + ;; sh64) basic_machine=sh64-unknown ;; @@ -901,6 +1082,9 @@ case $basic_machine in basic_machine=i860-stratus os=-sysv4 ;; + strongarm-* | thumb-*) + basic_machine=arm-`echo $basic_machine | sed 's/^[^-]*-//'` + ;; sun2) basic_machine=m68000-sun ;; @@ -957,17 +1141,9 @@ case $basic_machine in basic_machine=t90-cray os=-unicos ;; - tic54x | c54x*) - basic_machine=tic54x-unknown - os=-coff - ;; - tic55x | c55x*) - basic_machine=tic55x-unknown - os=-coff - ;; - tic6x | c6x*) - basic_machine=tic6x-unknown - os=-coff + tile*) + basic_machine=$basic_machine-unknown + os=-linux-gnu ;; tx39) basic_machine=mipstx39-unknown @@ -1029,9 +1205,16 @@ case $basic_machine in basic_machine=hppa1.1-winbond os=-proelf ;; + xbox) + basic_machine=i686-pc + os=-mingw32 + ;; xps | xps100) basic_machine=xps100-honeywell ;; + xscale-* | xscalee[bl]-*) + basic_machine=`echo $basic_machine | sed 's/^xscale/arm/'` + ;; ymp) basic_machine=ymp-cray os=-unicos @@ -1040,6 +1223,10 @@ case $basic_machine in basic_machine=z8k-unknown os=-sim ;; + z80-*-coff) + basic_machine=z80-unknown + os=-sim + ;; none) basic_machine=none-none os=-none @@ -1059,6 +1246,9 @@ case $basic_machine in romp) basic_machine=romp-ibm ;; + mmix) + basic_machine=mmix-knuth + ;; rs6000) basic_machine=rs6000-ibm ;; @@ -1075,13 +1265,10 @@ case $basic_machine in we32k) basic_machine=we32k-att ;; - sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele) + sh[1234] | sh[24]a | sh[24]aeb | sh[34]eb | sh[1234]le | sh[23]ele) basic_machine=sh-unknown ;; - sh64) - basic_machine=sh64-unknown - ;; - sparc | sparcv9 | sparcv9b) + sparc | sparcv8 | sparcv9 | sparcv9b | sparcv9v) basic_machine=sparc-sun ;; cydra) @@ -1125,9 +1312,12 @@ esac if [ x"$os" != x"" ] then case $os in - # First match some system type aliases - # that might get confused with valid system types. + # First match some system type aliases + # that might get confused with valid system types. # -solaris* is a basic system type, with this one exception. + -auroraux) + os=-auroraux + ;; -solaris1 | -solaris1.*) os=`echo $os | sed -e 's|solaris1|sunos4|'` ;; @@ -1148,26 +1338,31 @@ case $os in # Each alternative MUST END IN A *, to match a version number. # -sysv* is not here because it comes later, after sysvr4. -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \ + | -*vms* | -sco* | -esix* | -isc* | -aix* | -cnk* | -sunos | -sunos[34]*\ + | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ + | -sym* | -kopensolaris* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* \ + | -aos* | -aros* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \ + | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ + | -openbsd* | -solidbsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* \ - | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \ + | -chorusos* | -chorusrdb* | -cegcc* \ + | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ + | -mingw32* | -linux-gnu* | -linux-android* \ + | -linux-newlib* | -linux-uclibc* \ + | -uxpv* | -beos* | -mpeix* | -udk* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*) + | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1185,7 +1380,7 @@ case $os in os=`echo $os | sed -e 's|nto|nto-qnx|'` ;; -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* \ + | -windows* | -osx | -abug | -netware* | -os9* | -beos* | -haiku* \ | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) ;; -mac*) @@ -1206,7 +1401,7 @@ case $os in -opened*) os=-openedition ;; - -os400*) + -os400*) os=-os400 ;; -wince*) @@ -1255,7 +1450,7 @@ case $os in -sinix*) os=-sysv4 ;; - -tpf*) + -tpf*) os=-tpf ;; -triton*) @@ -1294,6 +1489,14 @@ case $os in -kaos*) os=-kaos ;; + -zvmoe) + os=-zvmoe + ;; + -dicos*) + os=-dicos + ;; + -nacl*) + ;; -none) ;; *) @@ -1316,6 +1519,12 @@ else # system, and we'll never get to this point. case $basic_machine in + score-*) + os=-elf + ;; + spu-*) + os=-elf + ;; *-acorn) os=-riscix1.2 ;; @@ -1325,9 +1534,18 @@ case $basic_machine in arm*-semi) os=-aout ;; - c4x-* | tic4x-*) - os=-coff - ;; + c4x-* | tic4x-*) + os=-coff + ;; + tic54x-*) + os=-coff + ;; + tic55x-*) + os=-coff + ;; + tic6x-*) + os=-coff + ;; # This must come before the *-dec entry. pdp10-*) os=-tops20 @@ -1346,13 +1564,13 @@ case $basic_machine in ;; m68000-sun) os=-sunos3 - # This also exists in the configure program, but was not the - # default. - # os=-sunos4 ;; m68*-cisco) os=-aout ;; + mep-*) + os=-elf + ;; mips*-cisco) os=-elf ;; @@ -1371,9 +1589,15 @@ case $basic_machine in *-be) os=-beos ;; + *-haiku) + os=-haiku + ;; *-ibm) os=-aix ;; + *-knuth) + os=-mmixware + ;; *-wec) os=-proelf ;; @@ -1476,7 +1700,7 @@ case $basic_machine in -sunos*) vendor=sun ;; - -aix*) + -cnk*|-aix*) vendor=ibm ;; -beos*) @@ -1539,7 +1763,7 @@ case $basic_machine in esac echo $basic_machine$os -exit 0 +exit # Local variables: # eval: (add-hook 'write-file-hooks 'time-stamp) diff --git a/deps/jemalloc/configure b/deps/jemalloc/configure index 610884fb323..1d2b8beece0 100755 --- a/deps/jemalloc/configure +++ b/deps/jemalloc/configure @@ -1,11 +1,11 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.65. +# Generated by GNU Autoconf 2.68. # # # Copyright (C) 1992, 1993, 1994, 1995, 1996, 1998, 1999, 2000, 2001, -# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software Foundation, -# Inc. +# 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010 Free Software +# Foundation, Inc. # # # This configure script is free software; the Free Software Foundation @@ -89,6 +89,7 @@ fi IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. +as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -214,11 +215,18 @@ IFS=$as_save_IFS # We cannot yet assume a decent shell, so we have to provide a # neutralization value for shells without unset; and this also # works around shells that cannot unset nonexistent variables. + # Preserve -v and -x to the replacement shell. BASH_ENV=/dev/null ENV=/dev/null (unset BASH_ENV) >/dev/null 2>&1 && unset BASH_ENV ENV export CONFIG_SHELL - exec "$CONFIG_SHELL" "$as_myself" ${1+"$@"} + case $- in # (((( + *v*x* | *x*v* ) as_opts=-vx ;; + *v* ) as_opts=-v ;; + *x* ) as_opts=-x ;; + * ) as_opts= ;; + esac + exec "$CONFIG_SHELL" $as_opts "$as_myself" ${1+"$@"} fi if test x$as_have_required = xno; then : @@ -316,7 +324,7 @@ $as_echo X"$as_dir" | test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error "cannot create directory $as_dir" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p @@ -356,19 +364,19 @@ else fi # as_fn_arith -# as_fn_error ERROR [LINENO LOG_FD] -# --------------------------------- +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with status $?, using 1 if that was 0. +# script with STATUS, using 1 if that was 0. as_fn_error () { - as_status=$?; test $as_status -eq 0 && as_status=1 - if test "$3"; then - as_lineno=${as_lineno-"$2"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $1" >&$3 + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi - $as_echo "$as_me: error: $1" >&2 + $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error @@ -530,7 +538,7 @@ test -n "$DJDIR" || exec 7<&0 &1 # Name of the host. -# hostname on some systems (SVR3.2, Linux) returns a bogus exit status, +# hostname on some systems (SVR3.2, old GNU/Linux) returns a bogus exit status, # so uname gets run too. ac_hostname=`(hostname || uname -n) 2>/dev/null | sed 1q` @@ -605,18 +613,19 @@ jemalloc_version_bugfix jemalloc_version_minor jemalloc_version_major jemalloc_version -enable_dynamic_page_shift -enable_sysv enable_xmalloc +enable_valgrind +enable_utrace enable_fill enable_dss -enable_swap +enable_munmap +enable_mremap enable_tcache -enable_tiny enable_prof enable_stats enable_debug install_suffix +enable_experimental AUTOCONF LD AR @@ -626,6 +635,21 @@ INSTALL_SCRIPT INSTALL_PROGRAM enable_autogen RPATH_EXTRA +CC_MM +MKLIB +LDTARGET +CTARGET +PIC_CFLAGS +SOREV +EXTRA_LDFLAGS +DSO_LDFLAGS +libprefix +exe +a +o +importlib +so +LD_PRELOAD_VAR RPATH abi host_os @@ -658,6 +682,7 @@ abs_objroot objroot abs_srcroot srcroot +rev target_alias host_alias build_alias @@ -702,6 +727,8 @@ enable_option_checking with_xslroot with_rpath enable_autogen +enable_experimental +with_mangling with_jemalloc_prefix with_private_namespace with_install_suffix @@ -713,14 +740,14 @@ enable_prof_libunwind with_static_libunwind enable_prof_libgcc enable_prof_gcc -enable_tiny enable_tcache -enable_swap +enable_mremap +enable_munmap enable_dss enable_fill +enable_utrace +enable_valgrind enable_xmalloc -enable_sysv -enable_dynamic_page_shift enable_lazy_lock enable_tls ' @@ -795,8 +822,9 @@ do fi case $ac_option in - *=*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; - *) ac_optarg=yes ;; + *=?*) ac_optarg=`expr "X$ac_option" : '[^=]*=\(.*\)'` ;; + *=) ac_optarg= ;; + *) ac_optarg=yes ;; esac # Accept the important Cygnus configure options, so we can diagnose typos. @@ -841,7 +869,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*disable-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid feature name: $ac_useropt" + as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -867,7 +895,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*enable-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid feature name: $ac_useropt" + as_fn_error $? "invalid feature name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1071,7 +1099,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*with-\([^=]*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid package name: $ac_useropt" + as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1087,7 +1115,7 @@ do ac_useropt=`expr "x$ac_option" : 'x-*without-\(.*\)'` # Reject names that are not valid shell variable names. expr "x$ac_useropt" : ".*[^-+._$as_cr_alnum]" >/dev/null && - as_fn_error "invalid package name: $ac_useropt" + as_fn_error $? "invalid package name: $ac_useropt" ac_useropt_orig=$ac_useropt ac_useropt=`$as_echo "$ac_useropt" | sed 's/[-+.]/_/g'` case $ac_user_opts in @@ -1117,8 +1145,8 @@ do | --x-librar=* | --x-libra=* | --x-libr=* | --x-lib=* | --x-li=* | --x-l=*) x_libraries=$ac_optarg ;; - -*) as_fn_error "unrecognized option: \`$ac_option' -Try \`$0 --help' for more information." + -*) as_fn_error $? "unrecognized option: \`$ac_option' +Try \`$0 --help' for more information" ;; *=*) @@ -1126,7 +1154,7 @@ Try \`$0 --help' for more information." # Reject names that are not valid shell variable names. case $ac_envvar in #( '' | [0-9]* | *[!_$as_cr_alnum]* ) - as_fn_error "invalid variable name: \`$ac_envvar'" ;; + as_fn_error $? "invalid variable name: \`$ac_envvar'" ;; esac eval $ac_envvar=\$ac_optarg export $ac_envvar ;; @@ -1136,7 +1164,7 @@ Try \`$0 --help' for more information." $as_echo "$as_me: WARNING: you should use --build, --host, --target" >&2 expr "x$ac_option" : ".*[^-._$as_cr_alnum]" >/dev/null && $as_echo "$as_me: WARNING: invalid host type: $ac_option" >&2 - : ${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option} + : "${build_alias=$ac_option} ${host_alias=$ac_option} ${target_alias=$ac_option}" ;; esac @@ -1144,13 +1172,13 @@ done if test -n "$ac_prev"; then ac_option=--`echo $ac_prev | sed 's/_/-/g'` - as_fn_error "missing argument to $ac_option" + as_fn_error $? "missing argument to $ac_option" fi if test -n "$ac_unrecognized_opts"; then case $enable_option_checking in no) ;; - fatal) as_fn_error "unrecognized options: $ac_unrecognized_opts" ;; + fatal) as_fn_error $? "unrecognized options: $ac_unrecognized_opts" ;; *) $as_echo "$as_me: WARNING: unrecognized options: $ac_unrecognized_opts" >&2 ;; esac fi @@ -1173,7 +1201,7 @@ do [\\/$]* | ?:[\\/]* ) continue;; NONE | '' ) case $ac_var in *prefix ) continue;; esac;; esac - as_fn_error "expected an absolute directory name for --$ac_var: $ac_val" + as_fn_error $? "expected an absolute directory name for --$ac_var: $ac_val" done # There might be people who depend on the old broken behavior: `$host' @@ -1187,8 +1215,8 @@ target=$target_alias if test "x$host_alias" != x; then if test "x$build_alias" = x; then cross_compiling=maybe - $as_echo "$as_me: WARNING: If you wanted to set the --build type, don't use --host. - If a cross compiler is detected then cross compile mode will be used." >&2 + $as_echo "$as_me: WARNING: if you wanted to set the --build type, don't use --host. + If a cross compiler is detected then cross compile mode will be used" >&2 elif test "x$build_alias" != "x$host_alias"; then cross_compiling=yes fi @@ -1203,9 +1231,9 @@ test "$silent" = yes && exec 6>/dev/null ac_pwd=`pwd` && test -n "$ac_pwd" && ac_ls_di=`ls -di .` && ac_pwd_ls_di=`cd "$ac_pwd" && ls -di .` || - as_fn_error "working directory cannot be determined" + as_fn_error $? "working directory cannot be determined" test "X$ac_ls_di" = "X$ac_pwd_ls_di" || - as_fn_error "pwd does not report name of working directory" + as_fn_error $? "pwd does not report name of working directory" # Find the source files, if location was not specified. @@ -1244,11 +1272,11 @@ else fi if test ! -r "$srcdir/$ac_unique_file"; then test "$ac_srcdir_defaulted" = yes && srcdir="$ac_confdir or .." - as_fn_error "cannot find sources ($ac_unique_file) in $srcdir" + as_fn_error $? "cannot find sources ($ac_unique_file) in $srcdir" fi ac_msg="sources are in $srcdir, but \`cd $srcdir' does not work" ac_abs_confdir=`( - cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error "$ac_msg" + cd "$srcdir" && test -r "./$ac_unique_file" || as_fn_error $? "$ac_msg" pwd)` # When building in place, set srcdir=. if test "$ac_abs_confdir" = "$ac_pwd"; then @@ -1288,7 +1316,7 @@ Configuration: --help=short display options specific to this package --help=recursive display the short help of all the included packages -V, --version display version information and exit - -q, --quiet, --silent do not print \`checking...' messages + -q, --quiet, --silent do not print \`checking ...' messages --cache-file=FILE cache test results in FILE [disabled] -C, --config-cache alias for \`--cache-file=config.cache' -n, --no-create do not create output files @@ -1346,25 +1374,24 @@ Optional Features: --disable-FEATURE do not include FEATURE (same as --enable-FEATURE=no) --enable-FEATURE[=ARG] include FEATURE [ARG=yes] --enable-autogen Automatically regenerate configure output + --disable-experimental Disable support for the experimental API --enable-cc-silence Silence irrelevant compiler warnings --enable-debug Build debugging code - --enable-stats Enable statistics calculation/reporting + --disable-stats Disable statistics calculation/reporting --enable-prof Enable allocation profiling --enable-prof-libunwind Use libunwind for backtracing --disable-prof-libgcc Do not use libgcc for backtracing --disable-prof-gcc Do not use gcc intrinsics for backtracing - --disable-tiny Disable tiny (sub-quantum) allocations --disable-tcache Disable per thread caches - --enable-swap Enable mmap()ped swap files + --enable-mremap Enable mremap(2) for huge realloc() + --disable-munmap Disable VM deallocation via munmap(2) --enable-dss Enable allocation from DSS - --enable-fill Support junk/zero filling option + --disable-fill Disable support for junk/zero filling, quarantine, + and redzones + --enable-utrace Enable utrace(2)-based tracing + --disable-valgrind Disable support for Valgrind --enable-xmalloc Support xmalloc option - --enable-sysv Support SYSV semantics option - --enable-dynamic-page-shift - Determine page size at run time (don't trust - configure result) - --disable-lazy-lock Disable lazy locking (always lock, even when - single-threaded) + --enable-lazy-lock Enable lazy locking (only lock when multi-threaded) --disable-tls Disable thread-local storage (__thread keyword) Optional Packages: @@ -1372,6 +1399,7 @@ Optional Packages: --without-PACKAGE do not use PACKAGE (same as --with-PACKAGE=no) --with-xslroot= XSL stylesheet root path --with-rpath= Colon-separated rpath (ELF systems only) + --with-mangling= Mangle symbols in --with-jemalloc-prefix= Prefix to prepend to all public APIs --with-private-namespace= @@ -1459,9 +1487,9 @@ test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF configure -generated by GNU Autoconf 2.65 +generated by GNU Autoconf 2.68 -Copyright (C) 2009 Free Software Foundation, Inc. +Copyright (C) 2010 Free Software Foundation, Inc. This configure script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it. _ACEOF @@ -1505,11 +1533,48 @@ sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=1 fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_compile +# ac_fn_c_try_cpp LINENO +# ---------------------- +# Try to preprocess conftest.$ac_ext, and return whether this succeeded. +ac_fn_c_try_cpp () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if { { ac_try="$ac_cpp conftest.$ac_ext" +case "(($ac_try" in + *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; + *) ac_try_echo=$ac_try;; +esac +eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" +$as_echo "$ac_try_echo"; } >&5 + (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err + ac_status=$? + if test -s conftest.err; then + grep -v '^ *+' conftest.err >conftest.er1 + cat conftest.er1 >&5 + mv -f conftest.er1 conftest.err + fi + $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 + test $ac_status = 0; } > conftest.i && { + test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || + test ! -s conftest.err + }; then : + ac_retval=0 +else + $as_echo "$as_me: failed program was:" >&5 +sed 's/^/| /' conftest.$ac_ext >&5 + + ac_retval=1 +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + as_fn_set_status $ac_retval + +} # ac_fn_c_try_cpp + # ac_fn_c_try_run LINENO # ---------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. Assumes @@ -1547,48 +1612,11 @@ sed 's/^/| /' conftest.$ac_ext >&5 ac_retval=$ac_status fi rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_run -# ac_fn_c_try_cpp LINENO -# ---------------------- -# Try to preprocess conftest.$ac_ext, and return whether this succeeded. -ac_fn_c_try_cpp () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { { ac_try="$ac_cpp conftest.$ac_ext" -case "(($ac_try" in - *\"* | *\`* | *\\*) ac_try_echo=\$ac_try;; - *) ac_try_echo=$ac_try;; -esac -eval ac_try_echo="\"\$as_me:${as_lineno-$LINENO}: $ac_try_echo\"" -$as_echo "$ac_try_echo"; } >&5 - (eval "$ac_cpp conftest.$ac_ext") 2>conftest.err - ac_status=$? - if test -s conftest.err; then - grep -v '^ *+' conftest.err >conftest.er1 - cat conftest.er1 >&5 - mv -f conftest.er1 conftest.err - fi - $as_echo "$as_me:${as_lineno-$LINENO}: \$? = $ac_status" >&5 - test $ac_status = 0; } >/dev/null && { - test -z "$ac_c_preproc_warn_flag$ac_c_werror_flag" || - test ! -s conftest.err - }; then : - ac_retval=0 -else - $as_echo "$as_me: failed program was:" >&5 -sed 's/^/| /' conftest.$ac_ext >&5 - - ac_retval=1 -fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} - as_fn_set_status $ac_retval - -} # ac_fn_c_try_cpp - # ac_fn_c_compute_int LINENO EXPR VAR INCLUDES # -------------------------------------------- # Tries to find the compile-time value of EXPR in a program that includes @@ -1762,7 +1790,7 @@ rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ rm -f conftest.val fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_compute_int @@ -1776,7 +1804,7 @@ ac_fn_c_check_header_compile () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : +if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -1794,97 +1822,10 @@ fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_header_compile -# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES -# ------------------------------------------------------- -# Tests whether HEADER exists, giving a warning if it cannot be compiled using -# the include files in INCLUDES and setting the cache variable VAR -# accordingly. -ac_fn_c_check_header_mongrel () -{ - as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - $as_echo_n "(cached) " >&6 -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -else - # Is the header compilable? -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 -$as_echo_n "checking $2 usability... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -$4 -#include <$2> -_ACEOF -if ac_fn_c_try_compile "$LINENO"; then : - ac_header_compiler=yes -else - ac_header_compiler=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 -$as_echo "$ac_header_compiler" >&6; } - -# Is the header present? -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 -$as_echo_n "checking $2 presence... " >&6; } -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#include <$2> -_ACEOF -if ac_fn_c_try_cpp "$LINENO"; then : - ac_header_preproc=yes -else - ac_header_preproc=no -fi -rm -f conftest.err conftest.$ac_ext -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 -$as_echo "$ac_header_preproc" >&6; } - -# So? What about this header? -case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( - yes:no: ) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 -$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 -$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} - ;; - no:yes:* ) - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 -$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 -$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 -$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 -$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} - { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 -$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} - ;; -esac - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 -$as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : - $as_echo_n "(cached) " >&6 -else - eval "$3=\$ac_header_compiler" -fi -eval ac_res=\$$3 - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 -$as_echo "$ac_res" >&6; } -fi - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} - -} # ac_fn_c_check_header_mongrel - # ac_fn_c_try_link LINENO # ----------------------- # Try to link conftest.$ac_ext, and return whether this succeeded. @@ -1926,7 +1867,7 @@ fi # interfere with the next link command; also delete a directory that is # left behind by Apple's compiler. We do this before executing the actions. rm -rf conftest.dSYM conftest_ipa8_conftest.oo - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno as_fn_set_status $ac_retval } # ac_fn_c_try_link @@ -1939,7 +1880,7 @@ ac_fn_c_check_func () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : +if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -1994,10 +1935,97 @@ fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_func +# ac_fn_c_check_header_mongrel LINENO HEADER VAR INCLUDES +# ------------------------------------------------------- +# Tests whether HEADER exists, giving a warning if it cannot be compiled using +# the include files in INCLUDES and setting the cache variable VAR +# accordingly. +ac_fn_c_check_header_mongrel () +{ + as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + if eval \${$3+:} false; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +else + # Is the header compilable? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 usability" >&5 +$as_echo_n "checking $2 usability... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +$4 +#include <$2> +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + ac_header_compiler=yes +else + ac_header_compiler=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_compiler" >&5 +$as_echo "$ac_header_compiler" >&6; } + +# Is the header present? +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking $2 presence" >&5 +$as_echo_n "checking $2 presence... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include <$2> +_ACEOF +if ac_fn_c_try_cpp "$LINENO"; then : + ac_header_preproc=yes +else + ac_header_preproc=no +fi +rm -f conftest.err conftest.i conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_header_preproc" >&5 +$as_echo "$ac_header_preproc" >&6; } + +# So? What about this header? +case $ac_header_compiler:$ac_header_preproc:$ac_c_preproc_warn_flag in #(( + yes:no: ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&5 +$as_echo "$as_me: WARNING: $2: accepted by the compiler, rejected by the preprocessor!" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; + no:yes:* ) + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: present but cannot be compiled" >&5 +$as_echo "$as_me: WARNING: $2: present but cannot be compiled" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: check for missing prerequisite headers?" >&5 +$as_echo "$as_me: WARNING: $2: check for missing prerequisite headers?" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: see the Autoconf documentation" >&5 +$as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&5 +$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;} + { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5 +$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;} + ;; +esac + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 +$as_echo_n "checking for $2... " >&6; } +if eval \${$3+:} false; then : + $as_echo_n "(cached) " >&6 +else + eval "$3=\$ac_header_compiler" +fi +eval ac_res=\$$3 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 +$as_echo "$ac_res" >&6; } +fi + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno + +} # ac_fn_c_check_header_mongrel + # ac_fn_c_check_type LINENO TYPE VAR INCLUDES # ------------------------------------------- # Tests whether TYPE exists after having included INCLUDES, setting cache @@ -2007,7 +2035,7 @@ ac_fn_c_check_type () as_lineno=${as_lineno-"$1"} as_lineno_stack=as_lineno_stack=$as_lineno_stack { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $2" >&5 $as_echo_n "checking for $2... " >&6; } -if { as_var=$3; eval "test \"\${$as_var+set}\" = set"; }; then : +if eval \${$3+:} false; then : $as_echo_n "(cached) " >&6 else eval "$3=no" @@ -2048,7 +2076,7 @@ fi eval ac_res=\$$3 { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_res" >&5 $as_echo "$ac_res" >&6; } - eval $as_lineno_stack; test "x$as_lineno_stack" = x && { as_lineno=; unset as_lineno;} + eval $as_lineno_stack; ${as_lineno_stack:+:} unset as_lineno } # ac_fn_c_check_type cat >config.log <<_ACEOF @@ -2056,7 +2084,7 @@ This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. It was created by $as_me, which was -generated by GNU Autoconf 2.65. Invocation command line was +generated by GNU Autoconf 2.68. Invocation command line was $ $0 $@ @@ -2166,11 +2194,9 @@ trap 'exit_status=$? { echo - cat <<\_ASBOX -## ---------------- ## + $as_echo "## ---------------- ## ## Cache variables. ## -## ---------------- ## -_ASBOX +## ---------------- ##" echo # The following way of writing the cache mishandles newlines in values, ( @@ -2204,11 +2230,9 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; ) echo - cat <<\_ASBOX -## ----------------- ## + $as_echo "## ----------------- ## ## Output variables. ## -## ----------------- ## -_ASBOX +## ----------------- ##" echo for ac_var in $ac_subst_vars do @@ -2221,11 +2245,9 @@ _ASBOX echo if test -n "$ac_subst_files"; then - cat <<\_ASBOX -## ------------------- ## + $as_echo "## ------------------- ## ## File substitutions. ## -## ------------------- ## -_ASBOX +## ------------------- ##" echo for ac_var in $ac_subst_files do @@ -2239,11 +2261,9 @@ _ASBOX fi if test -s confdefs.h; then - cat <<\_ASBOX -## ----------- ## + $as_echo "## ----------- ## ## confdefs.h. ## -## ----------- ## -_ASBOX +## ----------- ##" echo cat confdefs.h echo @@ -2298,7 +2318,12 @@ _ACEOF ac_site_file1=NONE ac_site_file2=NONE if test -n "$CONFIG_SITE"; then - ac_site_file1=$CONFIG_SITE + # We do not want a PATH search for config.site. + case $CONFIG_SITE in #(( + -*) ac_site_file1=./$CONFIG_SITE;; + */*) ac_site_file1=$CONFIG_SITE;; + *) ac_site_file1=./$CONFIG_SITE;; + esac elif test "x$prefix" != xNONE; then ac_site_file1=$prefix/share/config.site ac_site_file2=$prefix/etc/config.site @@ -2313,7 +2338,11 @@ do { $as_echo "$as_me:${as_lineno-$LINENO}: loading site script $ac_site_file" >&5 $as_echo "$as_me: loading site script $ac_site_file" >&6;} sed 's/^/| /' "$ac_site_file" >&5 - . "$ac_site_file" + . "$ac_site_file" \ + || { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "failed to load site script $ac_site_file +See \`config.log' for more details" "$LINENO" 5; } fi done @@ -2389,7 +2418,7 @@ if $ac_cache_corrupted; then $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} { $as_echo "$as_me:${as_lineno-$LINENO}: error: changes in the environment can compromise the build" >&5 $as_echo "$as_me: error: changes in the environment can compromise the build" >&2;} - as_fn_error "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 + as_fn_error $? "run \`make distclean' and/or \`rm $cache_file' and start over" "$LINENO" 5 fi ## -------------------- ## ## Main body of script. ## @@ -2409,6 +2438,9 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu +rev=1 + + srcroot=$srcdir if test "x${srcroot}" = "x." ; then srcroot="" @@ -2452,7 +2484,7 @@ MANDIR=`eval echo $MANDIR` set dummy xsltproc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_XSLTPROC+set}" = set; then : +if ${ac_cv_path_XSLTPROC+:} false; then : $as_echo_n "(cached) " >&6 else case $XSLTPROC in @@ -2488,16 +2520,25 @@ $as_echo "no" >&6; } fi +if test -d "/usr/share/xml/docbook/stylesheet/docbook-xsl" ; then + DEFAULT_XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +elif test -d "/usr/share/sgml/docbook/xsl-stylesheets" ; then + DEFAULT_XSLROOT="/usr/share/sgml/docbook/xsl-stylesheets" +else + DEFAULT_XSLROOT="" +fi # Check whether --with-xslroot was given. if test "${with_xslroot+set}" = set; then : - withval=$with_xslroot; if test "x$with_xslroot" = "xno" ; then - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" + withval=$with_xslroot; +if test "x$with_xslroot" = "xno" ; then + XSLROOT="${DEFAULT_XSLROOT}" else XSLROOT="${with_xslroot}" fi + else - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" + XSLROOT="${DEFAULT_XSLROOT}" fi @@ -2514,7 +2555,7 @@ if test -n "$ac_tool_prefix"; then set dummy ${ac_tool_prefix}gcc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2554,7 +2595,7 @@ if test -z "$ac_cv_prog_CC"; then set dummy gcc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then : +if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then @@ -2607,7 +2648,7 @@ if test -z "$CC"; then set dummy ${ac_tool_prefix}cc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2647,7 +2688,7 @@ if test -z "$CC"; then set dummy cc; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2706,7 +2747,7 @@ if test -z "$CC"; then set dummy $ac_tool_prefix$ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_CC+set}" = set; then : +if ${ac_cv_prog_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$CC"; then @@ -2750,7 +2791,7 @@ do set dummy $ac_prog; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_ac_ct_CC+set}" = set; then : +if ${ac_cv_prog_ac_ct_CC+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_CC"; then @@ -2804,8 +2845,8 @@ fi test -z "$CC" && { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "no acceptable C compiler found in \$PATH -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "no acceptable C compiler found in \$PATH +See \`config.log' for more details" "$LINENO" 5; } # Provide some information about the compiler. $as_echo "$as_me:${as_lineno-$LINENO}: checking for C compiler version" >&5 @@ -2919,9 +2960,8 @@ sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "C compiler cannot create executables -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "C compiler cannot create executables +See \`config.log' for more details" "$LINENO" 5; } else { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } @@ -2963,8 +3003,8 @@ done else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot compute suffix of executables: cannot compile and link -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "cannot compute suffix of executables: cannot compile and link +See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest conftest$ac_cv_exeext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_exeext" >&5 @@ -3021,9 +3061,9 @@ $as_echo "$ac_try_echo"; } >&5 else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run C compiled programs. +as_fn_error $? "cannot run C compiled programs. If you meant to cross compile, use \`--host'. -See \`config.log' for more details." "$LINENO" 5; } +See \`config.log' for more details" "$LINENO" 5; } fi fi fi @@ -3034,7 +3074,7 @@ rm -f conftest.$ac_ext conftest$ac_cv_exeext conftest.out ac_clean_files=$ac_clean_files_save { $as_echo "$as_me:${as_lineno-$LINENO}: checking for suffix of object files" >&5 $as_echo_n "checking for suffix of object files... " >&6; } -if test "${ac_cv_objext+set}" = set; then : +if ${ac_cv_objext+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -3074,8 +3114,8 @@ sed 's/^/| /' conftest.$ac_ext >&5 { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot compute suffix of object files: cannot compile -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "cannot compute suffix of object files: cannot compile +See \`config.log' for more details" "$LINENO" 5; } fi rm -f conftest.$ac_cv_objext conftest.$ac_ext fi @@ -3085,7 +3125,7 @@ OBJEXT=$ac_cv_objext ac_objext=$OBJEXT { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether we are using the GNU C compiler" >&5 $as_echo_n "checking whether we are using the GNU C compiler... " >&6; } -if test "${ac_cv_c_compiler_gnu+set}" = set; then : +if ${ac_cv_c_compiler_gnu+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -3122,7 +3162,7 @@ ac_test_CFLAGS=${CFLAGS+set} ac_save_CFLAGS=$CFLAGS { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether $CC accepts -g" >&5 $as_echo_n "checking whether $CC accepts -g... " >&6; } -if test "${ac_cv_prog_cc_g+set}" = set; then : +if ${ac_cv_prog_cc_g+:} false; then : $as_echo_n "(cached) " >&6 else ac_save_c_werror_flag=$ac_c_werror_flag @@ -3200,7 +3240,7 @@ else fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $CC option to accept ISO C89" >&5 $as_echo_n "checking for $CC option to accept ISO C89... " >&6; } -if test "${ac_cv_prog_cc_c89+set}" = set; then : +if ${ac_cv_prog_cc_c89+:} false; then : $as_echo_n "(cached) " >&6 else ac_cv_prog_cc_c89=no @@ -3295,12 +3335,44 @@ ac_compile='$CC -c $CFLAGS $CPPFLAGS conftest.$ac_ext >&5' ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $LIBS >&5' ac_compiler_gnu=$ac_cv_c_compiler_gnu -if test "x$CFLAGS" = "x" ; then - no_CFLAGS="yes" - if test "x$GCC" = "xyes" ; then +if test "x$GCC" != "xyes" ; then +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler is MSVC" >&5 +$as_echo_n "checking whether compiler is MSVC... " >&6; } +if ${je_cv_msvc+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -std=gnu99" >&5 +int +main () +{ + +#ifndef _MSC_VER + int fail-1; +#endif + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + je_cv_msvc=yes +else + je_cv_msvc=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_msvc" >&5 +$as_echo "$je_cv_msvc" >&6; } +fi + +if test "x$CFLAGS" = "x" ; then + no_CFLAGS="yes" + if test "x$GCC" = "xyes" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -std=gnu99" >&5 $as_echo_n "checking whether compiler supports -std=gnu99... " >&6; } TCFLAGS="${CFLAGS}" if test "x${CFLAGS}" = "x" ; then @@ -3308,13 +3380,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -std=gnu99" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3328,7 +3394,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -3337,10 +3403,7 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Wall" >&5 @@ -3351,13 +3414,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -Wall" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3371,7 +3428,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -3380,10 +3437,7 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -pipe" >&5 @@ -3394,13 +3448,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -pipe" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3414,7 +3462,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -3423,10 +3471,7 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -g3" >&5 @@ -3437,13 +3482,43 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -g3" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + elif test "x$je_cv_msvc" = "xyes" ; then + CC="$CC -nologo" + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Zi" >&5 +$as_echo_n "checking whether compiler supports -Zi... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-Zi" +else + CFLAGS="${CFLAGS} -Zi" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3457,7 +3532,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -3466,11 +3541,77 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -MT" >&5 +$as_echo_n "checking whether compiler supports -MT... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-MT" +else + CFLAGS="${CFLAGS} -MT" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -W3" >&5 +$as_echo_n "checking whether compiler supports -W3... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-W3" +else + CFLAGS="${CFLAGS} -W3" fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat" fi fi if test "x$EXTRA_CFLAGS" != "x" ; then @@ -3483,13 +3624,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} $EXTRA_CFLAGS" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -3503,7 +3638,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -3512,10 +3647,7 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi ac_ext=c @@ -3530,7 +3662,7 @@ if test -n "$CPP" && test -d "$CPP"; then CPP= fi if test -z "$CPP"; then - if test "${ac_cv_prog_CPP+set}" = set; then : + if ${ac_cv_prog_CPP+:} false; then : $as_echo_n "(cached) " >&6 else # Double quotes because CPP needs to be expanded @@ -3560,7 +3692,7 @@ else # Broken: fails on valid input. continue fi -rm -f conftest.err conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. @@ -3576,11 +3708,11 @@ else ac_preproc_ok=: break fi -rm -f conftest.err conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.err conftest.$ac_ext +rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : break fi @@ -3619,7 +3751,7 @@ else # Broken: fails on valid input. continue fi -rm -f conftest.err conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext # OK, works on sane cases. Now check whether nonexistent headers # can be detected and how. @@ -3635,18 +3767,18 @@ else ac_preproc_ok=: break fi -rm -f conftest.err conftest.$ac_ext +rm -f conftest.err conftest.i conftest.$ac_ext done # Because of `break', _AC_PREPROC_IFELSE's cleaning code was skipped. -rm -f conftest.err conftest.$ac_ext +rm -f conftest.i conftest.err conftest.$ac_ext if $ac_preproc_ok; then : else { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "C preprocessor \"$CPP\" fails sanity check -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "C preprocessor \"$CPP\" fails sanity check +See \`config.log' for more details" "$LINENO" 5; } fi ac_ext=c @@ -3659,7 +3791,7 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu { $as_echo "$as_me:${as_lineno-$LINENO}: checking for grep that handles long lines and -e" >&5 $as_echo_n "checking for grep that handles long lines and -e... " >&6; } -if test "${ac_cv_path_GREP+set}" = set; then : +if ${ac_cv_path_GREP+:} false; then : $as_echo_n "(cached) " >&6 else if test -z "$GREP"; then @@ -3708,7 +3840,7 @@ esac done IFS=$as_save_IFS if test -z "$ac_cv_path_GREP"; then - as_fn_error "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + as_fn_error $? "no acceptable grep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_GREP=$GREP @@ -3722,7 +3854,7 @@ $as_echo "$ac_cv_path_GREP" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for egrep" >&5 $as_echo_n "checking for egrep... " >&6; } -if test "${ac_cv_path_EGREP+set}" = set; then : +if ${ac_cv_path_EGREP+:} false; then : $as_echo_n "(cached) " >&6 else if echo a | $GREP -E '(a|b)' >/dev/null 2>&1 @@ -3774,7 +3906,7 @@ esac done IFS=$as_save_IFS if test -z "$ac_cv_path_EGREP"; then - as_fn_error "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 + as_fn_error $? "no acceptable egrep could be found in $PATH$PATH_SEPARATOR/usr/xpg4/bin" "$LINENO" 5 fi else ac_cv_path_EGREP=$EGREP @@ -3789,7 +3921,7 @@ $as_echo "$ac_cv_path_EGREP" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: checking for ANSI C header files" >&5 $as_echo_n "checking for ANSI C header files... " >&6; } -if test "${ac_cv_header_stdc+set}" = set; then : +if ${ac_cv_header_stdc+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -3906,8 +4038,7 @@ do : as_ac_Header=`$as_echo "ac_cv_header_$ac_header" | $as_tr_sh` ac_fn_c_check_header_compile "$LINENO" "$ac_header" "$as_ac_Header" "$ac_includes_default " -eval as_val=\$$as_ac_Header - if test "x$as_val" = x""yes; then : +if eval test \"x\$"$as_ac_Header"\" = x"yes"; then : cat >>confdefs.h <<_ACEOF #define `$as_echo "HAVE_$ac_header" | $as_tr_cpp` 1 _ACEOF @@ -3923,7 +4054,7 @@ done # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of void *" >&5 $as_echo_n "checking size of void *... " >&6; } -if test "${ac_cv_sizeof_void_p+set}" = set; then : +if ${ac_cv_sizeof_void_p+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (void *))" "ac_cv_sizeof_void_p" "$ac_includes_default"; then : @@ -3932,9 +4063,8 @@ else if test "$ac_cv_type_void_p" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "cannot compute sizeof (void *) -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "cannot compute sizeof (void *) +See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_void_p=0 fi @@ -3956,7 +4086,7 @@ if test "x${ac_cv_sizeof_void_p}" = "x8" ; then elif test "x${ac_cv_sizeof_void_p}" = "x4" ; then LG_SIZEOF_PTR=2 else - as_fn_error "Unsupported pointer size: ${ac_cv_sizeof_void_p}" "$LINENO" 5 + as_fn_error $? "Unsupported pointer size: ${ac_cv_sizeof_void_p}" "$LINENO" 5 fi cat >>confdefs.h <<_ACEOF #define LG_SIZEOF_PTR $LG_SIZEOF_PTR @@ -3969,7 +4099,7 @@ _ACEOF # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of int" >&5 $as_echo_n "checking size of int... " >&6; } -if test "${ac_cv_sizeof_int+set}" = set; then : +if ${ac_cv_sizeof_int+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (int))" "ac_cv_sizeof_int" "$ac_includes_default"; then : @@ -3978,9 +4108,8 @@ else if test "$ac_cv_type_int" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "cannot compute sizeof (int) -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "cannot compute sizeof (int) +See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_int=0 fi @@ -4002,7 +4131,7 @@ if test "x${ac_cv_sizeof_int}" = "x8" ; then elif test "x${ac_cv_sizeof_int}" = "x4" ; then LG_SIZEOF_INT=2 else - as_fn_error "Unsupported int size: ${ac_cv_sizeof_int}" "$LINENO" 5 + as_fn_error $? "Unsupported int size: ${ac_cv_sizeof_int}" "$LINENO" 5 fi cat >>confdefs.h <<_ACEOF #define LG_SIZEOF_INT $LG_SIZEOF_INT @@ -4015,7 +4144,7 @@ _ACEOF # This bug is HP SR number 8606223364. { $as_echo "$as_me:${as_lineno-$LINENO}: checking size of long" >&5 $as_echo_n "checking size of long... " >&6; } -if test "${ac_cv_sizeof_long+set}" = set; then : +if ${ac_cv_sizeof_long+:} false; then : $as_echo_n "(cached) " >&6 else if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (long))" "ac_cv_sizeof_long" "$ac_includes_default"; then : @@ -4024,9 +4153,8 @@ else if test "$ac_cv_type_long" = yes; then { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -{ as_fn_set_status 77 -as_fn_error "cannot compute sizeof (long) -See \`config.log' for more details." "$LINENO" 5; }; } +as_fn_error 77 "cannot compute sizeof (long) +See \`config.log' for more details" "$LINENO" 5; } else ac_cv_sizeof_long=0 fi @@ -4048,25 +4176,78 @@ if test "x${ac_cv_sizeof_long}" = "x8" ; then elif test "x${ac_cv_sizeof_long}" = "x4" ; then LG_SIZEOF_LONG=2 else - as_fn_error "Unsupported long size: ${ac_cv_sizeof_long}" "$LINENO" 5 + as_fn_error $? "Unsupported long size: ${ac_cv_sizeof_long}" "$LINENO" 5 fi cat >>confdefs.h <<_ACEOF #define LG_SIZEOF_LONG $LG_SIZEOF_LONG _ACEOF +# The cast to long int works around a bug in the HP C Compiler +# version HP92453-01 B.11.11.23709.GP, which incorrectly rejects +# declarations like `int a3[[(sizeof (unsigned char)) >= 0]];'. +# This bug is HP SR number 8606223364. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking size of intmax_t" >&5 +$as_echo_n "checking size of intmax_t... " >&6; } +if ${ac_cv_sizeof_intmax_t+:} false; then : + $as_echo_n "(cached) " >&6 +else + if ac_fn_c_compute_int "$LINENO" "(long int) (sizeof (intmax_t))" "ac_cv_sizeof_intmax_t" "$ac_includes_default"; then : + +else + if test "$ac_cv_type_intmax_t" = yes; then + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error 77 "cannot compute sizeof (intmax_t) +See \`config.log' for more details" "$LINENO" 5; } + else + ac_cv_sizeof_intmax_t=0 + fi +fi + +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_sizeof_intmax_t" >&5 +$as_echo "$ac_cv_sizeof_intmax_t" >&6; } + + + +cat >>confdefs.h <<_ACEOF +#define SIZEOF_INTMAX_T $ac_cv_sizeof_intmax_t +_ACEOF + + +if test "x${ac_cv_sizeof_intmax_t}" = "x16" ; then + LG_SIZEOF_INTMAX_T=4 +elif test "x${ac_cv_sizeof_intmax_t}" = "x8" ; then + LG_SIZEOF_INTMAX_T=3 +elif test "x${ac_cv_sizeof_intmax_t}" = "x4" ; then + LG_SIZEOF_INTMAX_T=2 +else + as_fn_error $? "Unsupported intmax_t size: ${ac_cv_sizeof_intmax_t}" "$LINENO" 5 +fi +cat >>confdefs.h <<_ACEOF +#define LG_SIZEOF_INTMAX_T $LG_SIZEOF_INTMAX_T +_ACEOF + + ac_aux_dir= for ac_dir in "$srcdir" "$srcdir/.." "$srcdir/../.."; do - for ac_t in install-sh install.sh shtool; do - if test -f "$ac_dir/$ac_t"; then - ac_aux_dir=$ac_dir - ac_install_sh="$ac_aux_dir/$ac_t -c" - break 2 - fi - done + if test -f "$ac_dir/install-sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install-sh -c" + break + elif test -f "$ac_dir/install.sh"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/install.sh -c" + break + elif test -f "$ac_dir/shtool"; then + ac_aux_dir=$ac_dir + ac_install_sh="$ac_aux_dir/shtool install -c" + break + fi done if test -z "$ac_aux_dir"; then - as_fn_error "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 + as_fn_error $? "cannot find install-sh, install.sh, or shtool in \"$srcdir\" \"$srcdir/..\" \"$srcdir/../..\"" "$LINENO" 5 fi # These three variables are undocumented and unsupported, @@ -4080,27 +4261,27 @@ ac_configure="$SHELL $ac_aux_dir/configure" # Please don't use this var. # Make sure we can run config.sub. $SHELL "$ac_aux_dir/config.sub" sun4 >/dev/null 2>&1 || - as_fn_error "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 + as_fn_error $? "cannot run $SHELL $ac_aux_dir/config.sub" "$LINENO" 5 { $as_echo "$as_me:${as_lineno-$LINENO}: checking build system type" >&5 $as_echo_n "checking build system type... " >&6; } -if test "${ac_cv_build+set}" = set; then : +if ${ac_cv_build+:} false; then : $as_echo_n "(cached) " >&6 else ac_build_alias=$build_alias test "x$ac_build_alias" = x && ac_build_alias=`$SHELL "$ac_aux_dir/config.guess"` test "x$ac_build_alias" = x && - as_fn_error "cannot guess build type; you must specify one" "$LINENO" 5 + as_fn_error $? "cannot guess build type; you must specify one" "$LINENO" 5 ac_cv_build=`$SHELL "$ac_aux_dir/config.sub" $ac_build_alias` || - as_fn_error "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $ac_build_alias failed" "$LINENO" 5 fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_build" >&5 $as_echo "$ac_cv_build" >&6; } case $ac_cv_build in *-*-*) ;; -*) as_fn_error "invalid value of canonical build" "$LINENO" 5;; +*) as_fn_error $? "invalid value of canonical build" "$LINENO" 5;; esac build=$ac_cv_build ac_save_IFS=$IFS; IFS='-' @@ -4118,14 +4299,14 @@ case $build_os in *\ *) build_os=`echo "$build_os" | sed 's/ /-/g'`;; esac { $as_echo "$as_me:${as_lineno-$LINENO}: checking host system type" >&5 $as_echo_n "checking host system type... " >&6; } -if test "${ac_cv_host+set}" = set; then : +if ${ac_cv_host+:} false; then : $as_echo_n "(cached) " >&6 else if test "x$host_alias" = x; then ac_cv_host=$ac_cv_build else ac_cv_host=`$SHELL "$ac_aux_dir/config.sub" $host_alias` || - as_fn_error "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 + as_fn_error $? "$SHELL $ac_aux_dir/config.sub $host_alias failed" "$LINENO" 5 fi fi @@ -4133,7 +4314,7 @@ fi $as_echo "$ac_cv_host" >&6; } case $ac_cv_host in *-*-*) ;; -*) as_fn_error "invalid value of canonical host" "$LINENO" 5;; +*) as_fn_error $? "invalid value of canonical host" "$LINENO" 5;; esac host=$ac_cv_host ac_save_IFS=$IFS; IFS='-' @@ -4157,11 +4338,8 @@ case "${host_cpu}" in { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __asm__ is compilable" >&5 $as_echo_n "checking whether __asm__ is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +if ${je_cv_asm+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4174,22 +4352,18 @@ __asm__ volatile("pause"); return 0; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - asm="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_asm=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - asm="no" - + je_cv_asm=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_asm" >&5 +$as_echo "$je_cv_asm" >&6; } - - if test "x${asm}" = "xyes" ; then + if test "x${je_cv_asm}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi ;; @@ -4197,11 +4371,8 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __asm__ syntax is compilable" >&5 $as_echo_n "checking whether __asm__ syntax is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +if ${je_cv_asm+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4214,22 +4385,18 @@ __asm__ volatile("pause"); return 0; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - asm="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_asm=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - asm="no" - + je_cv_asm=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_asm" >&5 +$as_echo "$je_cv_asm" >&6; } - - if test "x${asm}" = "xyes" ; then + if test "x${je_cv_asm}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi ;; @@ -4241,28 +4408,54 @@ cat >>confdefs.h <<_ACEOF _ACEOF +LD_PRELOAD_VAR="LD_PRELOAD" +so="so" +importlib="${so}" +o="$ac_objext" +a="a" +exe="$ac_exeext" +libprefix="lib" +DSO_LDFLAGS='-shared -Wl,-soname,$(@F)' +RPATH='-Wl,-rpath,$(1)' +SOREV="${so}.${rev}" +PIC_CFLAGS='-fPIC -DPIC' +CTARGET='-o $@' +LDTARGET='-o $@' +EXTRA_LDFLAGS= +MKLIB='ar crus $@' +CC_MM=1 + +default_munmap="1" case "${host}" in *-*-darwin*) - CFLAGS="$CFLAGS -fno-common -no-cpp-precomp" + CFLAGS="$CFLAGS" abi="macho" - $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h RPATH="" + LD_PRELOAD_VAR="DYLD_INSERT_LIBRARIES" + so="dylib" + importlib="${so}" + force_tls="0" + DSO_LDFLAGS='-shared -Wl,-dylib_install_name,$(@F)' + SOREV="${rev}.${so}" ;; *-*-freebsd*) CFLAGS="$CFLAGS" abi="elf" - $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h - RPATH="-Wl,-rpath," + force_lazy_lock="1" ;; *-*-linux*) CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" abi="elf" - $as_echo "#define JEMALLOC_PURGE_MADVISE_DONTNEED 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_DONTNEED " >>confdefs.h + + $as_echo "#define JEMALLOC_THREADED_INIT " >>confdefs.h - RPATH="-Wl,-rpath," + default_munmap="0" ;; *-*-netbsd*) { $as_echo "$as_me:${as_lineno-$LINENO}: checking ABI" >&5 @@ -4291,35 +4484,79 @@ fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: result: $abi" >&5 $as_echo "$abi" >&6; } - $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE 1" >>confdefs.h + $as_echo "#define JEMALLOC_PURGE_MADVISE_FREE " >>confdefs.h - RPATH="-Wl,-rpath," ;; *-*-solaris2*) CFLAGS="$CFLAGS" abi="elf" - RPATH="-Wl,-R," + RPATH='-Wl,-R,$(1)' CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" LIBS="$LIBS -lposix4 -lsocket -lnsl" ;; + *-ibm-aix*) + if "$LG_SIZEOF_PTR" = "8"; then + LD_PRELOAD_VAR="LDR_PRELOAD64" + else + LD_PRELOAD_VAR="LDR_PRELOAD" + fi + abi="xcoff" + ;; + *-*-mingw*) + abi="pecoff" + force_tls="0" + RPATH="" + so="dll" + if test "x$je_cv_msvc" = "xyes" ; then + importlib="lib" + DSO_LDFLAGS="-LD" + EXTRA_LDFLAGS="-link -DEBUG" + CTARGET='-Fo$@' + LDTARGET='-Fe$@' + MKLIB='lib -nologo -out:$@' + CC_MM= + else + importlib="${so}" + DSO_LDFLAGS="-shared" + fi + a="lib" + libprefix="" + SOREV="${so}" + PIC_CFLAGS="" + ;; *) { $as_echo "$as_me:${as_lineno-$LINENO}: result: Unsupported operating system: ${host}" >&5 $as_echo "Unsupported operating system: ${host}" >&6; } abi="elf" - RPATH="-Wl,-rpath," ;; esac + + + + + + + + + + + + + + +if test "x$abi" != "xpecoff"; then + LIBS="$LIBS -lm" +fi + + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether __attribute__ syntax is compilable" >&5 $as_echo_n "checking whether __attribute__ syntax is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +if ${je_cv_attribute+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4332,22 +4569,18 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - attribute="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_attribute=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - attribute="no" - + je_cv_attribute=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_attribute" >&5 +$as_echo "$je_cv_attribute" >&6; } - -if test "x${attribute}" = "xyes" ; then +if test "x${je_cv_attribute}" = "xyes" ; then $as_echo "#define JEMALLOC_HAVE_ATTR " >>confdefs.h if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then @@ -4360,13 +4593,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -fvisibility=hidden" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4380,7 +4607,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -4389,56 +4616,81 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi fi +SAVED_CFLAGS="${CFLAGS}" - -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether mremap(...MREMAP_FIXED...) is compilable" >&5 -$as_echo_n "checking whether mremap(...MREMAP_FIXED...) is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -Werror" >&5 +$as_echo_n "checking whether compiler supports -Werror... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-Werror" else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext + CFLAGS="${CFLAGS} -Werror" +fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#define _GNU_SOURCE -#include int main () { -void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); + return 0; ; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } - mremap_fixed="yes" else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } - mremap_fixed="no" + CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether tls_model attribute is compilable" >&5 +$as_echo_n "checking whether tls_model attribute is compilable... " >&6; } +if ${je_cv_tls_model+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +int +main () +{ +static __thread int + __attribute__((tls_model("initial-exec"))) foo; + foo = 0; + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_tls_model=yes +else + je_cv_tls_model=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_tls_model" >&5 +$as_echo "$je_cv_tls_model" >&6; } +CFLAGS="${SAVED_CFLAGS}" +if test "x${je_cv_tls_model}" = "xyes" ; then + $as_echo "#define JEMALLOC_TLS_MODEL __attribute__((tls_model(\"initial-exec\")))" >>confdefs.h -if test "x${mremap_fixed}" = "xyes" ; then - $as_echo "#define JEMALLOC_MREMAP_FIXED 1" >>confdefs.h +else + $as_echo "#define JEMALLOC_TLS_MODEL " >>confdefs.h fi @@ -4489,7 +4741,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for a BSD-compatible install" >&5 $as_echo_n "checking for a BSD-compatible install... " >&6; } if test -z "$INSTALL"; then -if test "${ac_cv_path_install+set}" = set; then : +if ${ac_cv_path_install+:} false; then : $as_echo_n "(cached) " >&6 else as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -4570,7 +4822,7 @@ if test -n "$ac_tool_prefix"; then set dummy ${ac_tool_prefix}ranlib; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_RANLIB+set}" = set; then : +if ${ac_cv_prog_RANLIB+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$RANLIB"; then @@ -4610,7 +4862,7 @@ if test -z "$ac_cv_prog_RANLIB"; then set dummy ranlib; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_prog_ac_ct_RANLIB+set}" = set; then : +if ${ac_cv_prog_ac_ct_RANLIB+:} false; then : $as_echo_n "(cached) " >&6 else if test -n "$ac_ct_RANLIB"; then @@ -4661,7 +4913,7 @@ fi set dummy ar; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_AR+set}" = set; then : +if ${ac_cv_path_AR+:} false; then : $as_echo_n "(cached) " >&6 else case $AR in @@ -4701,7 +4953,7 @@ fi set dummy ld; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_LD+set}" = set; then : +if ${ac_cv_path_LD+:} false; then : $as_echo_n "(cached) " >&6 else case $LD in @@ -4741,7 +4993,7 @@ fi set dummy autoconf; ac_word=$2 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for $ac_word" >&5 $as_echo_n "checking for $ac_word... " >&6; } -if test "${ac_cv_path_AUTOCONF+set}" = set; then : +if ${ac_cv_path_AUTOCONF+:} false; then : $as_echo_n "(cached) " >&6 else case $AUTOCONF in @@ -4778,12 +5030,68 @@ fi +public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free malloc_usable_size malloc_stats_print mallctl mallctlnametomib mallctlbymib" + +ac_fn_c_check_func "$LINENO" "memalign" "ac_cv_func_memalign" +if test "x$ac_cv_func_memalign" = xyes; then : + $as_echo "#define JEMALLOC_OVERRIDE_MEMALIGN " >>confdefs.h + + public_syms="${public_syms} memalign" +fi + +ac_fn_c_check_func "$LINENO" "valloc" "ac_cv_func_valloc" +if test "x$ac_cv_func_valloc" = xyes; then : + $as_echo "#define JEMALLOC_OVERRIDE_VALLOC " >>confdefs.h + + public_syms="${public_syms} valloc" +fi + + +# Check whether --enable-experimental was given. +if test "${enable_experimental+set}" = set; then : + enableval=$enable_experimental; if test "x$enable_experimental" = "xno" ; then + enable_experimental="0" +else + enable_experimental="1" +fi + +else + enable_experimental="1" + +fi + +if test "x$enable_experimental" = "x1" ; then + $as_echo "#define JEMALLOC_EXPERIMENTAL " >>confdefs.h + + public_syms="${public_syms} allocm dallocm nallocm rallocm sallocm" +fi + + + +# Check whether --with-mangling was given. +if test "${with_mangling+set}" = set; then : + withval=$with_mangling; mangling_map="$with_mangling" +else + mangling_map="" +fi + +for nm in `echo ${mangling_map} |tr ',' ' '` ; do + k="`echo ${nm} |tr ':' ' ' |awk '{print $1}'`" + n="je_${k}" + m=`echo ${nm} |tr ':' ' ' |awk '{print $2}'` + cat >>confdefs.h <<_ACEOF +#define ${n} ${m} +_ACEOF + + public_syms=`for sym in ${public_syms}; do echo "${sym}"; done |grep -v "^${k}\$" |tr '\n' ' '` +done + # Check whether --with-jemalloc_prefix was given. if test "${with_jemalloc_prefix+set}" = set; then : withval=$with_jemalloc_prefix; JEMALLOC_PREFIX="$with_jemalloc_prefix" else - if test "x$abi" != "xmacho" ; then + if test "x$abi" != "xmacho" -a "x$abi" != "xpecoff"; then JEMALLOC_PREFIX="" else JEMALLOC_PREFIX="je_" @@ -4801,11 +5109,15 @@ _ACEOF #define JEMALLOC_CPREFIX "$JEMALLOC_CPREFIX" _ACEOF +fi +for stem in ${public_syms}; do + n="je_${stem}" + m="${JEMALLOC_PREFIX}${stem}" cat >>confdefs.h <<_ACEOF -#define JEMALLOC_P(string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix) ${JEMALLOC_PREFIX}##string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix +#define ${n} ${m} _ACEOF -fi +done # Check whether --with-private_namespace was given. @@ -4869,8 +5181,10 @@ cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in" cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/size_classes.sh" cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in" @@ -4888,7 +5202,7 @@ else fi if test "x$enable_cc_silence" = "x1" ; then - $as_echo "#define JEMALLOC_CC_SILENCE 1" >>confdefs.h + $as_echo "#define JEMALLOC_CC_SILENCE " >>confdefs.h fi @@ -4927,13 +5241,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -O3" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4947,7 +5255,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -4956,10 +5264,7 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -funroll-loops" >&5 @@ -4970,13 +5275,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -funroll-loops" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -4990,7 +5289,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -4999,11 +5298,43 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + + elif test "x$je_cv_msvc" = "xyes" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -O2" >&5 +$as_echo_n "checking whether compiler supports -O2... " >&6; } +TCFLAGS="${CFLAGS}" +if test "x${CFLAGS}" = "x" ; then + CFLAGS="-O2" +else + CFLAGS="${CFLAGS} -O2" fi +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +int +main () +{ + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + CFLAGS="${TCFLAGS}" + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + else { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether compiler supports -O" >&5 @@ -5014,13 +5345,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} -O" fi -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } -else - cat confdefs.h - <<_ACEOF >conftest.$ac_ext +cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5034,7 +5359,7 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : +if ac_fn_c_try_compile "$LINENO"; then : { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 $as_echo "yes" >&6; } else @@ -5043,10 +5368,7 @@ $as_echo "no" >&6; } CFLAGS="${TCFLAGS}" fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext fi fi @@ -5061,7 +5383,7 @@ else fi else - enable_stats="0" + enable_stats="1" fi @@ -5110,7 +5432,7 @@ if test "${with_static_libunwind+set}" = set; then : LUNWIND="-lunwind" else if test ! -f "$with_static_libunwind" ; then - as_fn_error "Static libunwind not found: $with_static_libunwind" "$LINENO" 5 + as_fn_error $? "Static libunwind not found: $with_static_libunwind" "$LINENO" 5 fi LUNWIND="$with_static_libunwind" fi @@ -5123,7 +5445,7 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then for ac_header in libunwind.h do : ac_fn_c_check_header_mongrel "$LINENO" "libunwind.h" "ac_cv_header_libunwind_h" "$ac_includes_default" -if test "x$ac_cv_header_libunwind_h" = x""yes; then : +if test "x$ac_cv_header_libunwind_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_LIBUNWIND_H 1 _ACEOF @@ -5137,7 +5459,7 @@ done if test "x$LUNWIND" = "x-lunwind" ; then { $as_echo "$as_me:${as_lineno-$LINENO}: checking for backtrace in -lunwind" >&5 $as_echo_n "checking for backtrace in -lunwind... " >&6; } -if test "${ac_cv_lib_unwind_backtrace+set}" = set; then : +if ${ac_cv_lib_unwind_backtrace+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5171,7 +5493,7 @@ LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_unwind_backtrace" >&5 $as_echo "$ac_cv_lib_unwind_backtrace" >&6; } -if test "x$ac_cv_lib_unwind_backtrace" = x""yes; then : +if test "x$ac_cv_lib_unwind_backtrace" = xyes; then : LIBS="$LIBS $LUNWIND" else enable_prof_libunwind="0" @@ -5205,7 +5527,7 @@ if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \ for ac_header in unwind.h do : ac_fn_c_check_header_mongrel "$LINENO" "unwind.h" "ac_cv_header_unwind_h" "$ac_includes_default" -if test "x$ac_cv_header_unwind_h" = x""yes; then : +if test "x$ac_cv_header_unwind_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_UNWIND_H 1 _ACEOF @@ -5218,7 +5540,7 @@ done { $as_echo "$as_me:${as_lineno-$LINENO}: checking for _Unwind_Backtrace in -lgcc" >&5 $as_echo_n "checking for _Unwind_Backtrace in -lgcc... " >&6; } -if test "${ac_cv_lib_gcc__Unwind_Backtrace+set}" = set; then : +if ${ac_cv_lib_gcc__Unwind_Backtrace+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5252,7 +5574,7 @@ LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_gcc__Unwind_Backtrace" >&5 $as_echo "$ac_cv_lib_gcc__Unwind_Backtrace" >&6; } -if test "x$ac_cv_lib_gcc__Unwind_Backtrace" = x""yes; then : +if test "x$ac_cv_lib_gcc__Unwind_Backtrace" = xyes; then : LIBS="$LIBS -lgcc" else enable_prof_libgcc="0" @@ -5316,65 +5638,106 @@ $as_echo_n "checking configured backtracing method... " >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: $backtrace_method" >&5 $as_echo "$backtrace_method" >&6; } if test "x$enable_prof" = "x1" ; then - LIBS="$LIBS -lm" + if test "x${force_tls}" = "x0" ; then + as_fn_error $? "Heap profiling requires TLS" "$LINENO" 5; + fi + force_tls="1" $as_echo "#define JEMALLOC_PROF " >>confdefs.h fi -# Check whether --enable-tiny was given. -if test "${enable_tiny+set}" = set; then : - enableval=$enable_tiny; if test "x$enable_tiny" = "xno" ; then - enable_tiny="0" +# Check whether --enable-tcache was given. +if test "${enable_tcache+set}" = set; then : + enableval=$enable_tcache; if test "x$enable_tcache" = "xno" ; then + enable_tcache="0" else - enable_tiny="1" + enable_tcache="1" fi else - enable_tiny="1" + enable_tcache="1" fi -if test "x$enable_tiny" = "x1" ; then - $as_echo "#define JEMALLOC_TINY " >>confdefs.h +if test "x$enable_tcache" = "x1" ; then + $as_echo "#define JEMALLOC_TCACHE " >>confdefs.h fi -# Check whether --enable-tcache was given. -if test "${enable_tcache+set}" = set; then : - enableval=$enable_tcache; if test "x$enable_tcache" = "xno" ; then - enable_tcache="0" +# Check whether --enable-mremap was given. +if test "${enable_mremap+set}" = set; then : + enableval=$enable_mremap; if test "x$enable_mremap" = "xno" ; then + enable_mremap="0" else - enable_tcache="1" + enable_mremap="1" fi else - enable_tcache="1" + enable_mremap="0" fi -if test "x$enable_tcache" = "x1" ; then - $as_echo "#define JEMALLOC_TCACHE " >>confdefs.h +if test "x$enable_mremap" = "x1" ; then + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether mremap(...MREMAP_FIXED...) is compilable" >&5 +$as_echo_n "checking whether mremap(...MREMAP_FIXED...) is compilable... " >&6; } +if ${je_cv_mremap_fixed+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define _GNU_SOURCE +#include + +int +main () +{ + +void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_mremap_fixed=yes +else + je_cv_mremap_fixed=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_mremap_fixed" >&5 +$as_echo "$je_cv_mremap_fixed" >&6; } + + if test "x${je_cv_mremap_fixed}" = "xno" ; then + enable_mremap="0" + fi +fi +if test "x$enable_mremap" = "x1" ; then + $as_echo "#define JEMALLOC_MREMAP " >>confdefs.h fi -# Check whether --enable-swap was given. -if test "${enable_swap+set}" = set; then : - enableval=$enable_swap; if test "x$enable_swap" = "xno" ; then - enable_swap="0" +# Check whether --enable-munmap was given. +if test "${enable_munmap+set}" = set; then : + enableval=$enable_munmap; if test "x$enable_munmap" = "xno" ; then + enable_munmap="0" else - enable_swap="1" + enable_munmap="1" fi else - enable_swap="0" + enable_munmap="${default_munmap}" fi -if test "x$enable_swap" = "x1" ; then - $as_echo "#define JEMALLOC_SWAP " >>confdefs.h +if test "x$enable_munmap" = "x1" ; then + $as_echo "#define JEMALLOC_MUNMAP " >>confdefs.h fi @@ -5392,6 +5755,20 @@ else fi +ac_fn_c_check_func "$LINENO" "sbrk" "ac_cv_func_sbrk" +if test "x$ac_cv_func_sbrk" = xyes; then : + have_sbrk="1" +else + have_sbrk="0" +fi + +if test "x$have_sbrk" = "x1" ; then + $as_echo "#define JEMALLOC_HAVE_SBRK " >>confdefs.h + +else + enable_dss="0" +fi + if test "x$enable_dss" = "x1" ; then $as_echo "#define JEMALLOC_DSS " >>confdefs.h @@ -5407,7 +5784,7 @@ else fi else - enable_fill="0" + enable_fill="1" fi @@ -5417,76 +5794,163 @@ if test "x$enable_fill" = "x1" ; then fi -# Check whether --enable-xmalloc was given. -if test "${enable_xmalloc+set}" = set; then : - enableval=$enable_xmalloc; if test "x$enable_xmalloc" = "xno" ; then - enable_xmalloc="0" +# Check whether --enable-utrace was given. +if test "${enable_utrace+set}" = set; then : + enableval=$enable_utrace; if test "x$enable_utrace" = "xno" ; then + enable_utrace="0" else - enable_xmalloc="1" + enable_utrace="1" fi else - enable_xmalloc="0" + enable_utrace="0" fi -if test "x$enable_xmalloc" = "x1" ; then - $as_echo "#define JEMALLOC_XMALLOC " >>confdefs.h -fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether utrace(2) is compilable" >&5 +$as_echo_n "checking whether utrace(2) is compilable... " >&6; } +if ${je_cv_utrace+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include +#include +#include +#include -# Check whether --enable-sysv was given. -if test "${enable_sysv+set}" = set; then : - enableval=$enable_sysv; if test "x$enable_sysv" = "xno" ; then - enable_sysv="0" -else - enable_sysv="1" -fi +int +main () +{ -else - enable_sysv="0" + utrace((void *)0, 0); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_utrace=yes +else + je_cv_utrace=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_utrace" >&5 +$as_echo "$je_cv_utrace" >&6; } -if test "x$enable_sysv" = "x1" ; then - $as_echo "#define JEMALLOC_SYSV " >>confdefs.h +if test "x${je_cv_utrace}" = "xno" ; then + enable_utrace="0" +fi +if test "x$enable_utrace" = "x1" ; then + $as_echo "#define JEMALLOC_UTRACE " >>confdefs.h fi -# Check whether --enable-dynamic_page_shift was given. -if test "${enable_dynamic_page_shift+set}" = set; then : - enableval=$enable_dynamic_page_shift; if test "x$enable_dynamic_page_shift" = "xno" ; then - enable_dynamic_page_shift="0" +# Check whether --enable-valgrind was given. +if test "${enable_valgrind+set}" = set; then : + enableval=$enable_valgrind; if test "x$enable_valgrind" = "xno" ; then + enable_valgrind="0" else - enable_dynamic_page_shift="1" + enable_valgrind="1" fi else - enable_dynamic_page_shift="0" + enable_valgrind="1" fi -if test "x$enable_dynamic_page_shift" = "x1" ; then - $as_echo "#define DYNAMIC_PAGE_SHIFT " >>confdefs.h +if test "x$enable_valgrind" = "x1" ; then -fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether valgrind is compilable" >&5 +$as_echo_n "checking whether valgrind is compilable... " >&6; } +if ${je_cv_valgrind+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +#include -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking STATIC_PAGE_SHIFT" >&5 +#if !defined(VALGRIND_RESIZEINPLACE_BLOCK) +# error "Incompatible Valgrind version" +#endif + +int +main () +{ + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_valgrind=yes +else + je_cv_valgrind=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_valgrind" >&5 +$as_echo "$je_cv_valgrind" >&6; } + + if test "x${je_cv_valgrind}" = "xno" ; then + enable_valgrind="0" + fi + if test "x$enable_valgrind" = "x1" ; then + $as_echo "#define JEMALLOC_VALGRIND " >>confdefs.h + + fi +fi + + +# Check whether --enable-xmalloc was given. +if test "${enable_xmalloc+set}" = set; then : + enableval=$enable_xmalloc; if test "x$enable_xmalloc" = "xno" ; then + enable_xmalloc="0" +else + enable_xmalloc="1" +fi + +else + enable_xmalloc="0" + +fi + +if test "x$enable_xmalloc" = "x1" ; then + $as_echo "#define JEMALLOC_XMALLOC " >>confdefs.h + +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking STATIC_PAGE_SHIFT" >&5 $as_echo_n "checking STATIC_PAGE_SHIFT... " >&6; } -if test "$cross_compiling" = yes; then : +if ${je_cv_static_page_shift+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 $as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include -#include + #include +#ifdef _WIN32 +#include +#else +#include +#endif +#include int main () @@ -5495,16 +5959,24 @@ main () long result; FILE *f; +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + result = si.dwPageSize; +#else result = sysconf(_SC_PAGESIZE); +#endif if (result == -1) { return 1; } + result = ffsl(result) - 1; + f = fopen("conftest.out", "w"); if (f == NULL) { return 1; } - fprintf(f, "%u\n", ffs((int)result) - 1); - close(f); + fprintf(f, "%u\n", result); + fclose(f); return 0; @@ -5513,21 +5985,26 @@ main () } _ACEOF if ac_fn_c_try_run "$LINENO"; then : - STATIC_PAGE_SHIFT=`cat conftest.out` - { $as_echo "$as_me:${as_lineno-$LINENO}: result: $STATIC_PAGE_SHIFT" >&5 -$as_echo "$STATIC_PAGE_SHIFT" >&6; } - cat >>confdefs.h <<_ACEOF -#define STATIC_PAGE_SHIFT $STATIC_PAGE_SHIFT -_ACEOF - + je_cv_static_page_shift=`cat conftest.out` else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: error" >&5 -$as_echo "error" >&6; } + je_cv_static_page_shift=undefined fi rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ conftest.$ac_objext conftest.beam conftest.$ac_ext fi +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_static_page_shift" >&5 +$as_echo "$je_cv_static_page_shift" >&6; } + +if test "x$je_cv_static_page_shift" != "xundefined"; then + cat >>confdefs.h <<_ACEOF +#define STATIC_PAGE_SHIFT $je_cv_static_page_shift +_ACEOF + +else + as_fn_error $? "cannot determine value for STATIC_PAGE_SHIFT" "$LINENO" 5 +fi if test -d "${srcroot}.git" ; then @@ -5547,23 +6024,24 @@ jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print $5}' -for ac_header in pthread.h +if test "x$abi" != "xpecoff" ; then + for ac_header in pthread.h do : ac_fn_c_check_header_mongrel "$LINENO" "pthread.h" "ac_cv_header_pthread_h" "$ac_includes_default" -if test "x$ac_cv_header_pthread_h" = x""yes; then : +if test "x$ac_cv_header_pthread_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_PTHREAD_H 1 _ACEOF else - as_fn_error "pthread.h is missing" "$LINENO" 5 + as_fn_error $? "pthread.h is missing" "$LINENO" 5 fi done -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lpthread" >&5 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for pthread_create in -lpthread" >&5 $as_echo_n "checking for pthread_create in -lpthread... " >&6; } -if test "${ac_cv_lib_pthread_pthread_create+set}" = set; then : +if ${ac_cv_lib_pthread_pthread_create+:} false; then : $as_echo_n "(cached) " >&6 else ac_check_lib_save_LIBS=$LIBS @@ -5597,15 +6075,100 @@ LIBS=$ac_check_lib_save_LIBS fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_pthread_pthread_create" >&5 $as_echo "$ac_cv_lib_pthread_pthread_create" >&6; } -if test "x$ac_cv_lib_pthread_pthread_create" = x""yes; then : +if test "x$ac_cv_lib_pthread_pthread_create" = xyes; then : LIBS="$LIBS -lpthread" else - as_fn_error "libpthread is missing" "$LINENO" 5 + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 +$as_echo_n "checking for library containing pthread_create... " >&6; } +if ${ac_cv_search_pthread_create+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (); +int +main () +{ +return pthread_create (); + ; + return 0; +} +_ACEOF +for ac_lib in '' ; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_pthread_create=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_pthread_create+:} false; then : + break +fi +done +if ${ac_cv_search_pthread_create+:} false; then : + +else + ac_cv_search_pthread_create=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 +$as_echo "$ac_cv_search_pthread_create" >&6; } +ac_res=$ac_cv_search_pthread_create +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" +else + as_fn_error $? "libpthread is missing" "$LINENO" 5 +fi + +fi + +fi CPPFLAGS="$CPPFLAGS -D_REENTRANT" +ac_fn_c_check_func "$LINENO" "_malloc_thread_cleanup" "ac_cv_func__malloc_thread_cleanup" +if test "x$ac_cv_func__malloc_thread_cleanup" = xyes; then : + have__malloc_thread_cleanup="1" +else + have__malloc_thread_cleanup="0" + +fi + +if test "x$have__malloc_thread_cleanup" = "x1" ; then + $as_echo "#define JEMALLOC_MALLOC_THREAD_CLEANUP " >>confdefs.h + + force_tls="1" +fi + +ac_fn_c_check_func "$LINENO" "_pthread_mutex_init_calloc_cb" "ac_cv_func__pthread_mutex_init_calloc_cb" +if test "x$ac_cv_func__pthread_mutex_init_calloc_cb" = xyes; then : + have__pthread_mutex_init_calloc_cb="1" +else + have__pthread_mutex_init_calloc_cb="0" + +fi + +if test "x$have__pthread_mutex_init_calloc_cb" = "x1" ; then + $as_echo "#define JEMALLOC_MUTEX_INIT_CB 1" >>confdefs.h + +fi + # Check whether --enable-lazy_lock was given. if test "${enable_lazy_lock+set}" = set; then : enableval=$enable_lazy_lock; if test "x$enable_lazy_lock" = "xno" ; then @@ -5615,193 +6178,378 @@ else fi else - enable_lazy_lock="1" + enable_lazy_lock="0" fi +if test "x$enable_lazy_lock" = "x0" -a "x${force_lazy_lock}" = "x1" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Forcing lazy-lock to avoid allocator/threading bootstrap issues" >&5 +$as_echo "Forcing lazy-lock to avoid allocator/threading bootstrap issues" >&6; } + enable_lazy_lock="1" +fi if test "x$enable_lazy_lock" = "x1" ; then - for ac_header in dlfcn.h + if test "x$abi" != "xpecoff" ; then + for ac_header in dlfcn.h do : ac_fn_c_check_header_mongrel "$LINENO" "dlfcn.h" "ac_cv_header_dlfcn_h" "$ac_includes_default" -if test "x$ac_cv_header_dlfcn_h" = x""yes; then : +if test "x$ac_cv_header_dlfcn_h" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE_DLFCN_H 1 _ACEOF -else - as_fn_error "dlfcn.h is missing" "$LINENO" 5 +else + as_fn_error $? "dlfcn.h is missing" "$LINENO" 5 +fi + +done + + ac_fn_c_check_func "$LINENO" "dlsym" "ac_cv_func_dlsym" +if test "x$ac_cv_func_dlsym" = xyes; then : + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlsym in -ldl" >&5 +$as_echo_n "checking for dlsym in -ldl... " >&6; } +if ${ac_cv_lib_dl_dlsym+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_check_lib_save_LIBS=$LIBS +LIBS="-ldl $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char dlsym (); +int +main () +{ +return dlsym (); + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + ac_cv_lib_dl_dlsym=yes +else + ac_cv_lib_dl_dlsym=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlsym" >&5 +$as_echo "$ac_cv_lib_dl_dlsym" >&6; } +if test "x$ac_cv_lib_dl_dlsym" = xyes; then : + LIBS="$LIBS -ldl" +else + as_fn_error $? "libdl is missing" "$LINENO" 5 +fi + + +fi + + fi + $as_echo "#define JEMALLOC_LAZY_LOCK " >>confdefs.h + +fi + + +# Check whether --enable-tls was given. +if test "${enable_tls+set}" = set; then : + enableval=$enable_tls; if test "x$enable_tls" = "xno" ; then + enable_tls="0" +else + enable_tls="1" +fi + +else + enable_tls="1" + +fi + +if test "x${enable_tls}" = "x0" -a "x${force_tls}" = "x1" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Forcing TLS to avoid allocator/threading bootstrap issues" >&5 +$as_echo "Forcing TLS to avoid allocator/threading bootstrap issues" >&6; } + enable_tls="1" +fi +if test "x${enable_tls}" = "x1" -a "x${force_tls}" = "x0" ; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: Forcing no TLS to avoid allocator/threading bootstrap issues" >&5 +$as_echo "Forcing no TLS to avoid allocator/threading bootstrap issues" >&6; } + enable_tls="0" +fi +if test "x${enable_tls}" = "x1" ; then +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for TLS" >&5 +$as_echo_n "checking for TLS... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + __thread int x; + +int +main () +{ + + x = 42; + + return 0; + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + enable_tls="0" +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi + +if test "x${enable_tls}" = "x1" ; then + cat >>confdefs.h <<_ACEOF +#define JEMALLOC_TLS +_ACEOF + +elif test "x${force_tls}" = "x1" ; then + as_fn_error $? "Failed to configure TLS, which is mandatory for correct function" "$LINENO" 5 +fi + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether a program using ffsl is compilable" >&5 +$as_echo_n "checking whether a program using ffsl is compilable... " >&6; } +if ${je_cv_function_ffsl+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include + +int +main () +{ + + { + int rv = ffsl(0x08); + } + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_function_ffsl=yes +else + je_cv_function_ffsl=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_function_ffsl" >&5 +$as_echo "$je_cv_function_ffsl" >&6; } + +if test "x${je_cv_function_ffsl}" != "xyes" ; then + as_fn_error $? "Cannot build without ffsl(3)" "$LINENO" 5 +fi + + + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether atomic(9) is compilable" >&5 +$as_echo_n "checking whether atomic(9) is compilable... " >&6; } +if ${je_cv_atomic9+:} false; then : + $as_echo_n "(cached) " >&6 +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#include +#include +#include + +int +main () +{ + + { + uint32_t x32 = 0; + volatile uint32_t *x32p = &x32; + atomic_fetchadd_32(x32p, 1); + } + { + unsigned long xlong = 0; + volatile unsigned long *xlongp = &xlong; + atomic_fetchadd_long(xlongp, 1); + } + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_link "$LINENO"; then : + je_cv_atomic9=yes +else + je_cv_atomic9=no +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_atomic9" >&5 +$as_echo "$je_cv_atomic9" >&6; } + +if test "x${je_cv_atomic9}" = "xyes" ; then + $as_echo "#define JEMALLOC_ATOMIC9 1" >>confdefs.h + fi -done - { $as_echo "$as_me:${as_lineno-$LINENO}: checking for dlopen in -ldl" >&5 -$as_echo_n "checking for dlopen in -ldl... " >&6; } -if test "${ac_cv_lib_dl_dlopen+set}" = set; then : + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSAtomic*() is compilable" >&5 +$as_echo_n "checking whether Darwin OSAtomic*() is compilable... " >&6; } +if ${je_cv_osatomic+:} false; then : $as_echo_n "(cached) " >&6 else - ac_check_lib_save_LIBS=$LIBS -LIBS="-ldl $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext + cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -/* Override any GCC internal prototype to avoid an error. - Use char because int might match the return type of a GCC - builtin and then its argument prototype would still apply. */ -#ifdef __cplusplus -extern "C" -#endif -char dlopen (); +#include +#include + int main () { -return dlopen (); + + { + int32_t x32 = 0; + volatile int32_t *x32p = &x32; + OSAtomicAdd32(1, x32p); + } + { + int64_t x64 = 0; + volatile int64_t *x64p = &x64; + OSAtomicAdd64(1, x64p); + } + ; return 0; } _ACEOF if ac_fn_c_try_link "$LINENO"; then : - ac_cv_lib_dl_dlopen=yes + je_cv_osatomic=yes else - ac_cv_lib_dl_dlopen=no + je_cv_osatomic=no fi rm -f core conftest.err conftest.$ac_objext \ conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_dl_dlopen" >&5 -$as_echo "$ac_cv_lib_dl_dlopen" >&6; } -if test "x$ac_cv_lib_dl_dlopen" = x""yes; then : - LIBS="$LIBS -ldl" -else - as_fn_error "libdl is missing" "$LINENO" 5 fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_osatomic" >&5 +$as_echo "$je_cv_osatomic" >&6; } - $as_echo "#define JEMALLOC_LAZY_LOCK " >>confdefs.h +if test "x${je_cv_osatomic}" = "xyes" ; then + $as_echo "#define JEMALLOC_OSATOMIC " >>confdefs.h fi -# Check whether --enable-tls was given. -if test "${enable_tls+set}" = set; then : - enableval=$enable_tls; if test "x$enable_tls" = "xno" ; then - enable_tls="0" -else - enable_tls="1" -fi -else - enable_tls="1" -fi +if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then -if test "x${enable_tls}" = "x1" ; then -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for TLS" >&5 -$as_echo_n "checking for TLS... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to force 32-bit __sync_{add,sub}_and_fetch()" >&5 +$as_echo_n "checking whether to force 32-bit __sync_{add,sub}_and_fetch()... " >&6; } +if ${je_cv_sync_compare_and_swap_4+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ - __thread int x; + #include int main () { - x = 42; - - return 0; + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 + { + uint32_t x32 = 0; + __sync_add_and_fetch(&x32, 42); + __sync_sub_and_fetch(&x32, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 is defined, no need to force + #endif ; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } +if ac_fn_c_try_link "$LINENO"; then : + je_cv_sync_compare_and_swap_4=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - enable_tls="0" -fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext -fi - -fi - -if test "x${enable_tls}" = "x0" ; then - cat >>confdefs.h <<_ACEOF -#define NO_TLS -_ACEOF - + je_cv_sync_compare_and_swap_4=no fi - - -ac_fn_c_check_func "$LINENO" "ffsl" "ac_cv_func_ffsl" -if test "x$ac_cv_func_ffsl" = x""yes; then : - -else - as_fn_error "Cannot build without ffsl(3)" "$LINENO" 5 +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sync_compare_and_swap_4" >&5 +$as_echo "$je_cv_sync_compare_and_swap_4" >&6; } + if test "x${je_cv_sync_compare_and_swap_4}" = "xyes" ; then + $as_echo "#define JE_FORCE_SYNC_COMPARE_AND_SWAP_4 " >>confdefs.h + fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSAtomic*() is compilable" >&5 -$as_echo_n "checking whether Darwin OSAtomic*() is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } + { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether to force 64-bit __sync_{add,sub}_and_fetch()" >&5 +$as_echo_n "checking whether to force 64-bit __sync_{add,sub}_and_fetch()... " >&6; } +if ${je_cv_sync_compare_and_swap_8+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include -#include + #include int main () { - { - int32_t x32 = 0; - volatile int32_t *x32p = &x32; - OSAtomicAdd32(1, x32p); - } - { - int64_t x64 = 0; - volatile int64_t *x64p = &x64; - OSAtomicAdd64(1, x64p); - } + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 + { + uint64_t x64 = 0; + __sync_add_and_fetch(&x64, 42); + __sync_sub_and_fetch(&x64, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 is defined, no need to force + #endif ; return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - osatomic="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_sync_compare_and_swap_8=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - osatomic="no" - + je_cv_sync_compare_and_swap_8=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_sync_compare_and_swap_8" >&5 +$as_echo "$je_cv_sync_compare_and_swap_8" >&6; } + if test "x${je_cv_sync_compare_and_swap_8}" = "xyes" ; then + $as_echo "#define JE_FORCE_SYNC_COMPARE_AND_SWAP_8 " >>confdefs.h -if test "x${osatomic}" = "xyes" ; then - $as_echo "#define JEMALLOC_OSATOMIC 1" >>confdefs.h + fi fi @@ -5809,11 +6557,8 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking whether Darwin OSSpin*() is compilable" >&5 $as_echo_n "checking whether Darwin OSSpin*() is compilable... " >&6; } -if test "$cross_compiling" = yes; then : - { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 -$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} -as_fn_error "cannot run test program while cross compiling -See \`config.log' for more details." "$LINENO" 5; } +if ${je_cv_osspin+:} false; then : + $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ @@ -5833,110 +6578,183 @@ main () return 0; } _ACEOF -if ac_fn_c_try_run "$LINENO"; then : - { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 -$as_echo "yes" >&6; } - osspin="yes" +if ac_fn_c_try_link "$LINENO"; then : + je_cv_osspin=yes else - { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 -$as_echo "no" >&6; } - osspin="no" - + je_cv_osspin=no fi -rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ - conftest.$ac_objext conftest.beam conftest.$ac_ext +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext conftest.$ac_ext fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $je_cv_osspin" >&5 +$as_echo "$je_cv_osspin" >&6; } - -if test "x${osspin}" = "xyes" ; then - $as_echo "#define JEMALLOC_OSSPIN 1" >>confdefs.h +if test "x${je_cv_osspin}" = "xyes" ; then + $as_echo "#define JEMALLOC_OSSPIN " >>confdefs.h fi -ac_fn_c_check_func "$LINENO" "memalign" "ac_cv_func_memalign" -if test "x$ac_cv_func_memalign" = x""yes; then : - $as_echo "#define JEMALLOC_OVERRIDE_MEMALIGN 1" >>confdefs.h +if test "x${abi}" = "xmacho" ; then + $as_echo "#define JEMALLOC_IVSALLOC " >>confdefs.h -fi + $as_echo "#define JEMALLOC_ZONE " >>confdefs.h -ac_fn_c_check_func "$LINENO" "valloc" "ac_cv_func_valloc" -if test "x$ac_cv_func_valloc" = x""yes; then : - $as_echo "#define JEMALLOC_OVERRIDE_VALLOC 1" >>confdefs.h -fi + { $as_echo "$as_me:${as_lineno-$LINENO}: checking malloc zone version" >&5 +$as_echo_n "checking malloc zone version... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 14 ? 1 : -1] -if test "x${abi}" = "xmacho" ; then - $as_echo "#define JEMALLOC_IVSALLOC 1" >>confdefs.h + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=3 +else - $as_echo "#define JEMALLOC_ZONE 1" >>confdefs.h + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 15 ? 1 : -1] + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=5 +else - { $as_echo "$as_me:${as_lineno-$LINENO}: checking malloc zone version" >&5 -$as_echo_n "checking malloc zone version... " >&6; } cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ -#include #include int main () { +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 16 ? 1 : -1] + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : - static malloc_zone_t zone; - static struct malloc_introspection_t zone_introspect; - - zone.size = NULL; - zone.malloc = NULL; - zone.calloc = NULL; - zone.valloc = NULL; - zone.free = NULL; - zone.realloc = NULL; - zone.destroy = NULL; - zone.zone_name = "jemalloc_zone"; - zone.batch_malloc = NULL; - zone.batch_free = NULL; - zone.introspect = &zone_introspect; - zone.version = 6; - zone.memalign = NULL; - zone.free_definite_size = NULL; - - zone_introspect.enumerator = NULL; - zone_introspect.good_size = NULL; - zone_introspect.check = NULL; - zone_introspect.print = NULL; - zone_introspect.log = NULL; - zone_introspect.force_lock = NULL; - zone_introspect.force_unlock = NULL; - zone_introspect.statistics = NULL; - zone_introspect.zone_locked = NULL; + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_introspection_t) == sizeof(void *) * 9 ? 1 : -1] ; return 0; } _ACEOF if ac_fn_c_try_compile "$LINENO"; then : - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_ZONE_VERSION 6 + JEMALLOC_ZONE_VERSION=6 +else + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_introspection_t) == sizeof(void *) * 13 ? 1 : -1] + + ; + return 0; +} _ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=7 +else + JEMALLOC_ZONE_VERSION= + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +else + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) == sizeof(void *) * 17 ? 1 : -1] - { $as_echo "$as_me:${as_lineno-$LINENO}: result: 6" >&5 -$as_echo "6" >&6; } + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=8 else - cat >>confdefs.h <<_ACEOF -#define JEMALLOC_ZONE_VERSION 3 + + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +static foo[sizeof(malloc_zone_t) > sizeof(void *) * 17 ? 1 : -1] + + ; + return 0; +} _ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + JEMALLOC_ZONE_VERSION=9 +else + JEMALLOC_ZONE_VERSION= - { $as_echo "$as_me:${as_lineno-$LINENO}: result: 3" >&5 -$as_echo "3" >&6; } fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + if test "x${JEMALLOC_ZONE_VERSION}" = "x"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: unsupported" >&5 +$as_echo "unsupported" >&6; } + as_fn_error $? "Unsupported malloc zone version" "$LINENO" 5 + fi + if test "${JEMALLOC_ZONE_VERSION}" = 9; then + JEMALLOC_ZONE_VERSION=8 + { $as_echo "$as_me:${as_lineno-$LINENO}: result: > 8" >&5 +$as_echo "> 8" >&6; } + else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JEMALLOC_ZONE_VERSION" >&5 +$as_echo "$JEMALLOC_ZONE_VERSION" >&6; } + fi + cat >>confdefs.h <<_ACEOF +#define JEMALLOC_ZONE_VERSION $JEMALLOC_ZONE_VERSION +_ACEOF + fi { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdbool.h that conforms to C99" >&5 $as_echo_n "checking for stdbool.h that conforms to C99... " >&6; } -if test "${ac_cv_header_stdbool_h+set}" = set; then : +if ${ac_cv_header_stdbool_h+:} false; then : $as_echo_n "(cached) " >&6 else cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -5968,7 +6786,7 @@ else char b[false == 0 ? 1 : -1]; char c[__bool_true_false_are_defined == 1 ? 1 : -1]; char d[(bool) 0.5 == true ? 1 : -1]; - bool e = &s; + /* See body of main program for 'e'. */ char f[(_Bool) 0.0 == false ? 1 : -1]; char g[true]; char h[sizeof (_Bool)]; @@ -5979,25 +6797,6 @@ else _Bool n[m]; char o[sizeof n == m * sizeof n[0] ? 1 : -1]; char p[-1 - (_Bool) 0 < 0 && -1 - (bool) 0 < 0 ? 1 : -1]; -# if defined __xlc__ || defined __GNUC__ - /* Catch a bug in IBM AIX xlc compiler version 6.0.0.0 - reported by James Lemley on 2005-10-05; see - http://lists.gnu.org/archive/html/bug-coreutils/2005-10/msg00086.html - This test is not quite right, since xlc is allowed to - reject this program, as the initializer for xlcbug is - not one of the forms that C requires support for. - However, doing the test right would require a runtime - test, and that would make cross-compilation harder. - Let us hope that IBM fixes the xlc bug, and also adds - support for this kind of constant expression. In the - meantime, this test will reject xlc, which is OK, since - our stdbool.h substitute should suffice. We also test - this with GCC, where it should work, to detect more - quickly whether someone messes up the test in the - future. */ - char digs[] = "0123456789"; - int xlcbug = 1 / (&(digs + 5)[-2 + (bool) 1] == &digs[4] ? 1 : -1); -# endif /* Catch a bug in an HP-UX C compiler. See http://gcc.gnu.org/ml/gcc-patches/2003-12/msg02303.html http://lists.gnu.org/archive/html/bug-coreutils/2005-11/msg00161.html @@ -6009,6 +6808,7 @@ int main () { + bool e = &s; *pq |= q; *pq |= ! q; /* Refer to every declared value, to avoid compiler optimizations. */ @@ -6029,7 +6829,7 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_header_stdbool_h" >&5 $as_echo "$ac_cv_header_stdbool_h" >&6; } ac_fn_c_check_type "$LINENO" "_Bool" "ac_cv_type__Bool" "$ac_includes_default" -if test "x$ac_cv_type__Bool" = x""yes; then : +if test "x$ac_cv_type__Bool" = xyes; then : cat >>confdefs.h <<_ACEOF #define HAVE__BOOL 1 @@ -6045,12 +6845,15 @@ $as_echo "#define HAVE_STDBOOL_H 1" >>confdefs.h fi +ac_config_commands="$ac_config_commands include/jemalloc/internal/size_classes.h" + + ac_config_headers="$ac_config_headers $cfghdrs_tup" -ac_config_files="$ac_config_files $cfgoutputs_tup config.stamp" +ac_config_files="$ac_config_files $cfgoutputs_tup config.stamp bin/jemalloc.sh" @@ -6118,10 +6921,21 @@ $as_echo "$as_me: WARNING: cache variable $ac_var contains a newline" >&2;} ;; :end' >>confcache if diff "$cache_file" confcache >/dev/null 2>&1; then :; else if test -w "$cache_file"; then - test "x$cache_file" != "x/dev/null" && + if test "x$cache_file" != "x/dev/null"; then { $as_echo "$as_me:${as_lineno-$LINENO}: updating cache $cache_file" >&5 $as_echo "$as_me: updating cache $cache_file" >&6;} - cat confcache >$cache_file + if test ! -f "$cache_file" || test -h "$cache_file"; then + cat confcache >"$cache_file" + else + case $cache_file in #( + */* | ?:*) + mv -f confcache "$cache_file"$$ && + mv -f "$cache_file"$$ "$cache_file" ;; #( + *) + mv -f confcache "$cache_file" ;; + esac + fi + fi else { $as_echo "$as_me:${as_lineno-$LINENO}: not updating unwritable cache $cache_file" >&5 $as_echo "$as_me: not updating unwritable cache $cache_file" >&6;} @@ -6137,6 +6951,7 @@ DEFS=-DHAVE_CONFIG_H ac_libobjs= ac_ltlibobjs= +U= for ac_i in : $LIBOBJS; do test "x$ac_i" = x: && continue # 1. Remove the extension, and $U if already installed. ac_script='s/\$U\././;s/\.o$//;s/\.obj$//' @@ -6152,7 +6967,7 @@ LTLIBOBJS=$ac_ltlibobjs -: ${CONFIG_STATUS=./config.status} +: "${CONFIG_STATUS=./config.status}" ac_write_fail=0 ac_clean_files_save=$ac_clean_files ac_clean_files="$ac_clean_files $CONFIG_STATUS" @@ -6253,6 +7068,7 @@ fi IFS=" "" $as_nl" # Find who we are. Look in the path if we contain no directory separator. +as_myself= case $0 in #(( *[\\/]* ) as_myself=$0 ;; *) as_save_IFS=$IFS; IFS=$PATH_SEPARATOR @@ -6298,19 +7114,19 @@ export LANGUAGE (unset CDPATH) >/dev/null 2>&1 && unset CDPATH -# as_fn_error ERROR [LINENO LOG_FD] -# --------------------------------- +# as_fn_error STATUS ERROR [LINENO LOG_FD] +# ---------------------------------------- # Output "`basename $0`: error: ERROR" to stderr. If LINENO and LOG_FD are # provided, also output the error to LOG_FD, referencing LINENO. Then exit the -# script with status $?, using 1 if that was 0. +# script with STATUS, using 1 if that was 0. as_fn_error () { - as_status=$?; test $as_status -eq 0 && as_status=1 - if test "$3"; then - as_lineno=${as_lineno-"$2"} as_lineno_stack=as_lineno_stack=$as_lineno_stack - $as_echo "$as_me:${as_lineno-$LINENO}: error: $1" >&$3 + as_status=$1; test $as_status -eq 0 && as_status=1 + if test "$4"; then + as_lineno=${as_lineno-"$3"} as_lineno_stack=as_lineno_stack=$as_lineno_stack + $as_echo "$as_me:${as_lineno-$LINENO}: error: $2" >&$4 fi - $as_echo "$as_me: error: $1" >&2 + $as_echo "$as_me: error: $2" >&2 as_fn_exit $as_status } # as_fn_error @@ -6506,7 +7322,7 @@ $as_echo X"$as_dir" | test -d "$as_dir" && break done test -z "$as_dirs" || eval "mkdir $as_dirs" - } || test -d "$as_dir" || as_fn_error "cannot create directory $as_dir" + } || test -d "$as_dir" || as_fn_error $? "cannot create directory $as_dir" } # as_fn_mkdir_p @@ -6560,7 +7376,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # values after options handling. ac_log=" This file was extended by $as_me, which was -generated by GNU Autoconf 2.65. Invocation command line was +generated by GNU Autoconf 2.68. Invocation command line was CONFIG_FILES = $CONFIG_FILES CONFIG_HEADERS = $CONFIG_HEADERS @@ -6586,6 +7402,7 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 # Files that config.status was made for. config_files="$ac_config_files" config_headers="$ac_config_headers" +config_commands="$ac_config_commands" _ACEOF @@ -6615,6 +7432,9 @@ $config_files Configuration headers: $config_headers +Configuration commands: +$config_commands + Report bugs to the package provider." _ACEOF @@ -6622,10 +7442,10 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ config.status -configured by $0, generated by GNU Autoconf 2.65, +configured by $0, generated by GNU Autoconf 2.68, with options \\"\$ac_cs_config\\" -Copyright (C) 2009 Free Software Foundation, Inc. +Copyright (C) 2010 Free Software Foundation, Inc. This config.status script is free software; the Free Software Foundation gives unlimited permission to copy, distribute and modify it." @@ -6641,11 +7461,16 @@ ac_need_defaults=: while test $# != 0 do case $1 in - --*=*) + --*=?*) ac_option=`expr "X$1" : 'X\([^=]*\)='` ac_optarg=`expr "X$1" : 'X[^=]*=\(.*\)'` ac_shift=: ;; + --*=) + ac_option=`expr "X$1" : 'X\([^=]*\)='` + ac_optarg= + ac_shift=: + ;; *) ac_option=$1 ac_optarg=$2 @@ -6667,6 +7492,7 @@ do $ac_shift case $ac_optarg in *\'*) ac_optarg=`$as_echo "$ac_optarg" | sed "s/'/'\\\\\\\\''/g"` ;; + '') as_fn_error $? "missing file argument" ;; esac as_fn_append CONFIG_FILES " '$ac_optarg'" ac_need_defaults=false;; @@ -6679,7 +7505,7 @@ do ac_need_defaults=false;; --he | --h) # Conflict between --help and --header - as_fn_error "ambiguous option: \`$1' + as_fn_error $? "ambiguous option: \`$1' Try \`$0 --help' for more information.";; --help | --hel | -h ) $as_echo "$ac_cs_usage"; exit ;; @@ -6688,7 +7514,7 @@ Try \`$0 --help' for more information.";; ac_cs_silent=: ;; # This is an error. - -*) as_fn_error "unrecognized option: \`$1' + -*) as_fn_error $? "unrecognized option: \`$1' Try \`$0 --help' for more information." ;; *) as_fn_append ac_config_targets " $1" @@ -6737,11 +7563,13 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 for ac_config_target in $ac_config_targets do case $ac_config_target in + "include/jemalloc/internal/size_classes.h") CONFIG_COMMANDS="$CONFIG_COMMANDS include/jemalloc/internal/size_classes.h" ;; "$cfghdrs_tup") CONFIG_HEADERS="$CONFIG_HEADERS $cfghdrs_tup" ;; "$cfgoutputs_tup") CONFIG_FILES="$CONFIG_FILES $cfgoutputs_tup" ;; "config.stamp") CONFIG_FILES="$CONFIG_FILES config.stamp" ;; + "bin/jemalloc.sh") CONFIG_FILES="$CONFIG_FILES bin/jemalloc.sh" ;; - *) as_fn_error "invalid argument: \`$ac_config_target'" "$LINENO" 5;; + *) as_fn_error $? "invalid argument: \`$ac_config_target'" "$LINENO" 5;; esac done @@ -6753,6 +7581,7 @@ done if $ac_need_defaults; then test "${CONFIG_FILES+set}" = set || CONFIG_FILES=$config_files test "${CONFIG_HEADERS+set}" = set || CONFIG_HEADERS=$config_headers + test "${CONFIG_COMMANDS+set}" = set || CONFIG_COMMANDS=$config_commands fi # Have a temporary directory for convenience. Make it in the build tree @@ -6763,9 +7592,10 @@ fi # after its creation but before its name has been assigned to `$tmp'. $debug || { - tmp= + tmp= ac_tmp= trap 'exit_status=$? - { test -z "$tmp" || test ! -d "$tmp" || rm -fr "$tmp"; } && exit $exit_status + : "${ac_tmp:=$tmp}" + { test ! -d "$ac_tmp" || rm -fr "$ac_tmp"; } && exit $exit_status ' 0 trap 'as_fn_exit 1' 1 2 13 15 } @@ -6773,12 +7603,13 @@ $debug || { tmp=`(umask 077 && mktemp -d "./confXXXXXX") 2>/dev/null` && - test -n "$tmp" && test -d "$tmp" + test -d "$tmp" } || { tmp=./conf$$-$RANDOM (umask 077 && mkdir "$tmp") -} || as_fn_error "cannot create a temporary directory in ." "$LINENO" 5 +} || as_fn_error $? "cannot create a temporary directory in ." "$LINENO" 5 +ac_tmp=$tmp # Set up the scripts for CONFIG_FILES section. # No need to generate them if there are no CONFIG_FILES. @@ -6795,12 +7626,12 @@ if test "x$ac_cr" = x; then fi ac_cs_awk_cr=`$AWK 'BEGIN { print "a\rb" }' /dev/null` if test "$ac_cs_awk_cr" = "a${ac_cr}b"; then - ac_cs_awk_cr='\r' + ac_cs_awk_cr='\\r' else ac_cs_awk_cr=$ac_cr fi -echo 'BEGIN {' >"$tmp/subs1.awk" && +echo 'BEGIN {' >"$ac_tmp/subs1.awk" && _ACEOF @@ -6809,18 +7640,18 @@ _ACEOF echo "$ac_subst_vars" | sed 's/.*/&!$&$ac_delim/' && echo "_ACEOF" } >conf$$subs.sh || - as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 -ac_delim_num=`echo "$ac_subst_vars" | grep -c '$'` + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 +ac_delim_num=`echo "$ac_subst_vars" | grep -c '^'` ac_delim='%!_!# ' for ac_last_try in false false false false false :; do . ./conf$$subs.sh || - as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 ac_delim_n=`sed -n "s/.*$ac_delim\$/X/p" conf$$subs.awk | grep -c X` if test $ac_delim_n = $ac_delim_num; then break elif $ac_last_try; then - as_fn_error "could not make $CONFIG_STATUS" "$LINENO" 5 + as_fn_error $? "could not make $CONFIG_STATUS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi @@ -6828,7 +7659,7 @@ done rm -f conf$$subs.sh cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 -cat >>"\$tmp/subs1.awk" <<\\_ACAWK && +cat >>"\$ac_tmp/subs1.awk" <<\\_ACAWK && _ACEOF sed -n ' h @@ -6876,7 +7707,7 @@ t delim rm -f conf$$subs.awk cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK -cat >>"\$tmp/subs1.awk" <<_ACAWK && +cat >>"\$ac_tmp/subs1.awk" <<_ACAWK && for (key in S) S_is_set[key] = 1 FS = "" @@ -6908,21 +7739,29 @@ if sed "s/$ac_cr//" < /dev/null > /dev/null 2>&1; then sed "s/$ac_cr\$//; s/$ac_cr/$ac_cs_awk_cr/g" else cat -fi < "$tmp/subs1.awk" > "$tmp/subs.awk" \ - || as_fn_error "could not setup config files machinery" "$LINENO" 5 +fi < "$ac_tmp/subs1.awk" > "$ac_tmp/subs.awk" \ + || as_fn_error $? "could not setup config files machinery" "$LINENO" 5 _ACEOF -# VPATH may cause trouble with some makes, so we remove $(srcdir), -# ${srcdir} and @srcdir@ from VPATH if srcdir is ".", strip leading and +# VPATH may cause trouble with some makes, so we remove sole $(srcdir), +# ${srcdir} and @srcdir@ entries from VPATH if srcdir is ".", strip leading and # trailing colons and then remove the whole line if VPATH becomes empty # (actually we leave an empty line to preserve line numbers). if test "x$srcdir" = x.; then - ac_vpsub='/^[ ]*VPATH[ ]*=/{ -s/:*\$(srcdir):*/:/ -s/:*\${srcdir}:*/:/ -s/:*@srcdir@:*/:/ -s/^\([^=]*=[ ]*\):*/\1/ + ac_vpsub='/^[ ]*VPATH[ ]*=[ ]*/{ +h +s/// +s/^/:/ +s/[ ]*$/:/ +s/:\$(srcdir):/:/g +s/:\${srcdir}:/:/g +s/:@srcdir@:/:/g +s/^:*// s/:*$// +x +s/\(=[ ]*\).*/\1/ +G +s/\n// s/^[^=]*=[ ]*$// }' fi @@ -6934,7 +7773,7 @@ fi # test -n "$CONFIG_FILES" # No need to generate them if there are no CONFIG_HEADERS. # This happens for instance with `./config.status Makefile'. if test -n "$CONFIG_HEADERS"; then -cat >"$tmp/defines.awk" <<\_ACAWK || +cat >"$ac_tmp/defines.awk" <<\_ACAWK || BEGIN { _ACEOF @@ -6946,11 +7785,11 @@ _ACEOF # handling of long lines. ac_delim='%!_!# ' for ac_last_try in false false :; do - ac_t=`sed -n "/$ac_delim/p" confdefs.h` - if test -z "$ac_t"; then + ac_tt=`sed -n "/$ac_delim/p" confdefs.h` + if test -z "$ac_tt"; then break elif $ac_last_try; then - as_fn_error "could not make $CONFIG_HEADERS" "$LINENO" 5 + as_fn_error $? "could not make $CONFIG_HEADERS" "$LINENO" 5 else ac_delim="$ac_delim!$ac_delim _$ac_delim!! " fi @@ -7035,11 +7874,11 @@ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 _ACAWK _ACEOF cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 - as_fn_error "could not setup config headers machinery" "$LINENO" 5 + as_fn_error $? "could not setup config headers machinery" "$LINENO" 5 fi # test -n "$CONFIG_HEADERS" -eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS " +eval set X " :F $CONFIG_FILES :H $CONFIG_HEADERS :C $CONFIG_COMMANDS" shift for ac_tag do @@ -7048,7 +7887,7 @@ do esac case $ac_mode$ac_tag in :[FHL]*:*);; - :L* | :C*:*) as_fn_error "invalid tag \`$ac_tag'" "$LINENO" 5;; + :L* | :C*:*) as_fn_error $? "invalid tag \`$ac_tag'" "$LINENO" 5;; :[FH]-) ac_tag=-:-;; :[FH]*) ac_tag=$ac_tag:$ac_tag.in;; esac @@ -7067,7 +7906,7 @@ do for ac_f do case $ac_f in - -) ac_f="$tmp/stdin";; + -) ac_f="$ac_tmp/stdin";; *) # Look for the file first in the build tree, then in the source tree # (if the path is not absolute). The absolute path cannot be DOS-style, # because $ac_f cannot contain `:'. @@ -7076,7 +7915,7 @@ do [\\/$]*) false;; *) test -f "$srcdir/$ac_f" && ac_f="$srcdir/$ac_f";; esac || - as_fn_error "cannot find input file: \`$ac_f'" "$LINENO" 5;; + as_fn_error 1 "cannot find input file: \`$ac_f'" "$LINENO" 5;; esac case $ac_f in *\'*) ac_f=`$as_echo "$ac_f" | sed "s/'/'\\\\\\\\''/g"`;; esac as_fn_append ac_file_inputs " '$ac_f'" @@ -7102,8 +7941,8 @@ $as_echo "$as_me: creating $ac_file" >&6;} esac case $ac_tag in - *:-:* | *:-) cat >"$tmp/stdin" \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 ;; + *:-:* | *:-) cat >"$ac_tmp/stdin" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; esac ;; esac @@ -7233,23 +8072,24 @@ s&@abs_top_builddir@&$ac_abs_top_builddir&;t t s&@INSTALL@&$ac_INSTALL&;t t $ac_datarootdir_hack " -eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$tmp/subs.awk" >$tmp/out \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 +eval sed \"\$ac_sed_extra\" "$ac_file_inputs" | $AWK -f "$ac_tmp/subs.awk" \ + >$ac_tmp/out || as_fn_error $? "could not create $ac_file" "$LINENO" 5 test -z "$ac_datarootdir_hack$ac_datarootdir_seen" && - { ac_out=`sed -n '/\${datarootdir}/p' "$tmp/out"`; test -n "$ac_out"; } && - { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' "$tmp/out"`; test -z "$ac_out"; } && + { ac_out=`sed -n '/\${datarootdir}/p' "$ac_tmp/out"`; test -n "$ac_out"; } && + { ac_out=`sed -n '/^[ ]*datarootdir[ ]*:*=/p' \ + "$ac_tmp/out"`; test -z "$ac_out"; } && { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined." >&5 +which seems to be undefined. Please make sure it is defined" >&5 $as_echo "$as_me: WARNING: $ac_file contains a reference to the variable \`datarootdir' -which seems to be undefined. Please make sure it is defined." >&2;} +which seems to be undefined. Please make sure it is defined" >&2;} - rm -f "$tmp/stdin" + rm -f "$ac_tmp/stdin" case $ac_file in - -) cat "$tmp/out" && rm -f "$tmp/out";; - *) rm -f "$ac_file" && mv "$tmp/out" "$ac_file";; + -) cat "$ac_tmp/out" && rm -f "$ac_tmp/out";; + *) rm -f "$ac_file" && mv "$ac_tmp/out" "$ac_file";; esac \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 ;; :H) # @@ -7258,27 +8098,37 @@ which seems to be undefined. Please make sure it is defined." >&2;} if test x"$ac_file" != x-; then { $as_echo "/* $configure_input */" \ - && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" - } >"$tmp/config.h" \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 - if diff "$ac_file" "$tmp/config.h" >/dev/null 2>&1; then + && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" + } >"$ac_tmp/config.h" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 + if diff "$ac_file" "$ac_tmp/config.h" >/dev/null 2>&1; then { $as_echo "$as_me:${as_lineno-$LINENO}: $ac_file is unchanged" >&5 $as_echo "$as_me: $ac_file is unchanged" >&6;} else rm -f "$ac_file" - mv "$tmp/config.h" "$ac_file" \ - || as_fn_error "could not create $ac_file" "$LINENO" 5 + mv "$ac_tmp/config.h" "$ac_file" \ + || as_fn_error $? "could not create $ac_file" "$LINENO" 5 fi else $as_echo "/* $configure_input */" \ - && eval '$AWK -f "$tmp/defines.awk"' "$ac_file_inputs" \ - || as_fn_error "could not create -" "$LINENO" 5 + && eval '$AWK -f "$ac_tmp/defines.awk"' "$ac_file_inputs" \ + || as_fn_error $? "could not create -" "$LINENO" 5 fi ;; - + :C) { $as_echo "$as_me:${as_lineno-$LINENO}: executing $ac_file commands" >&5 +$as_echo "$as_me: executing $ac_file commands" >&6;} + ;; esac + + case $ac_file$ac_mode in + "include/jemalloc/internal/size_classes.h":C) + mkdir -p "include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/size_classes.sh" > "${objroot}include/jemalloc/internal/size_classes.h" + ;; + + esac done # for ac_tag @@ -7287,7 +8137,7 @@ _ACEOF ac_clean_files=$ac_clean_files_save test $ac_write_fail = 0 || - as_fn_error "write failure creating $CONFIG_STATUS" "$LINENO" 5 + as_fn_error $? "write failure creating $CONFIG_STATUS" "$LINENO" 5 # configure is writing to config.log, and then calls config.status. @@ -7308,7 +8158,7 @@ if test "$no_create" != yes; then exec 5>>config.log # Use ||, not &&, to avoid exiting from the if with $? = 1, which # would make configure fail if this is the last instruction. - $ac_cs_success || as_fn_exit $? + $ac_cs_success || as_fn_exit 1 fi if test -n "$ac_unrecognized_opts" && test "$enable_option_checking" != no; then { $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: unrecognized options: $ac_unrecognized_opts" >&5 @@ -7318,8 +8168,10 @@ fi { $as_echo "$as_me:${as_lineno-$LINENO}: result: ===============================================================================" >&5 $as_echo "===============================================================================" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: jemalloc version : $jemalloc_version" >&5 -$as_echo "jemalloc version : $jemalloc_version" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: jemalloc version : ${jemalloc_version}" >&5 +$as_echo "jemalloc version : ${jemalloc_version}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: library revision : ${rev}" >&5 +$as_echo "library revision : ${rev}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: " >&5 $as_echo "" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: CC : ${CC}" >&5 @@ -7376,6 +8228,8 @@ $as_echo " : ${JEMALLOC_PRIVATE_NAMESPACE}" >&6; } $as_echo "install_suffix : ${install_suffix}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: autogen : ${enable_autogen}" >&5 $as_echo "autogen : ${enable_autogen}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: experimental : ${enable_experimental}" >&5 +$as_echo "experimental : ${enable_experimental}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: cc-silence : ${enable_cc_silence}" >&5 $as_echo "cc-silence : ${enable_cc_silence}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: debug : ${enable_debug}" >&5 @@ -7390,22 +8244,22 @@ $as_echo "prof-libunwind : ${enable_prof_libunwind}" >&6; } $as_echo "prof-libgcc : ${enable_prof_libgcc}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: prof-gcc : ${enable_prof_gcc}" >&5 $as_echo "prof-gcc : ${enable_prof_gcc}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: tiny : ${enable_tiny}" >&5 -$as_echo "tiny : ${enable_tiny}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: tcache : ${enable_tcache}" >&5 $as_echo "tcache : ${enable_tcache}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: fill : ${enable_fill}" >&5 $as_echo "fill : ${enable_fill}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: utrace : ${enable_utrace}" >&5 +$as_echo "utrace : ${enable_utrace}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: valgrind : ${enable_valgrind}" >&5 +$as_echo "valgrind : ${enable_valgrind}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: xmalloc : ${enable_xmalloc}" >&5 $as_echo "xmalloc : ${enable_xmalloc}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: sysv : ${enable_sysv}" >&5 -$as_echo "sysv : ${enable_sysv}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: swap : ${enable_swap}" >&5 -$as_echo "swap : ${enable_swap}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: mremap : ${enable_mremap}" >&5 +$as_echo "mremap : ${enable_mremap}" >&6; } +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: munmap : ${enable_munmap}" >&5 +$as_echo "munmap : ${enable_munmap}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: dss : ${enable_dss}" >&5 $as_echo "dss : ${enable_dss}" >&6; } -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: dynamic_page_shift : ${enable_dynamic_page_shift}" >&5 -$as_echo "dynamic_page_shift : ${enable_dynamic_page_shift}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: lazy_lock : ${enable_lazy_lock}" >&5 $as_echo "lazy_lock : ${enable_lazy_lock}" >&6; } { $as_echo "$as_me:${as_lineno-$LINENO}: result: tls : ${enable_tls}" >&5 diff --git a/deps/jemalloc/configure.ac b/deps/jemalloc/configure.ac index b58aa520998..a72019e5ed1 100644 --- a/deps/jemalloc/configure.ac +++ b/deps/jemalloc/configure.ac @@ -14,7 +14,7 @@ if test "x${CFLAGS}" = "x" ; then else CFLAGS="${CFLAGS} $1" fi -AC_RUN_IFELSE([AC_LANG_PROGRAM( +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [[ ]], [[ return 0; @@ -26,20 +26,25 @@ AC_RUN_IFELSE([AC_LANG_PROGRAM( ]) dnl JE_COMPILABLE(label, hcode, mcode, rvar) +dnl +dnl Use AC_LINK_IFELSE() rather than AC_COMPILE_IFELSE() so that linker errors +dnl cause failure. AC_DEFUN([JE_COMPILABLE], [ -AC_MSG_CHECKING([whether $1 is compilable]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[$2], [$3])], - AC_MSG_RESULT([yes]) - [$4="yes"], - AC_MSG_RESULT([no]) - [$4="no"] -) +AC_CACHE_CHECK([whether $1 is compilable], + [$4], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([$2], + [$3])], + [$4=yes], + [$4=no])]) ]) dnl ============================================================================ +dnl Library revision. +rev=1 +AC_SUBST([rev]) + srcroot=$srcdir if test "x${srcroot}" = "x." ; then srcroot="" @@ -82,14 +87,23 @@ AC_SUBST([MANDIR]) dnl Support for building documentation. AC_PATH_PROG([XSLTPROC], [xsltproc], , [$PATH]) +if test -d "/usr/share/xml/docbook/stylesheet/docbook-xsl" ; then + DEFAULT_XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +elif test -d "/usr/share/sgml/docbook/xsl-stylesheets" ; then + DEFAULT_XSLROOT="/usr/share/sgml/docbook/xsl-stylesheets" +else + dnl Documentation building will fail if this default gets used. + DEFAULT_XSLROOT="" +fi AC_ARG_WITH([xslroot], - [AS_HELP_STRING([--with-xslroot=], [XSL stylesheet root path])], + [AS_HELP_STRING([--with-xslroot=], [XSL stylesheet root path])], [ if test "x$with_xslroot" = "xno" ; then - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" + XSLROOT="${DEFAULT_XSLROOT}" else XSLROOT="${with_xslroot}" -fi, - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" +fi +], + XSLROOT="${DEFAULT_XSLROOT}" ) AC_SUBST([XSLROOT]) @@ -97,6 +111,19 @@ dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, dnl just prevent autoconf from molesting CFLAGS. CFLAGS=$CFLAGS AC_PROG_CC +if test "x$GCC" != "xyes" ; then + AC_CACHE_CHECK([whether compiler is MSVC], + [je_cv_msvc], + [AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], + [ +#ifndef _MSC_VER + int fail[-1]; +#endif +])], + [je_cv_msvc=yes], + [je_cv_msvc=no])]) +fi + if test "x$CFLAGS" = "x" ; then no_CFLAGS="yes" if test "x$GCC" = "xyes" ; then @@ -104,6 +131,12 @@ if test "x$CFLAGS" = "x" ; then JE_CFLAGS_APPEND([-Wall]) JE_CFLAGS_APPEND([-pipe]) JE_CFLAGS_APPEND([-g3]) + elif test "x$je_cv_msvc" = "xyes" ; then + CC="$CC -nologo" + JE_CFLAGS_APPEND([-Zi]) + JE_CFLAGS_APPEND([-MT]) + JE_CFLAGS_APPEND([-W3]) + CPPFLAGS="$CPPFLAGS -I${srcroot}/include/msvc_compat" fi fi dnl Append EXTRA_CFLAGS to CFLAGS, if defined. @@ -142,6 +175,18 @@ else fi AC_DEFINE_UNQUOTED([LG_SIZEOF_LONG], [$LG_SIZEOF_LONG]) +AC_CHECK_SIZEOF([intmax_t]) +if test "x${ac_cv_sizeof_intmax_t}" = "x16" ; then + LG_SIZEOF_INTMAX_T=4 +elif test "x${ac_cv_sizeof_intmax_t}" = "x8" ; then + LG_SIZEOF_INTMAX_T=3 +elif test "x${ac_cv_sizeof_intmax_t}" = "x4" ; then + LG_SIZEOF_INTMAX_T=2 +else + AC_MSG_ERROR([Unsupported intmax_t size: ${ac_cv_sizeof_intmax_t}]) +fi +AC_DEFINE_UNQUOTED([LG_SIZEOF_INTMAX_T], [$LG_SIZEOF_INTMAX_T]) + AC_CANONICAL_HOST dnl CPU-specific settings. CPU_SPINWAIT="" @@ -150,15 +195,15 @@ case "${host_cpu}" in ;; i686) JE_COMPILABLE([__asm__], [], [[__asm__ volatile("pause"); return 0;]], - [asm]) - if test "x${asm}" = "xyes" ; then + [je_cv_asm]) + if test "x${je_cv_asm}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi ;; x86_64) JE_COMPILABLE([__asm__ syntax], [], - [[__asm__ volatile("pause"); return 0;]], [asm]) - if test "x${asm}" = "xyes" ; then + [[__asm__ volatile("pause"); return 0;]], [je_cv_asm]) + if test "x${je_cv_asm}" = "xyes" ; then CPU_SPINWAIT='__asm__ volatile("pause")' fi ;; @@ -167,6 +212,23 @@ case "${host_cpu}" in esac AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT]) +LD_PRELOAD_VAR="LD_PRELOAD" +so="so" +importlib="${so}" +o="$ac_objext" +a="a" +exe="$ac_exeext" +libprefix="lib" +DSO_LDFLAGS='-shared -Wl,-soname,$(@F)' +RPATH='-Wl,-rpath,$(1)' +SOREV="${so}.${rev}" +PIC_CFLAGS='-fPIC -DPIC' +CTARGET='-o $@' +LDTARGET='-o $@' +EXTRA_LDFLAGS= +MKLIB='ar crus $@' +CC_MM=1 + dnl Platform-specific settings. abi and RPATH can probably be determined dnl programmatically, but doing so is error-prone, which makes it generally dnl not worth the trouble. @@ -174,25 +236,33 @@ dnl dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the dnl definitions need to be seen before any headers are included, which is a pain dnl to make happen otherwise. +default_munmap="1" case "${host}" in *-*-darwin*) - CFLAGS="$CFLAGS -fno-common -no-cpp-precomp" + CFLAGS="$CFLAGS" abi="macho" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) RPATH="" + LD_PRELOAD_VAR="DYLD_INSERT_LIBRARIES" + so="dylib" + importlib="${so}" + force_tls="0" + DSO_LDFLAGS='-shared -Wl,-dylib_install_name,$(@F)' + SOREV="${rev}.${so}" ;; *-*-freebsd*) CFLAGS="$CFLAGS" abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="-Wl,-rpath," + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + force_lazy_lock="1" ;; *-*-linux*) CFLAGS="$CFLAGS" CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED]) - RPATH="-Wl,-rpath," + AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ]) + AC_DEFINE([JEMALLOC_THREADED_INIT], [ ]) + default_munmap="0" ;; *-*-netbsd*) AC_MSG_CHECKING([ABI]) @@ -206,45 +276,100 @@ case "${host}" in [CFLAGS="$CFLAGS"; abi="elf"], [abi="aout"]) AC_MSG_RESULT([$abi]) - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="-Wl,-rpath," + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) ;; *-*-solaris2*) CFLAGS="$CFLAGS" abi="elf" - RPATH="-Wl,-R," + RPATH='-Wl,-R,$(1)' dnl Solaris needs this for sigwait(). CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" LIBS="$LIBS -lposix4 -lsocket -lnsl" ;; + *-ibm-aix*) + if "$LG_SIZEOF_PTR" = "8"; then + dnl 64bit AIX + LD_PRELOAD_VAR="LDR_PRELOAD64" + else + dnl 32bit AIX + LD_PRELOAD_VAR="LDR_PRELOAD" + fi + abi="xcoff" + ;; + *-*-mingw*) + abi="pecoff" + force_tls="0" + RPATH="" + so="dll" + if test "x$je_cv_msvc" = "xyes" ; then + importlib="lib" + DSO_LDFLAGS="-LD" + EXTRA_LDFLAGS="-link -DEBUG" + CTARGET='-Fo$@' + LDTARGET='-Fe$@' + MKLIB='lib -nologo -out:$@' + CC_MM= + else + importlib="${so}" + DSO_LDFLAGS="-shared" + fi + a="lib" + libprefix="" + SOREV="${so}" + PIC_CFLAGS="" + ;; *) AC_MSG_RESULT([Unsupported operating system: ${host}]) abi="elf" - RPATH="-Wl,-rpath," ;; esac AC_SUBST([abi]) AC_SUBST([RPATH]) +AC_SUBST([LD_PRELOAD_VAR]) +AC_SUBST([so]) +AC_SUBST([importlib]) +AC_SUBST([o]) +AC_SUBST([a]) +AC_SUBST([exe]) +AC_SUBST([libprefix]) +AC_SUBST([DSO_LDFLAGS]) +AC_SUBST([EXTRA_LDFLAGS]) +AC_SUBST([SOREV]) +AC_SUBST([PIC_CFLAGS]) +AC_SUBST([CTARGET]) +AC_SUBST([LDTARGET]) +AC_SUBST([MKLIB]) +AC_SUBST([CC_MM]) + +if test "x$abi" != "xpecoff"; then + dnl Heap profiling uses the log(3) function. + LIBS="$LIBS -lm" +fi JE_COMPILABLE([__attribute__ syntax], [static __attribute__((unused)) void foo(void){}], [], - [attribute]) -if test "x${attribute}" = "xyes" ; then + [je_cv_attribute]) +if test "x${je_cv_attribute}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_ATTR], [ ]) if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then JE_CFLAGS_APPEND([-fvisibility=hidden]) fi fi - -JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ -#define _GNU_SOURCE -#include -], [ -void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); -], [mremap_fixed]) -if test "x${mremap_fixed}" = "xyes" ; then - AC_DEFINE([JEMALLOC_MREMAP_FIXED]) +dnl Check for tls_model attribute support (clang 3.0 still lacks support). +SAVED_CFLAGS="${CFLAGS}" +JE_CFLAGS_APPEND([-Werror]) +JE_COMPILABLE([tls_model attribute], [], + [static __thread int + __attribute__((tls_model("initial-exec"))) foo; + foo = 0;], + [je_cv_tls_model]) +CFLAGS="${SAVED_CFLAGS}" +if test "x${je_cv_tls_model}" = "xyes" ; then + AC_DEFINE([JEMALLOC_TLS_MODEL], + [__attribute__((tls_model("initial-exec")))]) +else + AC_DEFINE([JEMALLOC_TLS_MODEL], [ ]) fi dnl Support optional additions to rpath. @@ -278,11 +403,52 @@ AC_PATH_PROG([AR], [ar], , [$PATH]) AC_PATH_PROG([LD], [ld], , [$PATH]) AC_PATH_PROG([AUTOCONF], [autoconf], , [$PATH]) +public_syms="malloc_conf malloc_message malloc calloc posix_memalign aligned_alloc realloc free malloc_usable_size malloc_stats_print mallctl mallctlnametomib mallctlbymib" + +dnl Check for allocator-related functions that should be wrapped. +AC_CHECK_FUNC([memalign], + [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ]) + public_syms="${public_syms} memalign"]) +AC_CHECK_FUNC([valloc], + [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC], [ ]) + public_syms="${public_syms} valloc"]) + +dnl Support the experimental API by default. +AC_ARG_ENABLE([experimental], + [AS_HELP_STRING([--disable-experimental], + [Disable support for the experimental API])], +[if test "x$enable_experimental" = "xno" ; then + enable_experimental="0" +else + enable_experimental="1" +fi +], +[enable_experimental="1"] +) +if test "x$enable_experimental" = "x1" ; then + AC_DEFINE([JEMALLOC_EXPERIMENTAL], [ ]) + public_syms="${public_syms} allocm dallocm nallocm rallocm sallocm" +fi +AC_SUBST([enable_experimental]) + +dnl Perform no name mangling by default. +AC_ARG_WITH([mangling], + [AS_HELP_STRING([--with-mangling=], [Mangle symbols in ])], + [mangling_map="$with_mangling"], [mangling_map=""]) +for nm in `echo ${mangling_map} |tr ',' ' '` ; do + k="`echo ${nm} |tr ':' ' ' |awk '{print $1}'`" + n="je_${k}" + m=`echo ${nm} |tr ':' ' ' |awk '{print $2}'` + AC_DEFINE_UNQUOTED([${n}], [${m}]) + dnl Remove key from public_syms so that it isn't redefined later. + public_syms=`for sym in ${public_syms}; do echo "${sym}"; done |grep -v "^${k}\$" |tr '\n' ' '` +done + dnl Do not prefix public APIs by default. AC_ARG_WITH([jemalloc_prefix], [AS_HELP_STRING([--with-jemalloc-prefix=], [Prefix to prepend to all public APIs])], [JEMALLOC_PREFIX="$with_jemalloc_prefix"], - [if test "x$abi" != "xmacho" ; then + [if test "x$abi" != "xmacho" -a "x$abi" != "xpecoff"; then JEMALLOC_PREFIX="" else JEMALLOC_PREFIX="je_" @@ -292,8 +458,15 @@ if test "x$JEMALLOC_PREFIX" != "x" ; then JEMALLOC_CPREFIX=`echo ${JEMALLOC_PREFIX} | tr "a-z" "A-Z"` AC_DEFINE_UNQUOTED([JEMALLOC_PREFIX], ["$JEMALLOC_PREFIX"]) AC_DEFINE_UNQUOTED([JEMALLOC_CPREFIX], ["$JEMALLOC_CPREFIX"]) - AC_DEFINE_UNQUOTED([JEMALLOC_P(string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix)], [${JEMALLOC_PREFIX}##string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix]) fi +dnl Generate macros to rename public symbols. All public symbols are prefixed +dnl with je_ in the source code, so these macro definitions are needed even if +dnl --with-jemalloc-prefix wasn't specified. +for stem in ${public_syms}; do + n="je_${stem}" + m="${JEMALLOC_PREFIX}${stem}" + AC_DEFINE_UNQUOTED([${n}], [${m}]) +done dnl Do not mangle library-private APIs by default. AC_ARG_WITH([private_namespace], @@ -342,8 +515,10 @@ cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in" cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" +cfghdrs_in="${cfghdrs_in} ${srcroot}include/jemalloc/internal/size_classes.sh" cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h" +cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in" @@ -361,7 +536,7 @@ fi [enable_cc_silence="0"] ) if test "x$enable_cc_silence" = "x1" ; then - AC_DEFINE([JEMALLOC_CC_SILENCE]) + AC_DEFINE([JEMALLOC_CC_SILENCE], [ ]) fi dnl Do not compile with debugging by default. @@ -390,22 +565,25 @@ if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then if test "x$GCC" = "xyes" ; then JE_CFLAGS_APPEND([-O3]) JE_CFLAGS_APPEND([-funroll-loops]) + elif test "x$je_cv_msvc" = "xyes" ; then + JE_CFLAGS_APPEND([-O2]) else JE_CFLAGS_APPEND([-O]) fi fi fi -dnl Do not enable statistics calculation by default. +dnl Enable statistics calculation by default. AC_ARG_ENABLE([stats], - [AS_HELP_STRING([--enable-stats], [Enable statistics calculation/reporting])], + [AS_HELP_STRING([--disable-stats], + [Disable statistics calculation/reporting])], [if test "x$enable_stats" = "xno" ; then enable_stats="0" else enable_stats="1" fi ], -[enable_stats="0"] +[enable_stats="1"] ) if test "x$enable_stats" = "x1" ; then AC_DEFINE([JEMALLOC_STATS], [ ]) @@ -531,27 +709,14 @@ fi AC_MSG_CHECKING([configured backtracing method]) AC_MSG_RESULT([$backtrace_method]) if test "x$enable_prof" = "x1" ; then - LIBS="$LIBS -lm" + if test "x${force_tls}" = "x0" ; then + AC_MSG_ERROR([Heap profiling requires TLS]); + fi + force_tls="1" AC_DEFINE([JEMALLOC_PROF], [ ]) fi AC_SUBST([enable_prof]) -dnl Enable tiny allocations by default. -AC_ARG_ENABLE([tiny], - [AS_HELP_STRING([--disable-tiny], [Disable tiny (sub-quantum) allocations])], -[if test "x$enable_tiny" = "xno" ; then - enable_tiny="0" -else - enable_tiny="1" -fi -], -[enable_tiny="1"] -) -if test "x$enable_tiny" = "x1" ; then - AC_DEFINE([JEMALLOC_TINY], [ ]) -fi -AC_SUBST([enable_tiny]) - dnl Enable thread-specific caching by default. AC_ARG_ENABLE([tcache], [AS_HELP_STRING([--disable-tcache], [Disable per thread caches])], @@ -568,21 +733,48 @@ if test "x$enable_tcache" = "x1" ; then fi AC_SUBST([enable_tcache]) -dnl Do not enable mmap()ped swap files by default. -AC_ARG_ENABLE([swap], - [AS_HELP_STRING([--enable-swap], [Enable mmap()ped swap files])], -[if test "x$enable_swap" = "xno" ; then - enable_swap="0" +dnl Disable mremap() for huge realloc() by default. +AC_ARG_ENABLE([mremap], + [AS_HELP_STRING([--enable-mremap], [Enable mremap(2) for huge realloc()])], +[if test "x$enable_mremap" = "xno" ; then + enable_mremap="0" else - enable_swap="1" + enable_mremap="1" fi ], -[enable_swap="0"] +[enable_mremap="0"] ) -if test "x$enable_swap" = "x1" ; then - AC_DEFINE([JEMALLOC_SWAP], [ ]) +if test "x$enable_mremap" = "x1" ; then + JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ +#define _GNU_SOURCE +#include +], [ +void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); +], [je_cv_mremap_fixed]) + if test "x${je_cv_mremap_fixed}" = "xno" ; then + enable_mremap="0" + fi +fi +if test "x$enable_mremap" = "x1" ; then + AC_DEFINE([JEMALLOC_MREMAP], [ ]) fi -AC_SUBST([enable_swap]) +AC_SUBST([enable_mremap]) + +dnl Enable VM deallocation via munmap() by default. +AC_ARG_ENABLE([munmap], + [AS_HELP_STRING([--disable-munmap], [Disable VM deallocation via munmap(2)])], +[if test "x$enable_munmap" = "xno" ; then + enable_munmap="0" +else + enable_munmap="1" +fi +], +[enable_munmap="${default_munmap}"] +) +if test "x$enable_munmap" = "x1" ; then + AC_DEFINE([JEMALLOC_MUNMAP], [ ]) +fi +AC_SUBST([enable_munmap]) dnl Do not enable allocation from DSS by default. AC_ARG_ENABLE([dss], @@ -595,102 +787,154 @@ fi ], [enable_dss="0"] ) +dnl Check whether the BSD/SUSv1 sbrk() exists. If not, disable DSS support. +AC_CHECK_FUNC([sbrk], [have_sbrk="1"], [have_sbrk="0"]) +if test "x$have_sbrk" = "x1" ; then + AC_DEFINE([JEMALLOC_HAVE_SBRK], [ ]) +else + enable_dss="0" +fi + if test "x$enable_dss" = "x1" ; then AC_DEFINE([JEMALLOC_DSS], [ ]) fi AC_SUBST([enable_dss]) -dnl Do not support the junk/zero filling option by default. +dnl Support the junk/zero filling option by default. AC_ARG_ENABLE([fill], - [AS_HELP_STRING([--enable-fill], [Support junk/zero filling option])], + [AS_HELP_STRING([--disable-fill], + [Disable support for junk/zero filling, quarantine, and redzones])], [if test "x$enable_fill" = "xno" ; then enable_fill="0" else enable_fill="1" fi ], -[enable_fill="0"] +[enable_fill="1"] ) if test "x$enable_fill" = "x1" ; then AC_DEFINE([JEMALLOC_FILL], [ ]) fi AC_SUBST([enable_fill]) -dnl Do not support the xmalloc option by default. -AC_ARG_ENABLE([xmalloc], - [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], -[if test "x$enable_xmalloc" = "xno" ; then - enable_xmalloc="0" +dnl Disable utrace(2)-based tracing by default. +AC_ARG_ENABLE([utrace], + [AS_HELP_STRING([--enable-utrace], [Enable utrace(2)-based tracing])], +[if test "x$enable_utrace" = "xno" ; then + enable_utrace="0" else - enable_xmalloc="1" + enable_utrace="1" fi ], -[enable_xmalloc="0"] +[enable_utrace="0"] ) -if test "x$enable_xmalloc" = "x1" ; then - AC_DEFINE([JEMALLOC_XMALLOC], [ ]) +JE_COMPILABLE([utrace(2)], [ +#include +#include +#include +#include +#include +], [ + utrace((void *)0, 0); +], [je_cv_utrace]) +if test "x${je_cv_utrace}" = "xno" ; then + enable_utrace="0" fi -AC_SUBST([enable_xmalloc]) +if test "x$enable_utrace" = "x1" ; then + AC_DEFINE([JEMALLOC_UTRACE], [ ]) +fi +AC_SUBST([enable_utrace]) -dnl Do not support the SYSV option by default. -AC_ARG_ENABLE([sysv], - [AS_HELP_STRING([--enable-sysv], [Support SYSV semantics option])], -[if test "x$enable_sysv" = "xno" ; then - enable_sysv="0" +dnl Support Valgrind by default. +AC_ARG_ENABLE([valgrind], + [AS_HELP_STRING([--disable-valgrind], [Disable support for Valgrind])], +[if test "x$enable_valgrind" = "xno" ; then + enable_valgrind="0" else - enable_sysv="1" + enable_valgrind="1" fi ], -[enable_sysv="0"] +[enable_valgrind="1"] ) -if test "x$enable_sysv" = "x1" ; then - AC_DEFINE([JEMALLOC_SYSV], [ ]) +if test "x$enable_valgrind" = "x1" ; then + JE_COMPILABLE([valgrind], [ +#include +#include + +#if !defined(VALGRIND_RESIZEINPLACE_BLOCK) +# error "Incompatible Valgrind version" +#endif +], [], [je_cv_valgrind]) + if test "x${je_cv_valgrind}" = "xno" ; then + enable_valgrind="0" + fi + if test "x$enable_valgrind" = "x1" ; then + AC_DEFINE([JEMALLOC_VALGRIND], [ ]) + fi fi -AC_SUBST([enable_sysv]) +AC_SUBST([enable_valgrind]) -dnl Do not determine page shift at run time by default. -AC_ARG_ENABLE([dynamic_page_shift], - [AS_HELP_STRING([--enable-dynamic-page-shift], - [Determine page size at run time (don't trust configure result)])], -[if test "x$enable_dynamic_page_shift" = "xno" ; then - enable_dynamic_page_shift="0" +dnl Do not support the xmalloc option by default. +AC_ARG_ENABLE([xmalloc], + [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], +[if test "x$enable_xmalloc" = "xno" ; then + enable_xmalloc="0" else - enable_dynamic_page_shift="1" + enable_xmalloc="1" fi ], -[enable_dynamic_page_shift="0"] +[enable_xmalloc="0"] ) -if test "x$enable_dynamic_page_shift" = "x1" ; then - AC_DEFINE([DYNAMIC_PAGE_SHIFT], [ ]) +if test "x$enable_xmalloc" = "x1" ; then + AC_DEFINE([JEMALLOC_XMALLOC], [ ]) fi -AC_SUBST([enable_dynamic_page_shift]) +AC_SUBST([enable_xmalloc]) -AC_MSG_CHECKING([STATIC_PAGE_SHIFT]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[[#include -#include +AC_CACHE_CHECK([STATIC_PAGE_SHIFT], + [je_cv_static_page_shift], + AC_RUN_IFELSE([AC_LANG_PROGRAM( +[[ #include -]], [[ +#ifdef _WIN32 +#include +#else +#include +#endif +#include +]], +[[ long result; FILE *f; +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + result = si.dwPageSize; +#else result = sysconf(_SC_PAGESIZE); +#endif if (result == -1) { return 1; } + result = ffsl(result) - 1; + f = fopen("conftest.out", "w"); if (f == NULL) { return 1; } - fprintf(f, "%u\n", ffs((int)result) - 1); - close(f); + fprintf(f, "%u\n", result); + fclose(f); return 0; ]])], - [STATIC_PAGE_SHIFT=`cat conftest.out`] - AC_MSG_RESULT([$STATIC_PAGE_SHIFT]) - AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$STATIC_PAGE_SHIFT]), - AC_MSG_RESULT([error])) + [je_cv_static_page_shift=`cat conftest.out`], + [je_cv_static_page_shift=undefined])) + +if test "x$je_cv_static_page_shift" != "xundefined"; then + AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$je_cv_static_page_shift]) +else + AC_MSG_ERROR([cannot determine value for STATIC_PAGE_SHIFT]) +fi dnl ============================================================================ dnl jemalloc configuration. @@ -716,28 +960,65 @@ AC_SUBST([jemalloc_version_gid]) dnl ============================================================================ dnl Configure pthreads. -AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) -AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], - [AC_MSG_ERROR([libpthread is missing])]) +if test "x$abi" != "xpecoff" ; then + AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) + dnl Some systems may embed pthreads functionality in libc; check for libpthread + dnl first, but try libc too before failing. + AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], + [AC_SEARCH_LIBS([pthread_create], , , + AC_MSG_ERROR([libpthread is missing]))]) +fi CPPFLAGS="$CPPFLAGS -D_REENTRANT" -dnl Enable lazy locking by default. +dnl Check whether the BSD-specific _malloc_thread_cleanup() exists. If so, use +dnl it rather than pthreads TSD cleanup functions to support cleanup during +dnl thread exit, in order to avoid pthreads library recursion during +dnl bootstrapping. +AC_CHECK_FUNC([_malloc_thread_cleanup], + [have__malloc_thread_cleanup="1"], + [have__malloc_thread_cleanup="0"] + ) +if test "x$have__malloc_thread_cleanup" = "x1" ; then + AC_DEFINE([JEMALLOC_MALLOC_THREAD_CLEANUP], [ ]) + force_tls="1" +fi + +dnl Check whether the BSD-specific _pthread_mutex_init_calloc_cb() exists. If +dnl so, mutex initialization causes allocation, and we need to implement this +dnl callback function in order to prevent recursive allocation. +AC_CHECK_FUNC([_pthread_mutex_init_calloc_cb], + [have__pthread_mutex_init_calloc_cb="1"], + [have__pthread_mutex_init_calloc_cb="0"] + ) +if test "x$have__pthread_mutex_init_calloc_cb" = "x1" ; then + AC_DEFINE([JEMALLOC_MUTEX_INIT_CB]) +fi + +dnl Disable lazy locking by default. AC_ARG_ENABLE([lazy_lock], - [AS_HELP_STRING([--disable-lazy-lock], - [Disable lazy locking (always lock, even when single-threaded)])], + [AS_HELP_STRING([--enable-lazy-lock], + [Enable lazy locking (only lock when multi-threaded)])], [if test "x$enable_lazy_lock" = "xno" ; then enable_lazy_lock="0" else enable_lazy_lock="1" fi ], -[enable_lazy_lock="1"] +[enable_lazy_lock="0"] ) +if test "x$enable_lazy_lock" = "x0" -a "x${force_lazy_lock}" = "x1" ; then + AC_MSG_RESULT([Forcing lazy-lock to avoid allocator/threading bootstrap issues]) + enable_lazy_lock="1" +fi if test "x$enable_lazy_lock" = "x1" ; then - AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) - AC_CHECK_LIB([dl], [dlopen], [LIBS="$LIBS -ldl"], - [AC_MSG_ERROR([libdl is missing])]) + if test "x$abi" != "xpecoff" ; then + AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) + AC_CHECK_FUNC([dlsym], [], + [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], + [AC_MSG_ERROR([libdl is missing])]) + ]) + fi AC_DEFINE([JEMALLOC_LAZY_LOCK], [ ]) fi AC_SUBST([enable_lazy_lock]) @@ -752,9 +1033,17 @@ fi , enable_tls="1" ) +if test "x${enable_tls}" = "x0" -a "x${force_tls}" = "x1" ; then + AC_MSG_RESULT([Forcing TLS to avoid allocator/threading bootstrap issues]) + enable_tls="1" +fi +if test "x${enable_tls}" = "x1" -a "x${force_tls}" = "x0" ; then + AC_MSG_RESULT([Forcing no TLS to avoid allocator/threading bootstrap issues]) + enable_tls="0" +fi if test "x${enable_tls}" = "x1" ; then AC_MSG_CHECKING([for TLS]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( +AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [[ __thread int x; ]], [[ @@ -767,17 +1056,50 @@ AC_RUN_IFELSE([AC_LANG_PROGRAM( enable_tls="0") fi AC_SUBST([enable_tls]) -if test "x${enable_tls}" = "x0" ; then - AC_DEFINE_UNQUOTED([NO_TLS], [ ]) +if test "x${enable_tls}" = "x1" ; then + AC_DEFINE_UNQUOTED([JEMALLOC_TLS], [ ]) +elif test "x${force_tls}" = "x1" ; then + AC_MSG_ERROR([Failed to configure TLS, which is mandatory for correct function]) fi dnl ============================================================================ dnl Check for ffsl(3), and fail if not found. This function exists on all dnl platforms that jemalloc currently has a chance of functioning on without dnl modification. +JE_COMPILABLE([a program using ffsl], [ +#include +#include +], [ + { + int rv = ffsl(0x08); + } +], [je_cv_function_ffsl]) +if test "x${je_cv_function_ffsl}" != "xyes" ; then + AC_MSG_ERROR([Cannot build without ffsl(3)]) +fi -AC_CHECK_FUNC([ffsl], [], - [AC_MSG_ERROR([Cannot build without ffsl(3)])]) +dnl ============================================================================ +dnl Check for atomic(9) operations as provided on FreeBSD. + +JE_COMPILABLE([atomic(9)], [ +#include +#include +#include +], [ + { + uint32_t x32 = 0; + volatile uint32_t *x32p = &x32; + atomic_fetchadd_32(x32p, 1); + } + { + unsigned long xlong = 0; + volatile unsigned long *xlongp = &xlong; + atomic_fetchadd_long(xlongp, 1); + } +], [je_cv_atomic9]) +if test "x${je_cv_atomic9}" = "xyes" ; then + AC_DEFINE([JEMALLOC_ATOMIC9]) +fi dnl ============================================================================ dnl Check for atomic(3) operations as provided on Darwin. @@ -796,9 +1118,43 @@ JE_COMPILABLE([Darwin OSAtomic*()], [ volatile int64_t *x64p = &x64; OSAtomicAdd64(1, x64p); } -], [osatomic]) -if test "x${osatomic}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSATOMIC]) +], [je_cv_osatomic]) +if test "x${je_cv_osatomic}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSATOMIC], [ ]) +fi + +dnl ============================================================================ +dnl Check whether __sync_{add,sub}_and_fetch() are available despite +dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. + +AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[ + AC_CACHE_CHECK([whether to force $1-bit __sync_{add,sub}_and_fetch()], + [je_cv_sync_compare_and_swap_$2], + [AC_LINK_IFELSE([AC_LANG_PROGRAM([ + #include + ], + [ + #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 + { + uint$1_t x$1 = 0; + __sync_add_and_fetch(&x$1, 42); + __sync_sub_and_fetch(&x$1, 1); + } + #else + #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 is defined, no need to force + #endif + ])], + [je_cv_sync_compare_and_swap_$2=yes], + [je_cv_sync_compare_and_swap_$2=no])]) + + if test "x${je_cv_sync_compare_and_swap_$2}" = "xyes" ; then + AC_DEFINE([JE_FORCE_SYNC_COMPARE_AND_SWAP_$2], [ ]) + fi +]) + +if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then + JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4) + JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8) fi dnl ============================================================================ @@ -811,69 +1167,59 @@ JE_COMPILABLE([Darwin OSSpin*()], [ OSSpinLock lock = 0; OSSpinLockLock(&lock); OSSpinLockUnlock(&lock); -], [osspin]) -if test "x${osspin}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSSPIN]) +], [je_cv_osspin]) +if test "x${je_cv_osspin}" = "xyes" ; then + AC_DEFINE([JEMALLOC_OSSPIN], [ ]) fi -dnl ============================================================================ -dnl Check for allocator-related functions that should be wrapped. - -AC_CHECK_FUNC([memalign], - [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN])]) -AC_CHECK_FUNC([valloc], - [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC])]) - dnl ============================================================================ dnl Darwin-related configuration. if test "x${abi}" = "xmacho" ; then - AC_DEFINE([JEMALLOC_IVSALLOC]) - AC_DEFINE([JEMALLOC_ZONE]) + AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) + AC_DEFINE([JEMALLOC_ZONE], [ ]) dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6 dnl releases. malloc_zone_t and malloc_introspection_t have new fields in dnl 10.6, which is the only source-level indication of the change. AC_MSG_CHECKING([malloc zone version]) - AC_TRY_COMPILE([#include -#include ], [ - static malloc_zone_t zone; - static struct malloc_introspection_t zone_introspect; - - zone.size = NULL; - zone.malloc = NULL; - zone.calloc = NULL; - zone.valloc = NULL; - zone.free = NULL; - zone.realloc = NULL; - zone.destroy = NULL; - zone.zone_name = "jemalloc_zone"; - zone.batch_malloc = NULL; - zone.batch_free = NULL; - zone.introspect = &zone_introspect; - zone.version = 6; - zone.memalign = NULL; - zone.free_definite_size = NULL; - - zone_introspect.enumerator = NULL; - zone_introspect.good_size = NULL; - zone_introspect.check = NULL; - zone_introspect.print = NULL; - zone_introspect.log = NULL; - zone_introspect.force_lock = NULL; - zone_introspect.force_unlock = NULL; - zone_introspect.statistics = NULL; - zone_introspect.zone_locked = NULL; -], [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [6]) - AC_MSG_RESULT([6])], - [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [3]) - AC_MSG_RESULT([3])]) + AC_DEFUN([JE_ZONE_PROGRAM], + [AC_LANG_PROGRAM( + [#include ], + [static foo[[sizeof($1) $2 sizeof(void *) * $3 ? 1 : -1]]] + )]) + + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,14)],[JEMALLOC_ZONE_VERSION=3],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,15)],[JEMALLOC_ZONE_VERSION=5],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,16)],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,9)],[JEMALLOC_ZONE_VERSION=6],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_introspection_t,==,13)],[JEMALLOC_ZONE_VERSION=7],[JEMALLOC_ZONE_VERSION=] + )])],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,==,17)],[JEMALLOC_ZONE_VERSION=8],[ + AC_COMPILE_IFELSE([JE_ZONE_PROGRAM(malloc_zone_t,>,17)],[JEMALLOC_ZONE_VERSION=9],[JEMALLOC_ZONE_VERSION=] + )])])])]) + if test "x${JEMALLOC_ZONE_VERSION}" = "x"; then + AC_MSG_RESULT([unsupported]) + AC_MSG_ERROR([Unsupported malloc zone version]) + fi + if test "${JEMALLOC_ZONE_VERSION}" = 9; then + JEMALLOC_ZONE_VERSION=8 + AC_MSG_RESULT([> 8]) + else + AC_MSG_RESULT([$JEMALLOC_ZONE_VERSION]) + fi + AC_DEFINE_UNQUOTED(JEMALLOC_ZONE_VERSION, [$JEMALLOC_ZONE_VERSION]) fi dnl ============================================================================ dnl Check for typedefs, structures, and compiler characteristics. AC_HEADER_STDBOOL +AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ + mkdir -p "include/jemalloc/internal" + "${srcdir}/include/jemalloc/internal/size_classes.sh" > "${objroot}include/jemalloc/internal/size_classes.h" +]) + dnl Process .in files. AC_SUBST([cfghdrs_in]) AC_SUBST([cfghdrs_out]) @@ -881,7 +1227,7 @@ AC_CONFIG_HEADERS([$cfghdrs_tup]) dnl ============================================================================ dnl Generate outputs. -AC_CONFIG_FILES([$cfgoutputs_tup config.stamp]) +AC_CONFIG_FILES([$cfgoutputs_tup config.stamp bin/jemalloc.sh]) AC_SUBST([cfgoutputs_in]) AC_SUBST([cfgoutputs_out]) AC_OUTPUT @@ -889,7 +1235,8 @@ AC_OUTPUT dnl ============================================================================ dnl Print out the results of configuration. AC_MSG_RESULT([===============================================================================]) -AC_MSG_RESULT([jemalloc version : $jemalloc_version]) +AC_MSG_RESULT([jemalloc version : ${jemalloc_version}]) +AC_MSG_RESULT([library revision : ${rev}]) AC_MSG_RESULT([]) AC_MSG_RESULT([CC : ${CC}]) AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) @@ -918,6 +1265,7 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([autogen : ${enable_autogen}]) +AC_MSG_RESULT([experimental : ${enable_experimental}]) AC_MSG_RESULT([cc-silence : ${enable_cc_silence}]) AC_MSG_RESULT([debug : ${enable_debug}]) AC_MSG_RESULT([stats : ${enable_stats}]) @@ -925,14 +1273,14 @@ AC_MSG_RESULT([prof : ${enable_prof}]) AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) -AC_MSG_RESULT([tiny : ${enable_tiny}]) AC_MSG_RESULT([tcache : ${enable_tcache}]) AC_MSG_RESULT([fill : ${enable_fill}]) +AC_MSG_RESULT([utrace : ${enable_utrace}]) +AC_MSG_RESULT([valgrind : ${enable_valgrind}]) AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) -AC_MSG_RESULT([sysv : ${enable_sysv}]) -AC_MSG_RESULT([swap : ${enable_swap}]) +AC_MSG_RESULT([mremap : ${enable_mremap}]) +AC_MSG_RESULT([munmap : ${enable_munmap}]) AC_MSG_RESULT([dss : ${enable_dss}]) -AC_MSG_RESULT([dynamic_page_shift : ${enable_dynamic_page_shift}]) AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) AC_MSG_RESULT([tls : ${enable_tls}]) AC_MSG_RESULT([===============================================================================]) diff --git a/deps/jemalloc/doc/jemalloc.3 b/deps/jemalloc/doc/jemalloc.3 index 0401cfe895e..5b5c78c4208 100644 --- a/deps/jemalloc/doc/jemalloc.3 +++ b/deps/jemalloc/doc/jemalloc.3 @@ -1,13 +1,13 @@ '\" t .\" Title: JEMALLOC .\" Author: Jason Evans -.\" Generator: DocBook XSL Stylesheets v1.75.2 -.\" Date: 11/14/2011 +.\" Generator: DocBook XSL Stylesheets v1.76.1 +.\" Date: 05/11/2012 .\" Manual: User Manual -.\" Source: jemalloc 2.2.5-0-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760 +.\" Source: jemalloc 3.0.0-0-gfc9b1dbf69f59d7ecfc4ac68da9847e017e1d046 .\" Language: English .\" -.TH "JEMALLOC" "3" "11/14/2011" "jemalloc 2.2.5-0-gfc1bb70e5f0d" "User Manual" +.TH "JEMALLOC" "3" "05/11/2012" "jemalloc 3.0.0-0-gfc9b1dbf69f5" "User Manual" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -31,7 +31,7 @@ jemalloc \- general purpose memory allocation functions .SH "LIBRARY" .PP -This manual describes jemalloc 2\&.2\&.5\-0\-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760\&. More information can be found at the +This manual describes jemalloc 3\&.0\&.0\-0\-gfc9b1dbf69f59d7ecfc4ac68da9847e017e1d046\&. More information can be found at the \m[blue]\fBjemalloc website\fR\m[]\&\s-2\u[1]\d\s+2\&. .SH "SYNOPSIS" .sp @@ -48,6 +48,8 @@ This manual describes jemalloc 2\&.2\&.5\-0\-gfc1bb70e5f0d9a58b39efa39cc549b5af5 .BI "void *calloc(size_t\ " "number" ", size_t\ " "size" ");" .HP \w'int\ posix_memalign('u .BI "int posix_memalign(void\ **" "ptr" ", size_t\ " "alignment" ", size_t\ " "size" ");" +.HP \w'void\ *aligned_alloc('u +.BI "void *aligned_alloc(size_t\ " "alignment" ", size_t\ " "size" ");" .HP \w'void\ *realloc('u .BI "void *realloc(void\ *" "ptr" ", size_t\ " "size" ");" .HP \w'void\ free('u @@ -76,6 +78,8 @@ const char *\fImalloc_conf\fR; .BI "int sallocm(const\ void\ *" "ptr" ", size_t\ *" "rsize" ", int\ " "flags" ");" .HP \w'int\ dallocm('u .BI "int dallocm(void\ *" "ptr" ", int\ " "flags" ");" +.HP \w'int\ nallocm('u +.BI "int nallocm(size_t\ *" "rsize" ", size_t\ " "size" ", int\ " "flags" ");" .SH "DESCRIPTION" .SS "Standard API" .PP @@ -110,6 +114,18 @@ must be a power of 2 at least as large as sizeof(\fBvoid *\fR)\&. .PP The +\fBaligned_alloc\fR\fB\fR +function allocates +\fIsize\fR +bytes of memory such that the allocation\*(Aqs base address is an even multiple of +\fIalignment\fR\&. The requested +\fIalignment\fR +must be a power of 2\&. Behavior is undefined if +\fIsize\fR +is not an integral multiple of +\fIalignment\fR\&. +.PP +The \fBrealloc\fR\fB\fR function changes the size of the previously allocated memory referenced by \fIptr\fR @@ -236,13 +252,16 @@ for (i = 0; i < nbins; i++) { .\} .SS "Experimental API" .PP -The experimental API is subject to change or removal without regard for backward compatibility\&. +The experimental API is subject to change or removal without regard for backward compatibility\&. If +\fB\-\-disable\-experimental\fR +is specified during configuration, the experimental API is omitted\&. .PP The \fBallocm\fR\fB\fR, \fBrallocm\fR\fB\fR, -\fBsallocm\fR\fB\fR, and -\fBdallocm\fR\fB\fR +\fBsallocm\fR\fB\fR, +\fBdallocm\fR\fB\fR, and +\fBnallocm\fR\fB\fR functions all have a \fIflags\fR argument that can be used to specify options\&. The functions only check the options that are contextually relevant\&. Use bitwise or (|) operations to specify one or more of the following: @@ -286,7 +305,10 @@ to the base address of the allocation, and sets to the real size of the allocation if \fIrsize\fR is not -\fBNULL\fR\&. +\fBNULL\fR\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR\&. .PP The \fBrallocm\fR\fB\fR @@ -306,6 +328,9 @@ is not is non\-zero, an attempt is made to resize the allocation to be at least \fIsize\fR + \fIextra\fR) bytes, though inability to allocate the extra byte(s) will not by itself result in failure\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR, or if (\fIsize\fR + \fIextra\fR > \fBSIZE_T_MAX\fR)\&. .PP The @@ -319,6 +344,23 @@ The function causes the memory referenced by \fIptr\fR to be made available for future allocations\&. +.PP +The +\fBnallocm\fR\fB\fR +function allocates no memory, but it performs the same size computation as the +\fBallocm\fR\fB\fR +function, and if +\fIrsize\fR +is not +\fBNULL\fR +it sets +\fI*rsize\fR +to the real size of the allocation that would result from the equivalent +\fBallocm\fR\fB\fR +function call\&. Behavior is undefined if +\fIsize\fR +is +\fB0\fR\&. .SH "TUNING" .PP Once, when the first call is made to one of the memory allocation routines, the allocator initializes its internals based in part on various options that can be specified at compile\- or run\-time\&. @@ -346,9 +388,9 @@ Traditionally, allocators have used to obtain memory, which is suboptimal for several reasons, including race conditions, increased fragmentation, and artificial limitations on maximum usable memory\&. If \fB\-\-enable\-dss\fR is specified during configuration, this allocator uses both -\fBsbrk\fR(2) +\fBmmap\fR(2) and -\fBmmap\fR(2), in that order of preference; otherwise only +\fBsbrk\fR(2), in that order of preference; otherwise only \fBmmap\fR(2) is used\&. .PP @@ -364,14 +406,8 @@ User objects are broken into three categories according to size: small, large, a .PP Each chunk that is managed by an arena tracks its contents as runs of contiguous pages (unused, backing a set of small objects, or backing one large object)\&. The combination of chunk alignment and chunk page maps makes it possible to determine all metadata regarding small and large allocations in constant time\&. .PP -Small objects are managed in groups by page runs\&. Each run maintains a frontier and free list to track which regions are in use\&. Unless -\fB\-\-disable\-tiny\fR -is specified during configuration, allocation requests that are no more than half the quantum (8 or 16, depending on architecture) are rounded up to the nearest power of two that is at least -sizeof(\fBvoid *\fR)\&. Allocation requests that are more than half the quantum, but no more than the minimum cacheline\-multiple size class (see the -"opt\&.lg_qspace_max" -option) are rounded up to the nearest multiple of the quantum\&. Allocation requests that are more than the minimum cacheline\-multiple size class, but no more than the minimum subpage\-multiple size class (see the -"opt\&.lg_cspace_max" -option) are rounded up to the nearest multiple of the cacheline size (64)\&. Allocation requests that are more than the minimum subpage\-multiple size class, but no more than the maximum subpage\-multiple size class are rounded up to the nearest multiple of the subpage size (256)\&. Allocation requests that are more than the maximum subpage\-multiple size class, but small enough to fit in an arena\-managed chunk (see the +Small objects are managed in groups by page runs\&. Each run maintains a frontier and free list to track which regions are in use\&. Allocation requests that are no more than half the quantum (8 or 16, depending on architecture) are rounded up to the nearest power of two that is at least +sizeof(\fBdouble\fR)\&. All other small object size classes are multiples of the quantum, spaced such that internal fragmentation is limited to approximately 25% for all but the smallest size classes\&. Allocation requests that are larger than the maximum small size class, but small enough to fit in an arena\-managed chunk (see the "opt\&.lg_chunk" option), are rounded up to the nearest run size\&. Allocation requests that are too large to fit in an arena\-managed chunk are rounded up to the nearest multiple of the chunk size\&. .PP @@ -387,51 +423,73 @@ Table 1\&. .B Table\ \&1.\ \&Size classes .TS allbox tab(:); -lB lB lB. +lB rB lB. T{ Category T}:T{ -Subcategory +Spacing T}:T{ Size T} .T& -l l l -^ l l -^ l l -^ l l -l s l -l s l. +l r l +^ r l +^ r l +^ r l +^ r l +^ r l +^ r l +l r l +l r l. T{ Small T}:T{ -Tiny +lg T}:T{ [8] T} :T{ -Quantum\-spaced +16 T}:T{ [16, 32, 48, \&.\&.\&., 128] T} :T{ -Cacheline\-spaced +32 +T}:T{ +[160, 192, 224, 256] +T} +:T{ +64 +T}:T{ +[320, 384, 448, 512] +T} +:T{ +128 +T}:T{ +[640, 768, 896, 1024] +T} +:T{ +256 T}:T{ -[192, 256, 320, \&.\&.\&., 512] +[1280, 1536, 1792, 2048] T} :T{ -Subpage\-spaced +512 T}:T{ -[768, 1024, 1280, \&.\&.\&., 3840] +[2560, 3072, 3584] T} T{ Large T}:T{ +4 KiB +T}:T{ [4 KiB, 8 KiB, 12 KiB, \&.\&.\&., 4072 KiB] T} T{ Huge T}:T{ +4 MiB +T}:T{ [4 MiB, 8 MiB, 12 MiB, \&.\&.\&.] T} .TE @@ -481,12 +539,6 @@ was specified during build configuration\&. was specified during build configuration\&. .RE .PP -"config\&.dynamic_page_shift" (\fBbool\fR) r\- -.RS 4 -\fB\-\-enable\-dynamic\-page\-shift\fR -was specified during build configuration\&. -.RE -.PP "config\&.fill" (\fBbool\fR) r\- .RS 4 \fB\-\-enable\-fill\fR @@ -499,6 +551,18 @@ was specified during build configuration\&. was specified during build configuration\&. .RE .PP +"config\&.mremap" (\fBbool\fR) r\- +.RS 4 +\fB\-\-enable\-mremap\fR +was specified during build configuration\&. +.RE +.PP +"config\&.munmap" (\fBbool\fR) r\- +.RS 4 +\fB\-\-enable\-munmap\fR +was specified during build configuration\&. +.RE +.PP "config\&.prof" (\fBbool\fR) r\- .RS 4 \fB\-\-enable\-prof\fR @@ -523,34 +587,28 @@ was specified during build configuration\&. was specified during build configuration\&. .RE .PP -"config\&.swap" (\fBbool\fR) r\- -.RS 4 -\fB\-\-enable\-swap\fR -was specified during build configuration\&. -.RE -.PP -"config\&.sysv" (\fBbool\fR) r\- -.RS 4 -\fB\-\-enable\-sysv\fR -was specified during build configuration\&. -.RE -.PP "config\&.tcache" (\fBbool\fR) r\- .RS 4 \fB\-\-disable\-tcache\fR was not specified during build configuration\&. .RE .PP -"config\&.tiny" (\fBbool\fR) r\- +"config\&.tls" (\fBbool\fR) r\- .RS 4 -\fB\-\-disable\-tiny\fR +\fB\-\-disable\-tls\fR was not specified during build configuration\&. .RE .PP -"config\&.tls" (\fBbool\fR) r\- +"config\&.utrace" (\fBbool\fR) r\- .RS 4 -\fB\-\-disable\-tls\fR -was not specified during build configuration\&. +\fB\-\-enable\-utrace\fR +was specified during build configuration\&. +.RE +.PP +"config\&.valgrind" (\fBbool\fR) r\- +.RS 4 +\fB\-\-enable\-valgrind\fR +was specified during build configuration\&. .RE .PP "config\&.xmalloc" (\fBbool\fR) r\- @@ -568,16 +626,6 @@ in these cases\&. This option is disabled by default unless is specified during configuration, in which case it is enabled by default\&. .RE .PP -"opt\&.lg_qspace_max" (\fBsize_t\fR) r\- -.RS 4 -Size (log base 2) of the maximum size class that is a multiple of the quantum (8 or 16 bytes, depending on architecture)\&. Above this size, cacheline spacing is used for size classes\&. The default value is 128 bytes (2^7)\&. -.RE -.PP -"opt\&.lg_cspace_max" (\fBsize_t\fR) r\- -.RS 4 -Size (log base 2) of the maximum size class that is a multiple of the cacheline size (64)\&. Above this size, subpage spacing (256 bytes) is used for size classes\&. The default value is 512 bytes (2^9)\&. -.RE -.PP "opt\&.lg_chunk" (\fBsize_t\fR) r\- .RS 4 Virtual memory chunk size (log base 2)\&. The default chunk size is 4 MiB (2^22)\&. @@ -615,6 +663,22 @@ Junk filling enabled/disabled\&. If enabled, each byte of uninitialized allocate is specified during configuration, in which case it is enabled by default\&. .RE .PP +"opt\&.quarantine" (\fBsize_t\fR) r\- [\fB\-\-enable\-fill\fR] +.RS 4 +Per thread quarantine size in bytes\&. If non\-zero, each thread maintains a FIFO object quarantine that stores up to the specified number of bytes of memory\&. The quarantined memory is not freed until it is released from quarantine, though it is immediately junk\-filled if the +"opt\&.junk" +option is enabled\&. This feature is of particular use in combination with +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2, which can detect attempts to access quarantined objects\&. This is intended for debugging and will impact performance negatively\&. The default quarantine size is 0\&. +.RE +.PP +"opt\&.redzone" (\fBbool\fR) r\- [\fB\-\-enable\-fill\fR] +.RS 4 +Redzones enabled/disabled\&. If enabled, small allocations have redzones before and after them\&. Furthermore, if the +"opt\&.junk" +option is enabled, the redzones are checked for corruption during deallocation\&. However, the primary intended purpose of this feature is to be used in combination with +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2, which needs redzones in order to do effective buffer overflow/underflow detection\&. This option is intended for debugging and will impact performance negatively\&. This option is disabled by default\&. +.RE +.PP "opt\&.zero" (\fBbool\fR) r\- [\fB\-\-enable\-fill\fR] .RS 4 Zero filling enabled/disabled\&. If enabled, each byte of uninitialized allocated memory will be initialized to 0\&. Note that this initialization only happens once for each byte, so @@ -624,13 +688,25 @@ and calls do not zero memory that was previously allocated\&. This is intended for debugging and will impact performance negatively\&. This option is disabled by default\&. .RE .PP -"opt\&.sysv" (\fBbool\fR) r\- [\fB\-\-enable\-sysv\fR] +"opt\&.utrace" (\fBbool\fR) r\- [\fB\-\-enable\-utrace\fR] .RS 4 -If enabled, attempting to allocate zero bytes will return a -\fBNULL\fR -pointer instead of a valid pointer\&. (The default behavior is to make a minimal allocation and return a pointer to it\&.) This option is provided for System V compatibility\&. This option is incompatible with the -"opt\&.xmalloc" -option\&. This option is disabled by default\&. +Allocation tracing based on +\fButrace\fR(2) +enabled/disabled\&. This option is disabled by default\&. +.RE +.PP +"opt\&.valgrind" (\fBbool\fR) r\- [\fB\-\-enable\-valgrind\fR] +.RS 4 +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2 +support enabled/disabled\&. If enabled, several other options are automatically modified during options processing to work well with Valgrind: +"opt\&.junk" +and +"opt\&.zero" +are set to false, +"opt\&.quarantine" +is set to 16 MiB, and +"opt\&.redzone" +is set to true\&. This option is disabled by default\&. .RE .PP "opt\&.xmalloc" (\fBbool\fR) r\- [\fB\-\-enable\-xmalloc\fR] @@ -656,15 +732,8 @@ This option is disabled by default\&. "opt\&.tcache" (\fBbool\fR) r\- [\fB\-\-enable\-tcache\fR] .RS 4 Thread\-specific caching enabled/disabled\&. When there are multiple threads, each thread uses a thread\-specific cache for objects up to a certain size\&. Thread\-specific caching allows many allocations to be satisfied without performing any thread synchronization, at the cost of increased memory use\&. See the -"opt\&.lg_tcache_gc_sweep" -and "opt\&.lg_tcache_max" -options for related tuning information\&. This option is enabled by default\&. -.RE -.PP -"opt\&.lg_tcache_gc_sweep" (\fBssize_t\fR) r\- [\fB\-\-enable\-tcache\fR] -.RS 4 -Approximate interval (log base 2) between full thread\-specific cache garbage collection sweeps, counted in terms of thread\-specific cache allocation/deallocation events\&. Garbage collection is actually performed incrementally, one size class at a time, in order to avoid large collection pauses\&. The default sweep interval is 8192 (2^13); setting this option to \-1 will disable garbage collection\&. +option for related tuning information\&. This option is enabled by default\&. .RE .PP "opt\&.lg_tcache_max" (\fBsize_t\fR) r\- [\fB\-\-enable\-tcache\fR] @@ -674,31 +743,22 @@ Maximum size class (log base 2) to cache in the thread\-specific cache\&. At a m .PP "opt\&.prof" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 -Memory profiling enabled/disabled\&. If enabled, profile memory allocation activity, and use an -\fBatexit\fR(3) -function to dump final memory usage to a file named according to the pattern -\&.\&.\&.f\&.heap, where - -is controlled by the -"opt\&.prof_prefix" -option\&. See the -"opt\&.lg_prof_bt_max" -option for backtrace depth control\&. See the +Memory profiling enabled/disabled\&. If enabled, profile memory allocation activity\&. See the "opt\&.prof_active" option for on\-the\-fly activation/deactivation\&. See the "opt\&.lg_prof_sample" option for probabilistic sampling control\&. See the "opt\&.prof_accum" option for control of cumulative sample reporting\&. See the -"opt\&.lg_prof_tcmax" -option for control of per thread backtrace caching\&. See the "opt\&.lg_prof_interval" -option for information on interval\-triggered profile dumping, and the +option for information on interval\-triggered profile dumping, the "opt\&.prof_gdump" -option for information on high\-water\-triggered profile dumping\&. Profile output is compatible with the included +option for information on high\-water\-triggered profile dumping, and the +"opt\&.prof_final" +option for final profile dumping\&. Profile output is compatible with the included \fBpprof\fR Perl script, which originates from the -\m[blue]\fBgoogle\-perftools package\fR\m[]\&\s-2\u[2]\d\s+2\&. +\m[blue]\fBgperftools package\fR\m[]\&\s-2\u[3]\d\s+2\&. .RE .PP "opt\&.prof_prefix" (\fBconst char *\fR) r\- [\fB\-\-enable\-prof\fR] @@ -707,11 +767,6 @@ Filename prefix for profile dumps\&. If the prefix is set to the empty string, n jeprof\&. .RE .PP -"opt\&.lg_prof_bt_max" (\fBsize_t\fR) r\- [\fB\-\-enable\-prof\fR] -.RS 4 -Maximum backtrace depth (log base 2) when profiling memory allocation activity\&. The default is 128 (2^7)\&. -.RE -.PP "opt\&.prof_active" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 Profiling activated/deactivated\&. This is a secondary control mechanism that makes it possible to start the application with profiling enabled (see the @@ -723,21 +778,12 @@ mallctl\&. This option is enabled by default\&. .PP "opt\&.lg_prof_sample" (\fBssize_t\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 -Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity\&. Increasing the sampling interval decreases profile fidelity, but also decreases the computational overhead\&. The default sample interval is 1 (2^0) (i\&.e\&. all allocations are sampled)\&. +Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity\&. Increasing the sampling interval decreases profile fidelity, but also decreases the computational overhead\&. The default sample interval is 512 KiB (2^19 B)\&. .RE .PP "opt\&.prof_accum" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 -Reporting of cumulative object/byte counts in profile dumps enabled/disabled\&. If this option is enabled, every unique backtrace must be stored for the duration of execution\&. Depending on the application, this can impose a large memory overhead, and the cumulative counts are not always of interest\&. See the -"opt\&.lg_prof_tcmax" -option for control of per thread backtrace caching, which has important interactions\&. This option is enabled by default\&. -.RE -.PP -"opt\&.lg_prof_tcmax" (\fBssize_t\fR) r\- [\fB\-\-enable\-prof\fR] -.RS 4 -Maximum per thread backtrace cache (log base 2) used for heap profiling\&. A backtrace can only be discarded if the -"opt\&.prof_accum" -option is disabled, and no thread caches currently refer to the backtrace\&. Therefore, a backtrace cache limit should be imposed if the intention is to limit how much memory is used by backtraces\&. By default, no limit is imposed (encoded as \-1)\&. +Reporting of cumulative object/byte counts in profile dumps enabled/disabled\&. If this option is enabled, every unique backtrace must be stored for the duration of execution\&. Depending on the application, this can impose a large memory overhead, and the cumulative counts are not always of interest\&. This option is disabled by default\&. .RE .PP "opt\&.lg_prof_interval" (\fBssize_t\fR) r\- [\fB\-\-enable\-prof\fR] @@ -760,33 +806,27 @@ is controlled by the option\&. This option is disabled by default\&. .RE .PP +"opt\&.prof_final" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] +.RS 4 +Use an +\fBatexit\fR(3) +function to dump final memory usage to a file named according to the pattern +\&.\&.\&.f\&.heap, where + +is controlled by the +"opt\&.prof_prefix" +option\&. This option is enabled by default\&. +.RE +.PP "opt\&.prof_leak" (\fBbool\fR) r\- [\fB\-\-enable\-prof\fR] .RS 4 Leak reporting enabled/disabled\&. If enabled, use an \fBatexit\fR(3) function to report memory leaks detected by allocation sampling\&. See the -"opt\&.lg_prof_bt_max" -option for backtrace depth control\&. See the "opt\&.prof" option for information on analyzing heap profile output\&. This option is disabled by default\&. .RE .PP -"opt\&.overcommit" (\fBbool\fR) r\- [\fB\-\-enable\-swap\fR] -.RS 4 -Over\-commit enabled/disabled\&. If enabled, over\-commit memory as a side effect of using anonymous -\fBmmap\fR(2) -or -\fBsbrk\fR(2) -for virtual memory allocation\&. In order for overcommit to be disabled, the -"swap\&.fds" -mallctl must have been successfully written to\&. This option is enabled by default\&. -.RE -.PP -"tcache\&.flush" (\fBvoid\fR) \-\- [\fB\-\-enable\-tcache\fR] -.RS 4 -Flush calling thread\*(Aqs tcache\&. This interface releases all cached objects and internal data structures associated with the calling thread\*(Aqs thread\-specific cache\&. Ordinarily, this interface need not be called, since automatic periodic incremental garbage collection occurs, and the thread cache is automatically discarded when a thread exits\&. However, garbage collection is triggered by allocation activity, so it is possible for a thread that stops allocating/deallocating to retain its cache indefinitely, in which case the developer may find manual flushing useful\&. -.RE -.PP "thread\&.arena" (\fBunsigned\fR) rw .RS 4 Get or set the arena associated with the calling thread\&. The arena index must be less than the maximum number of arenas (see the @@ -824,6 +864,17 @@ mallctl\&. This is useful for avoiding the overhead of repeated calls\&. .RE .PP +"thread\&.tcache\&.enabled" (\fBbool\fR) rw [\fB\-\-enable\-tcache\fR] +.RS 4 +Enable/disable calling thread\*(Aqs tcache\&. The tcache is implicitly flushed as a side effect of becoming disabled (see +"thread\&.tcache\&.flush")\&. +.RE +.PP +"thread\&.tcache\&.flush" (\fBvoid\fR) \-\- [\fB\-\-enable\-tcache\fR] +.RS 4 +Flush calling thread\*(Aqs tcache\&. This interface releases all cached objects and internal data structures associated with the calling thread\*(Aqs thread\-specific cache\&. Ordinarily, this interface need not be called, since automatic periodic incremental garbage collection occurs, and the thread cache is automatically discarded when a thread exits\&. However, garbage collection is triggered by allocation activity, so it is possible for a thread that stops allocating/deallocating to retain its cache indefinitely, in which case the developer may find manual flushing useful\&. +.RE +.PP "arenas\&.narenas" (\fBunsigned\fR) r\- .RS 4 Maximum number of arenas\&. @@ -841,94 +892,19 @@ booleans\&. Each boolean indicates whether the corresponding arena is initialize Quantum size\&. .RE .PP -"arenas\&.cacheline" (\fBsize_t\fR) r\- -.RS 4 -Assumed cacheline size\&. -.RE -.PP -"arenas\&.subpage" (\fBsize_t\fR) r\- -.RS 4 -Subpage size class interval\&. -.RE -.PP -"arenas\&.pagesize" (\fBsize_t\fR) r\- +"arenas\&.page" (\fBsize_t\fR) r\- .RS 4 Page size\&. .RE .PP -"arenas\&.chunksize" (\fBsize_t\fR) r\- -.RS 4 -Chunk size\&. -.RE -.PP -"arenas\&.tspace_min" (\fBsize_t\fR) r\- -.RS 4 -Minimum tiny size class\&. Tiny size classes are powers of two\&. -.RE -.PP -"arenas\&.tspace_max" (\fBsize_t\fR) r\- -.RS 4 -Maximum tiny size class\&. Tiny size classes are powers of two\&. -.RE -.PP -"arenas\&.qspace_min" (\fBsize_t\fR) r\- -.RS 4 -Minimum quantum\-spaced size class\&. -.RE -.PP -"arenas\&.qspace_max" (\fBsize_t\fR) r\- -.RS 4 -Maximum quantum\-spaced size class\&. -.RE -.PP -"arenas\&.cspace_min" (\fBsize_t\fR) r\- -.RS 4 -Minimum cacheline\-spaced size class\&. -.RE -.PP -"arenas\&.cspace_max" (\fBsize_t\fR) r\- -.RS 4 -Maximum cacheline\-spaced size class\&. -.RE -.PP -"arenas\&.sspace_min" (\fBsize_t\fR) r\- -.RS 4 -Minimum subpage\-spaced size class\&. -.RE -.PP -"arenas\&.sspace_max" (\fBsize_t\fR) r\- -.RS 4 -Maximum subpage\-spaced size class\&. -.RE -.PP "arenas\&.tcache_max" (\fBsize_t\fR) r\- [\fB\-\-enable\-tcache\fR] .RS 4 Maximum thread\-cached size class\&. .RE .PP -"arenas\&.ntbins" (\fBunsigned\fR) r\- -.RS 4 -Number of tiny bin size classes\&. -.RE -.PP -"arenas\&.nqbins" (\fBunsigned\fR) r\- -.RS 4 -Number of quantum\-spaced bin size classes\&. -.RE -.PP -"arenas\&.ncbins" (\fBunsigned\fR) r\- -.RS 4 -Number of cacheline\-spaced bin size classes\&. -.RE -.PP -"arenas\&.nsbins" (\fBunsigned\fR) r\- -.RS 4 -Number of subpage\-spaced bin size classes\&. -.RE -.PP "arenas\&.nbins" (\fBunsigned\fR) r\- .RS 4 -Total number of bin size classes\&. +Number of bin size classes\&. .RE .PP "arenas\&.nhbins" (\fBunsigned\fR) r\- [\fB\-\-enable\-tcache\fR] @@ -1011,12 +987,12 @@ Total number of bytes in active pages allocated by the application\&. This is a "stats\&.mapped" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Total number of bytes in chunks mapped on behalf of the application\&. This is a multiple of the chunk size, and is at least as large as -"stats\&.active"\&. This does not include inactive chunks backed by swap files\&. his does not include inactive chunks embedded in the DSS\&. +"stats\&.active"\&. This does not include inactive chunks\&. .RE .PP "stats\&.chunks\&.current" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 -Total number of chunks actively mapped on behalf of the application\&. This does not include inactive chunks backed by swap files\&. This does not include inactive chunks embedded in the DSS\&. +Total number of chunks actively mapped on behalf of the application\&. This does not include inactive chunks\&. .RE .PP "stats\&.chunks\&.total" (\fBuint64_t\fR) r\- [\fB\-\-enable\-stats\fR] @@ -1163,11 +1139,6 @@ Cumulative number of runs created\&. Cumulative number of times the current run from which to allocate changed\&. .RE .PP -"stats\&.arenas\&.\&.bins\&.\&.highruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] -.RS 4 -Maximum number of runs at any time thus far\&. -.RE -.PP "stats\&.arenas\&.\&.bins\&.\&.curruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Current number of runs\&. @@ -1188,44 +1159,10 @@ Cumulative number of deallocation requests for this size class served directly b Cumulative number of allocation requests for this size class\&. .RE .PP -"stats\&.arenas\&.\&.lruns\&.\&.highruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] -.RS 4 -Maximum number of runs at any time thus far for this size class\&. -.RE -.PP "stats\&.arenas\&.\&.lruns\&.\&.curruns" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats\fR] .RS 4 Current number of runs for this size class\&. .RE -.PP -"swap\&.avail" (\fBsize_t\fR) r\- [\fB\-\-enable\-stats \-\-enable\-swap\fR] -.RS 4 -Number of swap file bytes that are currently not associated with any chunk (i\&.e\&. mapped, but otherwise completely unmanaged)\&. -.RE -.PP -"swap\&.prezeroed" (\fBbool\fR) rw [\fB\-\-enable\-swap\fR] -.RS 4 -If true, the allocator assumes that the swap file(s) contain nothing but nil bytes\&. If this assumption is violated, allocator behavior is undefined\&. This value becomes read\-only after -"swap\&.fds" -is successfully written to\&. -.RE -.PP -"swap\&.nfds" (\fBsize_t\fR) r\- [\fB\-\-enable\-swap\fR] -.RS 4 -Number of file descriptors in use for swap\&. -.RE -.PP -"swap\&.fds" (\fBint *\fR) rw [\fB\-\-enable\-swap\fR] -.RS 4 -When written to, the files associated with the specified file descriptors are contiguously mapped via -\fBmmap\fR(2)\&. The resulting virtual memory region is preferred over anonymous -\fBmmap\fR(2) -and -\fBsbrk\fR(2) -memory\&. Note that if a file\*(Aqs size is not a multiple of the page size, it is automatically truncated to the nearest page size multiple\&. See the -"swap\&.prezeroed" -mallctl for specifying that the files are pre\-zeroed\&. -.RE .SH "DEBUGGING MALLOC PROBLEMS" .PP When debugging, it is a good idea to configure/build jemalloc with the @@ -1240,7 +1177,13 @@ option) tends to expose such bugs in the form of obviously incorrect results and "opt\&.zero" option) eliminates the symptoms of such bugs\&. Between these two options, it is usually possible to quickly detect, diagnose, and eliminate such bugs\&. .PP -This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information would be prohibitive\&. There are a number of allocator implementations available on the Internet which focus on detecting and pinpointing problems by trading performance for extra sanity checks and detailed diagnostics\&. +This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information would be prohibitive\&. However, jemalloc does integrate with the most excellent +\m[blue]\fBValgrind\fR\m[]\&\s-2\u[2]\d\s+2 +tool if the +\fB\-\-enable\-valgrind\fR +configuration option is enabled and the +"opt\&.valgrind" +option is enabled\&. .SH "DIAGNOSTIC MESSAGES" .PP If any of the memory allocation/deallocation functions detect an error or warning condition, a message will be printed to file descriptor @@ -1296,6 +1239,28 @@ Memory allocation error\&. .RE .PP The +\fBaligned_alloc\fR\fB\fR +function returns a pointer to the allocated memory if successful; otherwise a +\fBNULL\fR +pointer is returned and +\fIerrno\fR +is set\&. The +\fBaligned_alloc\fR\fB\fR +function will fail if: +.PP +EINVAL +.RS 4 +The +\fIalignment\fR +parameter is not a power of 2\&. +.RE +.PP +ENOMEM +.RS 4 +Memory allocation error\&. +.RE +.PP +The \fBrealloc\fR\fB\fR function returns a pointer, possibly identical to \fIptr\fR, to the allocated memory if successful; otherwise a @@ -1370,14 +1335,15 @@ read/write processing\&. The \fBallocm\fR\fB\fR, \fBrallocm\fR\fB\fR, -\fBsallocm\fR\fB\fR, and -\fBdallocm\fR\fB\fR +\fBsallocm\fR\fB\fR, +\fBdallocm\fR\fB\fR, and +\fBnallocm\fR\fB\fR functions return \fBALLOCM_SUCCESS\fR on success; otherwise they return an error value\&. The -\fBallocm\fR\fB\fR -and -\fBrallocm\fR\fB\fR +\fBallocm\fR\fB\fR, +\fBrallocm\fR\fB\fR, and +\fBnallocm\fR\fB\fR functions will fail if: .PP ALLOCM_ERR_OOM @@ -1442,6 +1408,7 @@ malloc_conf = "lg_chunk:24"; \fBmadvise\fR(2), \fBmmap\fR(2), \fBsbrk\fR(2), +\fButrace\fR(2), \fBalloca\fR(3), \fBatexit\fR(3), \fBgetpagesize\fR(3) @@ -1469,7 +1436,12 @@ jemalloc website \%http://www.canonware.com/jemalloc/ .RE .IP " 2." 4 -google-perftools package +Valgrind +.RS 4 +\%http://valgrind.org/ +.RE +.IP " 3." 4 +gperftools package .RS 4 -\%http://code.google.com/p/google-perftools/ +\%http://code.google.com/p/gperftools/ .RE diff --git a/deps/jemalloc/doc/jemalloc.html b/deps/jemalloc/doc/jemalloc.html index fc2ba878e54..415e298d485 100644 --- a/deps/jemalloc/doc/jemalloc.html +++ b/deps/jemalloc/doc/jemalloc.html @@ -1,8 +1,8 @@ -JEMALLOC

Name

jemalloc — general purpose memory allocation functions

LIBRARY

This manual describes jemalloc 2.2.5-0-gfc1bb70e5f0d9a58b39efa39cc549b5af5104760. More information +JEMALLOC

Name

jemalloc — general purpose memory allocation functions

LIBRARY

This manual describes jemalloc 3.0.0-0-gfc9b1dbf69f59d7ecfc4ac68da9847e017e1d046. More information can be found at the jemalloc website.

SYNOPSIS

#include <stdlib.h>
-#include <jemalloc/jemalloc.h>

Standard API

void *malloc(size_t size);
 
void *calloc(size_t number,
 size_t size);
 
int posix_memalign(void **ptr,
 size_t alignment,
 size_t size);
 
void *realloc(void *ptr,
 size_t size);
 
void free(void *ptr);
 

Non-standard API

size_t malloc_usable_size(const void *ptr);
 
void malloc_stats_print(void (*write_cb) +#include <jemalloc/jemalloc.h>

Standard API

void *malloc(size_t size);
 
void *calloc(size_t number,
 size_t size);
 
int posix_memalign(void **ptr,
 size_t alignment,
 size_t size);
 
void *aligned_alloc(size_t alignment,
 size_t size);
 
void *realloc(void *ptr,
 size_t size);
 
void free(void *ptr);
 

Non-standard API

size_t malloc_usable_size(const void *ptr);
 
void malloc_stats_print(void (*write_cb) (void *, const char *) - ,
 void *cbopaque,
 const char *opts);
 
int mallctl(const char *name,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
int mallctlnametomib(const char *name,
 size_t *mibp,
 size_t *miblenp);
 
int mallctlbymib(const size_t *mib,
 size_t miblen,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
void (*malloc_message)(void *cbopaque,
 const char *s);
 

const char *malloc_conf;

Experimental API

int allocm(void **ptr,
 size_t *rsize,
 size_t size,
 int flags);
 
int rallocm(void **ptr,
 size_t *rsize,
 size_t size,
 size_t extra,
 int flags);
 
int sallocm(const void *ptr,
 size_t *rsize,
 int flags);
 
int dallocm(void *ptr,
 int flags);
 

DESCRIPTION

Standard API

The malloc() function allocates + ,

 void *cbopaque,
 const char *opts);
 
int mallctl(const char *name,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
int mallctlnametomib(const char *name,
 size_t *mibp,
 size_t *miblenp);
 
int mallctlbymib(const size_t *mib,
 size_t miblen,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
void (*malloc_message)(void *cbopaque,
 const char *s);
 

const char *malloc_conf;

Experimental API

int allocm(void **ptr,
 size_t *rsize,
 size_t size,
 int flags);
 
int rallocm(void **ptr,
 size_t *rsize,
 size_t size,
 size_t extra,
 int flags);
 
int sallocm(const void *ptr,
 size_t *rsize,
 int flags);
 
int dallocm(void *ptr,
 int flags);
 
int nallocm(size_t *rsize,
 size_t size,
 int flags);
 

DESCRIPTION

Standard API

The malloc() function allocates size bytes of uninitialized memory. The allocated space is suitably aligned (after possible pointer coercion) for storage of any type of object.

The calloc() function allocates @@ -17,7 +17,13 @@ alignment, and returns the allocation in the value pointed to by ptr. The requested alignment must be a power of 2 at least as large - as sizeof(void *).

The realloc() function changes the + as sizeof(void *).

The aligned_alloc() function + allocates size bytes of memory such that the + allocation's base address is an even multiple of + alignment. The requested + alignment must be a power of 2. Behavior is + undefined if size is not an integral multiple of + alignment.

The realloc() function changes the size of the previously allocated memory referenced by ptr to size bytes. The contents of the memory are unchanged up to the lesser of the new and old @@ -32,7 +38,7 @@ malloc() for the specified size.

The free() function causes the allocated memory referenced by ptr to be made available for future allocations. If ptr is - NULL, no action occurs.

Non-standard API

The malloc_usable_size() function + NULL, no action occurs.

Non-standard API

The malloc_usable_size() function returns the usable size of the allocation pointed to by ptr. The return value may be larger than the size that was requested during allocation. The @@ -112,11 +118,14 @@ len = sizeof(bin_size); mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0); /* Do something with bin_size... */ -}

Experimental API

The experimental API is subject to change or removal without regard - for backward compatibility.

The allocm(), +}

Experimental API

The experimental API is subject to change or removal without regard + for backward compatibility. If --disable-experimental + is specified during configuration, the experimental API is + omitted.

The allocm(), rallocm(), - sallocm(), and - dallocm() functions all have a + sallocm(), + dallocm(), and + nallocm() functions all have a flags argument that can be used to specify options. The functions only check the options that are contextually relevant. Use bitwise or (|) operations to @@ -142,7 +151,9 @@ least size bytes of memory, sets *ptr to the base address of the allocation, and sets *rsize to the real size of the allocation if - rsize is not NULL.

The rallocm() function resizes the + rsize is not NULL. Behavior + is undefined if size is + 0.

The rallocm() function resizes the allocation at *ptr to be at least size bytes, sets *ptr to the base address of the allocation if it moved, and sets @@ -152,12 +163,20 @@ the allocation to be at least size + extra) bytes, though inability to allocate the extra byte(s) will not by itself result in failure. Behavior is - undefined if (size + + undefined if size is 0, or if + (size + extra > SIZE_T_MAX).

The sallocm() function sets *rsize to the real size of the allocation.

The dallocm() function causes the memory referenced by ptr to be made available for - future allocations.

TUNING

Once, when the first call is made to one of the memory allocation + future allocations.

The nallocm() function allocates no + memory, but it performs the same size computation as the + allocm() function, and if + rsize is not NULL it sets + *rsize to the real size of the allocation that + would result from the equivalent allocm() + function call. Behavior is undefined if + size is 0.

TUNING

Once, when the first call is made to one of the memory allocation routines, the allocator initializes its internals based in part on various options that can be specified at compile- or run-time.

The string pointed to by the global variable malloc_conf, the “name” of the file @@ -180,8 +199,8 @@ suboptimal for several reasons, including race conditions, increased fragmentation, and artificial limitations on maximum usable memory. If --enable-dss is specified during configuration, this - allocator uses both sbrk(2) and - mmap(2), in that order of preference; + allocator uses both mmap(2) and + sbrk(2), in that order of preference; otherwise only mmap(2) is used.

This allocator uses multiple arenas in order to reduce lock contention for threaded programs on multi-processor systems. This works well with regard to threading scalability, but incurs some costs. There is @@ -212,26 +231,14 @@ large object). The combination of chunk alignment and chunk page maps makes it possible to determine all metadata regarding small and large allocations in constant time.

Small objects are managed in groups by page runs. Each run maintains - a frontier and free list to track which regions are in use. Unless - --disable-tiny is specified during configuration, - allocation requests that are no more than half the quantum (8 or 16, - depending on architecture) are rounded up to the nearest power of two that - is at least sizeof(void *). - Allocation requests that are more than half the quantum, but no more than - the minimum cacheline-multiple size class (see the - "opt.lg_qspace_max" - - option) are rounded up to the nearest multiple of the quantum. Allocation - requests that are more than the minimum cacheline-multiple size class, but - no more than the minimum subpage-multiple size class (see the - "opt.lg_cspace_max" - - option) are rounded up to the nearest multiple of the cacheline size (64). - Allocation requests that are more than the minimum subpage-multiple size - class, but no more than the maximum subpage-multiple size class are rounded - up to the nearest multiple of the subpage size (256). Allocation requests - that are more than the maximum subpage-multiple size class, but small - enough to fit in an arena-managed chunk (see the + a frontier and free list to track which regions are in use. Allocation + requests that are no more than half the quantum (8 or 16, depending on + architecture) are rounded up to the nearest power of two that is at least + sizeof(double). All other small + object size classes are multiples of the quantum, spaced such that internal + fragmentation is limited to approximately 25% for all but the smallest size + classes. Allocation requests that are larger than the maximum small size + class, but small enough to fit in an arena-managed chunk (see the "opt.lg_chunk" option), are rounded up to the nearest run size. Allocation requests that are too large @@ -241,7 +248,7 @@ suffer from cacheline sharing, round your allocation requests up to the nearest multiple of the cacheline size, or specify cacheline alignment when allocating.

Assuming 4 MiB chunks, 4 KiB pages, and a 16-byte quantum on a 64-bit - system, the size classes in each category are as shown in Table 1.

Table 1. Size classes

CategorySubcategorySize
SmallTiny[8]
Quantum-spaced[16, 32, 48, ..., 128]
Cacheline-spaced[192, 256, 320, ..., 512]
Subpage-spaced[768, 1024, 1280, ..., 3840]
Large[4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB]
Huge[4 MiB, 8 MiB, 12 MiB, ...]

MALLCTL NAMESPACE

The following names are defined in the namespace accessible via the + system, the size classes in each category are as shown in Table 1.

Table 1. Size classes

CategorySpacingSize
Smalllg[8]
16[16, 32, 48, ..., 128]
32[160, 192, 224, 256]
64[320, 384, 448, 512]
128[640, 768, 896, 1024]
256[1280, 1536, 1792, 2048]
512[2560, 3072, 3584]
Large4 KiB[4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB]
Huge4 MiB[4 MiB, 8 MiB, 12 MiB, ...]

MALLCTL NAMESPACE

The following names are defined in the namespace accessible via the mallctl*() functions. Value types are specified in parentheses, their readable/writable statuses are encoded as rw, r-, -w, or @@ -290,13 +297,6 @@

--enable-dss was specified during build configuration.

- "config.dynamic_page_shift" - - (bool) - r- -

--enable-dynamic-page-shift was - specified during build configuration.

- "config.fill" (bool) @@ -311,6 +311,20 @@

--enable-lazy-lock was specified during build configuration.

+ "config.mremap" + + (bool) + r- +

--enable-mremap was specified during + build configuration.

+ + "config.munmap" + + (bool) + r- +

--enable-munmap was specified during + build configuration.

+ "config.prof" (bool) @@ -339,39 +353,32 @@

--enable-stats was specified during build configuration.

- "config.swap" + "config.tcache" (bool) r- -

--enable-swap was specified during - build configuration.

+

--disable-tcache was not specified + during build configuration.

- "config.sysv" + "config.tls" (bool) r- -

--enable-sysv was specified during +

--disable-tls was not specified during build configuration.

- "config.tcache" - - (bool) - r- -

--disable-tcache was not specified - during build configuration.

- - "config.tiny" + "config.utrace" (bool) r- -

--disable-tiny was not specified - during build configuration.

+

--enable-utrace was specified during + build configuration.

- "config.tls" + "config.valgrind" (bool) r- -

--disable-tls was not specified during +

--enable-valgrind was specified during build configuration.

"config.xmalloc" @@ -390,25 +397,7 @@ abort(3) in these cases. This option is disabled by default unless --enable-debug is specified during configuration, in which case it is enabled by default. -

- - "opt.lg_qspace_max" - - (size_t) - r- -

Size (log base 2) of the maximum size class that is a - multiple of the quantum (8 or 16 bytes, depending on architecture). - Above this size, cacheline spacing is used for size classes. The - default value is 128 bytes (2^7).

- - "opt.lg_cspace_max" - - (size_t) - r- -

Size (log base 2) of the maximum size class that is a - multiple of the cacheline size (64). Above this size, subpage spacing - (256 bytes) is used for size classes. The default value is 512 bytes - (2^9).

+

"opt.lg_chunk" @@ -465,7 +454,42 @@ 0x5a. This is intended for debugging and will impact performance negatively. This option is disabled by default unless --enable-debug is specified during - configuration, in which case it is enabled by default.

+ configuration, in which case it is enabled by default.

+ + "opt.quarantine" + + (size_t) + r- + [--enable-fill] +

Per thread quarantine size in bytes. If non-zero, each + thread maintains a FIFO object quarantine that stores up to the + specified number of bytes of memory. The quarantined memory is not + freed until it is released from quarantine, though it is immediately + junk-filled if the + "opt.junk" + option is + enabled. This feature is of particular use in combination with Valgrind, which can detect attempts + to access quarantined objects. This is intended for debugging and will + impact performance negatively. The default quarantine size is + 0.

+ + "opt.redzone" + + (bool) + r- + [--enable-fill] +

Redzones enabled/disabled. If enabled, small + allocations have redzones before and after them. Furthermore, if the + + "opt.junk" + option is + enabled, the redzones are checked for corruption during deallocation. + However, the primary intended purpose of this feature is to be used in + combination with Valgrind, + which needs redzones in order to do effective buffer overflow/underflow + detection. This option is intended for debugging and will impact + performance negatively. This option is disabled by + default.

"opt.zero" @@ -479,21 +503,38 @@ rallocm() calls do not zero memory that was previously allocated. This is intended for debugging and will impact performance negatively. This option is disabled by default. -

+

- "opt.sysv" + "opt.utrace" (bool) r- - [--enable-sysv] -

If enabled, attempting to allocate zero bytes will - return a NULL pointer instead of a valid pointer. - (The default behavior is to make a minimal allocation and return a - pointer to it.) This option is provided for System V compatibility. - This option is incompatible with the - "opt.xmalloc" - option. - This option is disabled by default.

+ [--enable-utrace] +

Allocation tracing based on + utrace(2) enabled/disabled. This option + is disabled by default.

+ + "opt.valgrind" + + (bool) + r- + [--enable-valgrind] +

Valgrind + support enabled/disabled. If enabled, several other options are + automatically modified during options processing to work well with + Valgrind: + "opt.junk" + + and + "opt.zero" + are set + to false, + "opt.quarantine" + is + set to 16 MiB, and + "opt.redzone" + is set to + true. This option is disabled by default.

"opt.xmalloc" @@ -521,27 +562,11 @@ objects up to a certain size. Thread-specific caching allows many allocations to be satisfied without performing any thread synchronization, at the cost of increased memory use. See the - - "opt.lg_tcache_gc_sweep" - - and + "opt.lg_tcache_max" - options for related tuning information. This option is enabled by - default.

- - "opt.lg_tcache_gc_sweep" - - (ssize_t) - r- - [--enable-tcache] -

Approximate interval (log base 2) between full - thread-specific cache garbage collection sweeps, counted in terms of - thread-specific cache allocation/deallocation events. Garbage - collection is actually performed incrementally, one size class at a - time, in order to avoid large collection pauses. The default sweep - interval is 8192 (2^13); setting this option to -1 will disable garbage - collection.

+ option for related tuning information. This option is enabled by + default.

"opt.lg_tcache_max" @@ -559,17 +584,7 @@ r- [--enable-prof]

Memory profiling enabled/disabled. If enabled, profile - memory allocation activity, and use an - atexit(3) function to dump final memory - usage to a file named according to the pattern - <prefix>.<pid>.<seq>.f.heap, - where <prefix> is controlled by the - "opt.prof_prefix" - - option. See the - "opt.lg_prof_bt_max" - - option for backtrace depth control. See the + memory allocation activity. See the "opt.prof_active" option for on-the-fly activation/deactivation. See the @@ -578,19 +593,19 @@ option for probabilistic sampling control. See the "opt.prof_accum" - option for control of cumulative sample reporting. See the - "opt.lg_prof_tcmax" - - option for control of per thread backtrace caching. See the + option for control of cumulative sample reporting. See the "opt.lg_prof_interval" - option for information on interval-triggered profile dumping, and the - + option for information on interval-triggered profile dumping, the "opt.prof_gdump" - option for information on high-water-triggered profile dumping. - Profile output is compatible with the included pprof - Perl script, which originates from the google-perftools + option for information on high-water-triggered profile dumping, and the + + "opt.prof_final" + + option for final profile dumping. Profile output is compatible with + the included pprof Perl script, which originates + from the gperftools package.

"opt.prof_prefix" @@ -602,15 +617,7 @@ set to the empty string, no automatic dumps will occur; this is primarily useful for disabling the automatic final heap dump (which also disables leak reporting, if enabled). The default prefix is - jeprof.

- - "opt.lg_prof_bt_max" - - (size_t) - r- - [--enable-prof] -

Maximum backtrace depth (log base 2) when profiling - memory allocation activity. The default is 128 (2^7).

+ jeprof.

"opt.prof_active" @@ -636,8 +643,8 @@

Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity. Increasing the sampling interval decreases profile fidelity, but also decreases the - computational overhead. The default sample interval is 1 (2^0) (i.e. - all allocations are sampled).

+ computational overhead. The default sample interval is 512 KiB (2^19 + B).

"opt.prof_accum" @@ -648,28 +655,8 @@ dumps enabled/disabled. If this option is enabled, every unique backtrace must be stored for the duration of execution. Depending on the application, this can impose a large memory overhead, and the - cumulative counts are not always of interest. See the - - "opt.lg_prof_tcmax" - - option for control of per thread backtrace caching, which has important - interactions. This option is enabled by default.

- - "opt.lg_prof_tcmax" - - (ssize_t) - r- - [--enable-prof] -

Maximum per thread backtrace cache (log base 2) used - for heap profiling. A backtrace can only be discarded if the - - "opt.prof_accum" - - option is disabled, and no thread caches currently refer to the - backtrace. Therefore, a backtrace cache limit should be imposed if the - intention is to limit how much memory is used by backtraces. By - default, no limit is imposed (encoded as -1). -

+ cumulative counts are not always of interest. This option is disabled + by default.

"opt.lg_prof_interval" @@ -702,7 +689,21 @@ where <prefix> is controlled by the "opt.prof_prefix" - option. This option is disabled by default.

+ option. This option is disabled by default.

+ + "opt.prof_final" + + (bool) + r- + [--enable-prof] +

Use an + atexit(3) function to dump final memory + usage to a file named according to the pattern + <prefix>.<pid>.<seq>.f.heap, + where <prefix> is controlled by the + "opt.prof_prefix" + + option. This option is enabled by default.

"opt.prof_leak" @@ -712,45 +713,11 @@

Leak reporting enabled/disabled. If enabled, use an atexit(3) function to report memory leaks detected by allocation sampling. See the - - "opt.lg_prof_bt_max" - - option for backtrace depth control. See the "opt.prof" option for information on analyzing heap profile output. This option is disabled - by default.

- - "opt.overcommit" - - (bool) - r- - [--enable-swap] -

Over-commit enabled/disabled. If enabled, over-commit - memory as a side effect of using anonymous - mmap(2) or - sbrk(2) for virtual memory allocation. - In order for overcommit to be disabled, the - "swap.fds" - mallctl must have - been successfully written to. This option is enabled by - default.

- - "tcache.flush" - - (void) - -- - [--enable-tcache] -

Flush calling thread's tcache. This interface releases - all cached objects and internal data structures associated with the - calling thread's thread-specific cache. Ordinarily, this interface - need not be called, since automatic periodic incremental garbage - collection occurs, and the thread cache is automatically discarded when - a thread exits. However, garbage collection is triggered by allocation - activity, so it is possible for a thread that stops - allocating/deallocating to retain its cache indefinitely, in which case - the developer may find manual flushing useful.

+ by default.

"thread.arena" @@ -810,7 +777,34 @@ "thread.deallocated" mallctl. This is useful for avoiding the overhead of repeated - mallctl*() calls.

+ mallctl*() calls.

+ + "thread.tcache.enabled" + + (bool) + rw + [--enable-tcache] +

Enable/disable calling thread's tcache. The tcache is + implicitly flushed as a side effect of becoming + disabled (see + "thread.tcache.flush" + ). +

+ + "thread.tcache.flush" + + (void) + -- + [--enable-tcache] +

Flush calling thread's tcache. This interface releases + all cached objects and internal data structures associated with the + calling thread's thread-specific cache. Ordinarily, this interface + need not be called, since automatic periodic incremental garbage + collection occurs, and the thread cache is automatically discarded when + a thread exits. However, garbage collection is triggered by allocation + activity, so it is possible for a thread that stops + allocating/deallocating to retain its cache indefinitely, in which case + the developer may find manual flushing useful.

"arenas.narenas" @@ -834,80 +828,12 @@ r-

Quantum size.

- "arenas.cacheline" - - (size_t) - r- -

Assumed cacheline size.

- - "arenas.subpage" - - (size_t) - r- -

Subpage size class interval.

- - "arenas.pagesize" + "arenas.page" (size_t) r-

Page size.

- "arenas.chunksize" - - (size_t) - r- -

Chunk size.

- - "arenas.tspace_min" - - (size_t) - r- -

Minimum tiny size class. Tiny size classes are powers - of two.

- - "arenas.tspace_max" - - (size_t) - r- -

Maximum tiny size class. Tiny size classes are powers - of two.

- - "arenas.qspace_min" - - (size_t) - r- -

Minimum quantum-spaced size class.

- - "arenas.qspace_max" - - (size_t) - r- -

Maximum quantum-spaced size class.

- - "arenas.cspace_min" - - (size_t) - r- -

Minimum cacheline-spaced size class.

- - "arenas.cspace_max" - - (size_t) - r- -

Maximum cacheline-spaced size class.

- - "arenas.sspace_min" - - (size_t) - r- -

Minimum subpage-spaced size class.

- - "arenas.sspace_max" - - (size_t) - r- -

Maximum subpage-spaced size class.

- "arenas.tcache_max" (size_t) @@ -915,38 +841,11 @@ [--enable-tcache]

Maximum thread-cached size class.

- "arenas.ntbins" - - (unsigned) - r- -

Number of tiny bin size classes.

- - "arenas.nqbins" - - (unsigned) - r- -

Number of quantum-spaced bin size - classes.

- - "arenas.ncbins" - - (unsigned) - r- -

Number of cacheline-spaced bin size - classes.

- - "arenas.nsbins" - - (unsigned) - r- -

Number of subpage-spaced bin size - classes.

- "arenas.nbins" (unsigned) r- -

Total number of bin size classes.

+

Number of bin size classes.

"arenas.nhbins" @@ -1079,8 +978,7 @@ large as "stats.active" . This - does not include inactive chunks backed by swap files. his does not - include inactive chunks embedded in the DSS.

+ does not include inactive chunks.

"stats.chunks.current" @@ -1088,8 +986,7 @@ r- [--enable-stats]

Total number of chunks actively mapped on behalf of the - application. This does not include inactive chunks backed by swap - files. This does not include inactive chunks embedded in the DSS. + application. This does not include inactive chunks.

"stats.chunks.total" @@ -1309,14 +1206,6 @@

Cumulative number of times the current run from which to allocate changed.

- "stats.arenas.<i>.bins.<j>.highruns" - - (size_t) - r- - [--enable-stats] -

Maximum number of runs at any time thus far. -

- "stats.arenas.<i>.bins.<j>.curruns" (size_t) @@ -1348,69 +1237,13 @@

Cumulative number of allocation requests for this size class.

- "stats.arenas.<i>.lruns.<j>.highruns" - - (size_t) - r- - [--enable-stats] -

Maximum number of runs at any time thus far for this - size class.

- "stats.arenas.<i>.lruns.<j>.curruns" (size_t) r- [--enable-stats]

Current number of runs for this size class. -

- - "swap.avail" - - (size_t) - r- - [--enable-stats --enable-swap] -

Number of swap file bytes that are currently not - associated with any chunk (i.e. mapped, but otherwise completely - unmanaged).

- - "swap.prezeroed" - - (bool) - rw - [--enable-swap] -

If true, the allocator assumes that the swap file(s) - contain nothing but nil bytes. If this assumption is violated, - allocator behavior is undefined. This value becomes read-only after - - "swap.fds" - is - successfully written to.

- - "swap.nfds" - - (size_t) - r- - [--enable-swap] -

Number of file descriptors in use for swap. -

- - "swap.fds" - - (int *) - rw - [--enable-swap] -

When written to, the files associated with the - specified file descriptors are contiguously mapped via - mmap(2). The resulting virtual memory - region is preferred over anonymous - mmap(2) and - sbrk(2) memory. Note that if a file's - size is not a multiple of the page size, it is automatically truncated - to the nearest page size multiple. See the - - "swap.prezeroed" - - mallctl for specifying that the files are pre-zeroed.

DEBUGGING MALLOC PROBLEMS

When debugging, it is a good idea to configure/build jemalloc with +

DEBUGGING MALLOC PROBLEMS

When debugging, it is a good idea to configure/build jemalloc with the --enable-debug and --enable-fill options, and recompile the program with suitable options and symbols for debugger support. When so configured, jemalloc incorporates a wide variety @@ -1428,10 +1261,13 @@ the symptoms of such bugs. Between these two options, it is usually possible to quickly detect, diagnose, and eliminate such bugs.

This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information - would be prohibitive. There are a number of allocator implementations - available on the Internet which focus on detecting and pinpointing problems - by trading performance for extra sanity checks and detailed - diagnostics.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an + would be prohibitive. However, jemalloc does integrate with the most + excellent Valgrind tool if the + --enable-valgrind configuration option is enabled and the + + "opt.valgrind" + option + is enabled.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an error or warning condition, a message will be printed to file descriptor STDERR_FILENO. Errors will result in the process dumping core. If the @@ -1447,7 +1283,7 @@ malloc_stats_print(), followed by a string pointer. Please note that doing anything which tries to allocate memory in this function is likely to result in a crash or deadlock.

All messages are prefixed by - “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and + “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and calloc() functions return a pointer to the allocated memory if successful; otherwise a NULL pointer is returned and errno is set to @@ -1459,6 +1295,14 @@ not a power of 2 at least as large as sizeof(void *).

ENOMEM

Memory allocation error.

+

The aligned_alloc() function returns + a pointer to the allocated memory if successful; otherwise a + NULL pointer is returned and + errno is set. The + aligned_alloc() function will fail if: +

EINVAL

The alignment parameter is + not a power of 2. +

ENOMEM

Memory allocation error.

The realloc() function returns a pointer, possibly identical to ptr, to the allocated memory if successful; otherwise a NULL @@ -1467,7 +1311,7 @@ allocation failure. The realloc() function always leaves the original buffer intact when an error occurs.

The free() function returns no - value.

Non-standard API

The malloc_usable_size() function + value.

Non-standard API

The malloc_usable_size() function returns the usable size of the allocation pointed to by ptr.

The mallctl(), mallctlnametomib(), and @@ -1486,13 +1330,15 @@ occurred.

EFAULT

An interface with side effects failed in some way not directly related to mallctl*() read/write processing.

-

Experimental API

The allocm(), +

Experimental API

The allocm(), rallocm(), - sallocm(), and - dallocm() functions return + sallocm(), + dallocm(), and + nallocm() functions return ALLOCM_SUCCESS on success; otherwise they return an - error value. The allocm() and - rallocm() functions will fail if: + error value. The allocm(), + rallocm(), and + nallocm() functions will fail if:

ALLOCM_ERR_OOM

Out of memory. Insufficient contiguous memory was available to service the allocation request. The allocm() function additionally sets @@ -1516,6 +1362,7 @@ malloc_conf = "lg_chunk:24";

SEE ALSO

madvise(2), mmap(2), sbrk(2), + utrace(2), alloca(3), atexit(3), getpagesize(3)

STANDARDS

The malloc(), diff --git a/deps/jemalloc/doc/jemalloc.xml.in b/deps/jemalloc/doc/jemalloc.xml.in index 7a32879abee..877c500f36d 100644 --- a/deps/jemalloc/doc/jemalloc.xml.in +++ b/deps/jemalloc/doc/jemalloc.xml.in @@ -30,6 +30,7 @@ malloc calloc posix_memalign + aligned_alloc realloc free malloc_usable_size @@ -41,6 +42,7 @@ rallocm sallocm dallocm + nallocm --> general purpose memory allocation functions @@ -72,6 +74,11 @@ size_t alignment size_t size + + void *aligned_alloc + size_t alignment + size_t size + void *realloc void *ptr @@ -154,6 +161,12 @@ void *ptr int flags + + int nallocm + size_t *rsize + size_t size + int flags + @@ -183,6 +196,14 @@ alignment must be a power of 2 at least as large as sizeof(void *). + The aligned_alloc function + allocates size bytes of memory such that the + allocation's base address is an even multiple of + alignment. The requested + alignment must be a power of 2. Behavior is + undefined if size is not an integral multiple of + alignment. + The realloc function changes the size of the previously allocated memory referenced by ptr to size bytes. The @@ -297,12 +318,15 @@ for (i = 0; i < nbins; i++) { Experimental API The experimental API is subject to change or removal without regard - for backward compatibility. + for backward compatibility. If + is specified during configuration, the experimental API is + omitted. The allocm, rallocm, - sallocm, and - dallocm functions all have a + sallocm, + dallocm, and + nallocm functions all have a flags argument that can be used to specify options. The functions only check the options that are contextually relevant. Use bitwise or (|) operations to @@ -351,7 +375,9 @@ for (i = 0; i < nbins; i++) { least size bytes of memory, sets *ptr to the base address of the allocation, and sets *rsize to the real size of the allocation if - rsize is not NULL. + rsize is not NULL. Behavior + is undefined if size is + 0. The rallocm function resizes the allocation at *ptr to be at least @@ -364,7 +390,8 @@ for (i = 0; i < nbins; i++) { language="C">size + extra) bytes, though inability to allocate the extra byte(s) will not by itself result in failure. Behavior is - undefined if (size + + undefined if size is 0, or if + (size + extra > SIZE_T_MAX). @@ -374,6 +401,15 @@ for (i = 0; i < nbins; i++) { The dallocm function causes the memory referenced by ptr to be made available for future allocations. + + The nallocm function allocates no + memory, but it performs the same size computation as the + allocm function, and if + rsize is not NULL it sets + *rsize to the real size of the allocation that + would result from the equivalent allocm + function call. Behavior is undefined if + size is 0. @@ -408,9 +444,9 @@ for (i = 0; i < nbins; i++) { suboptimal for several reasons, including race conditions, increased fragmentation, and artificial limitations on maximum usable memory. If is specified during configuration, this - allocator uses both sbrk + allocator uses both mmap 2 and - mmap + sbrk 2, in that order of preference; otherwise only mmap 2 is used. @@ -455,24 +491,14 @@ for (i = 0; i < nbins; i++) { allocations in constant time. Small objects are managed in groups by page runs. Each run maintains - a frontier and free list to track which regions are in use. Unless - is specified during configuration, - allocation requests that are no more than half the quantum (8 or 16, - depending on architecture) are rounded up to the nearest power of two that - is at least sizeof(void *). - Allocation requests that are more than half the quantum, but no more than - the minimum cacheline-multiple size class (see the opt.lg_qspace_max - option) are rounded up to the nearest multiple of the quantum. Allocation - requests that are more than the minimum cacheline-multiple size class, but - no more than the minimum subpage-multiple size class (see the opt.lg_cspace_max - option) are rounded up to the nearest multiple of the cacheline size (64). - Allocation requests that are more than the minimum subpage-multiple size - class, but no more than the maximum subpage-multiple size class are rounded - up to the nearest multiple of the subpage size (256). Allocation requests - that are more than the maximum subpage-multiple size class, but small - enough to fit in an arena-managed chunk (see the sizeof(double). All other small + object size classes are multiples of the quantum, spaced such that internal + fragmentation is limited to approximately 25% for all but the smallest size + classes. Allocation requests that are larger than the maximum small size + class, but small enough to fit in an arena-managed chunk (see the opt.lg_chunk option), are rounded up to the nearest run size. Allocation requests that are too large to fit in an arena-managed chunk are rounded up to the nearest multiple of @@ -490,41 +516,55 @@ for (i = 0; i < nbins; i++) { Size classes - - - - + + + + Category - Subcategory + Spacing Size - Small - Tiny + Small + lg [8] - Quantum-spaced + 16 [16, 32, 48, ..., 128] - Cacheline-spaced - [192, 256, 320, ..., 512] + 32 + [160, 192, 224, 256] - Subpage-spaced - [768, 1024, 1280, ..., 3840] + 64 + [320, 384, 448, 512] - Large + 128 + [640, 768, 896, 1024] + + + 256 + [1280, 1536, 1792, 2048] + + + 512 + [2560, 3072, 3584] + + + Large + 4 KiB [4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB] - Huge + Huge + 4 MiB [4 MiB, 8 MiB, 12 MiB, ...] @@ -592,32 +632,42 @@ for (i = 0; i < nbins; i++) { - config.dynamic_page_shift + config.fill (bool) r- - was - specified during build configuration. + was specified during + build configuration. - config.fill + config.lazy_lock (bool) r- - was specified during + was specified + during build configuration. + + + + + config.mremap + (bool) + r- + + was specified during build configuration. - config.lazy_lock + config.munmap (bool) r- - was specified - during build configuration. + was specified during + build configuration. @@ -662,51 +712,41 @@ for (i = 0; i < nbins; i++) { - config.swap + config.tcache (bool) r- - was specified during - build configuration. + was not specified + during build configuration. - config.sysv + config.tls (bool) r- - was specified during + was not specified during build configuration. - config.tcache - (bool) - r- - - was not specified - during build configuration. - - - - - config.tiny + config.utrace (bool) r- - was not specified - during build configuration. + was specified during + build configuration. - config.tls + config.valgrind (bool) r- - was not specified during + was specified during build configuration. @@ -735,30 +775,6 @@ for (i = 0; i < nbins; i++) { - - - opt.lg_qspace_max - (size_t) - r- - - Size (log base 2) of the maximum size class that is a - multiple of the quantum (8 or 16 bytes, depending on architecture). - Above this size, cacheline spacing is used for size classes. The - default value is 128 bytes (2^7). - - - - - opt.lg_cspace_max - (size_t) - r- - - Size (log base 2) of the maximum size class that is a - multiple of the cacheline size (64). Above this size, subpage spacing - (256 bytes) is used for size classes. The default value is 512 bytes - (2^9). - - opt.lg_chunk @@ -833,6 +849,45 @@ for (i = 0; i < nbins; i++) { configuration, in which case it is enabled by default. + + + opt.quarantine + (size_t) + r- + [] + + Per thread quarantine size in bytes. If non-zero, each + thread maintains a FIFO object quarantine that stores up to the + specified number of bytes of memory. The quarantined memory is not + freed until it is released from quarantine, though it is immediately + junk-filled if the opt.junk option is + enabled. This feature is of particular use in combination with Valgrind, which can detect attempts + to access quarantined objects. This is intended for debugging and will + impact performance negatively. The default quarantine size is + 0. + + + + + opt.redzone + (bool) + r- + [] + + Redzones enabled/disabled. If enabled, small + allocations have redzones before and after them. Furthermore, if the + opt.junk option is + enabled, the redzones are checked for corruption during deallocation. + However, the primary intended purpose of this feature is to be used in + combination with Valgrind, + which needs redzones in order to do effective buffer overflow/underflow + detection. This option is intended for debugging and will impact + performance negatively. This option is disabled by + default. + + opt.zero @@ -850,20 +905,36 @@ for (i = 0; i < nbins; i++) { - + - opt.sysv + opt.utrace (bool) r- - [] + [] - If enabled, attempting to allocate zero bytes will - return a NULL pointer instead of a valid pointer. - (The default behavior is to make a minimal allocation and return a - pointer to it.) This option is provided for System V compatibility. - This option is incompatible with the opt.xmalloc option. - This option is disabled by default. + Allocation tracing based on + utrace + 2 enabled/disabled. This option + is disabled by default. + + + + + opt.valgrind + (bool) + r- + [] + + Valgrind + support enabled/disabled. If enabled, several other options are + automatically modified during options processing to work well with + Valgrind: opt.junk + and opt.zero are set + to false, opt.quarantine is + set to 16 MiB, and opt.redzone is set to + true. This option is disabled by default. @@ -899,29 +970,11 @@ malloc_conf = "xmalloc:true";]]> allocations to be satisfied without performing any thread synchronization, at the cost of increased memory use. See the opt.lg_tcache_gc_sweep - and opt.lg_tcache_max - options for related tuning information. This option is enabled by + option for related tuning information. This option is enabled by default. - - - opt.lg_tcache_gc_sweep - (ssize_t) - r- - [] - - Approximate interval (log base 2) between full - thread-specific cache garbage collection sweeps, counted in terms of - thread-specific cache allocation/deallocation events. Garbage - collection is actually performed incrementally, one size class at a - time, in order to avoid large collection pauses. The default sweep - interval is 8192 (2^13); setting this option to -1 will disable garbage - collection. - - opt.lg_tcache_max @@ -943,31 +996,21 @@ malloc_conf = "xmalloc:true";]]> [] Memory profiling enabled/disabled. If enabled, profile - memory allocation activity, and use an - atexit - 3 function to dump final memory - usage to a file named according to the pattern - <prefix>.<pid>.<seq>.f.heap, - where <prefix> is controlled by the opt.prof_prefix - option. See the opt.lg_prof_bt_max - option for backtrace depth control. See the opt.prof_active option for on-the-fly activation/deactivation. See the opt.lg_prof_sample option for probabilistic sampling control. See the opt.prof_accum option for control of cumulative sample reporting. See the opt.lg_prof_tcmax - option for control of per thread backtrace caching. See the opt.lg_prof_interval - option for information on interval-triggered profile dumping, and the - opt.prof_gdump - option for information on high-water-triggered profile dumping. - Profile output is compatible with the included pprof - Perl script, which originates from the google-perftools + option for information on interval-triggered profile dumping, the opt.prof_gdump + option for information on high-water-triggered profile dumping, and the + opt.prof_final + option for final profile dumping. Profile output is compatible with + the included pprof Perl script, which originates + from the gperftools package. @@ -985,17 +1028,6 @@ malloc_conf = "xmalloc:true";]]> jeprof. - - - opt.lg_prof_bt_max - (size_t) - r- - [] - - Maximum backtrace depth (log base 2) when profiling - memory allocation activity. The default is 128 (2^7). - - opt.prof_active @@ -1023,8 +1055,8 @@ malloc_conf = "xmalloc:true";]]> Average interval (log base 2) between allocation samples, as measured in bytes of allocation activity. Increasing the sampling interval decreases profile fidelity, but also decreases the - computational overhead. The default sample interval is 1 (2^0) (i.e. - all allocations are sampled). + computational overhead. The default sample interval is 512 KiB (2^19 + B). @@ -1038,28 +1070,8 @@ malloc_conf = "xmalloc:true";]]> dumps enabled/disabled. If this option is enabled, every unique backtrace must be stored for the duration of execution. Depending on the application, this can impose a large memory overhead, and the - cumulative counts are not always of interest. See the - opt.lg_prof_tcmax - option for control of per thread backtrace caching, which has important - interactions. This option is enabled by default. - - - - - opt.lg_prof_tcmax - (ssize_t) - r- - [] - - Maximum per thread backtrace cache (log base 2) used - for heap profiling. A backtrace can only be discarded if the - opt.prof_accum - option is disabled, and no thread caches currently refer to the - backtrace. Therefore, a backtrace cache limit should be imposed if the - intention is to limit how much memory is used by backtraces. By - default, no limit is imposed (encoded as -1). - + cumulative counts are not always of interest. This option is disabled + by default. @@ -1099,6 +1111,23 @@ malloc_conf = "xmalloc:true";]]> option. This option is disabled by default. + + + opt.prof_final + (bool) + r- + [] + + Use an + atexit + 3 function to dump final memory + usage to a file named according to the pattern + <prefix>.<pid>.<seq>.f.heap, + where <prefix> is controlled by the opt.prof_prefix + option. This option is enabled by default. + + opt.prof_leak @@ -1110,51 +1139,11 @@ malloc_conf = "xmalloc:true";]]> atexit 3 function to report memory leaks detected by allocation sampling. See the - opt.lg_prof_bt_max - option for backtrace depth control. See the opt.prof option for information on analyzing heap profile output. This option is disabled by default. - - - opt.overcommit - (bool) - r- - [] - - Over-commit enabled/disabled. If enabled, over-commit - memory as a side effect of using anonymous - mmap - 2 or - sbrk - 2 for virtual memory allocation. - In order for overcommit to be disabled, the swap.fds mallctl must have - been successfully written to. This option is enabled by - default. - - - - - tcache.flush - (void) - -- - [] - - Flush calling thread's tcache. This interface releases - all cached objects and internal data structures associated with the - calling thread's thread-specific cache. Ordinarily, this interface - need not be called, since automatic periodic incremental garbage - collection occurs, and the thread cache is automatically discarded when - a thread exits. However, garbage collection is triggered by allocation - activity, so it is possible for a thread that stops - allocating/deallocating to retain its cache indefinitely, in which case - the developer may find manual flushing useful. - - thread.arena @@ -1226,6 +1215,38 @@ malloc_conf = "xmalloc:true";]]> mallctl* calls. + + + thread.tcache.enabled + (bool) + rw + [] + + Enable/disable calling thread's tcache. The tcache is + implicitly flushed as a side effect of becoming + disabled (see thread.tcache.flush). + + + + + + thread.tcache.flush + (void) + -- + [] + + Flush calling thread's tcache. This interface releases + all cached objects and internal data structures associated with the + calling thread's thread-specific cache. Ordinarily, this interface + need not be called, since automatic periodic incremental garbage + collection occurs, and the thread cache is automatically discarded when + a thread exits. However, garbage collection is triggered by allocation + activity, so it is possible for a thread that stops + allocating/deallocating to retain its cache indefinitely, in which case + the developer may find manual flushing useful. + + arenas.narenas @@ -1258,114 +1279,13 @@ malloc_conf = "xmalloc:true";]]> - arenas.cacheline - (size_t) - r- - - Assumed cacheline size. - - - - - arenas.subpage - (size_t) - r- - - Subpage size class interval. - - - - - arenas.pagesize + arenas.page (size_t) r- Page size. - - - arenas.chunksize - (size_t) - r- - - Chunk size. - - - - - arenas.tspace_min - (size_t) - r- - - Minimum tiny size class. Tiny size classes are powers - of two. - - - - - arenas.tspace_max - (size_t) - r- - - Maximum tiny size class. Tiny size classes are powers - of two. - - - - - arenas.qspace_min - (size_t) - r- - - Minimum quantum-spaced size class. - - - - - arenas.qspace_max - (size_t) - r- - - Maximum quantum-spaced size class. - - - - - arenas.cspace_min - (size_t) - r- - - Minimum cacheline-spaced size class. - - - - - arenas.cspace_max - (size_t) - r- - - Maximum cacheline-spaced size class. - - - - - arenas.sspace_min - (size_t) - r- - - Minimum subpage-spaced size class. - - - - - arenas.sspace_max - (size_t) - r- - - Maximum subpage-spaced size class. - - arenas.tcache_max @@ -1376,52 +1296,13 @@ malloc_conf = "xmalloc:true";]]> Maximum thread-cached size class. - - - arenas.ntbins - (unsigned) - r- - - Number of tiny bin size classes. - - - - - arenas.nqbins - (unsigned) - r- - - Number of quantum-spaced bin size - classes. - - - - - arenas.ncbins - (unsigned) - r- - - Number of cacheline-spaced bin size - classes. - - - - - arenas.nsbins - (unsigned) - r- - - Number of subpage-spaced bin size - classes. - - arenas.nbins (unsigned) r- - Total number of bin size classes. + Number of bin size classes. @@ -1590,8 +1471,7 @@ malloc_conf = "xmalloc:true";]]> application. This is a multiple of the chunk size, and is at least as large as stats.active. This - does not include inactive chunks backed by swap files. his does not - include inactive chunks embedded in the DSS. + does not include inactive chunks. @@ -1602,8 +1482,7 @@ malloc_conf = "xmalloc:true";]]> [] Total number of chunks actively mapped on behalf of the - application. This does not include inactive chunks backed by swap - files. This does not include inactive chunks embedded in the DSS. + application. This does not include inactive chunks. @@ -1908,17 +1787,6 @@ malloc_conf = "xmalloc:true";]]> to allocate changed. - - - stats.arenas.<i>.bins.<j>.highruns - (size_t) - r- - [] - - Maximum number of runs at any time thus far. - - - stats.arenas.<i>.bins.<j>.curruns @@ -1962,17 +1830,6 @@ malloc_conf = "xmalloc:true";]]> class. - - - stats.arenas.<i>.lruns.<j>.highruns - (size_t) - r- - [] - - Maximum number of runs at any time thus far for this - size class. - - stats.arenas.<i>.lruns.<j>.curruns @@ -1983,65 +1840,6 @@ malloc_conf = "xmalloc:true";]]> Current number of runs for this size class. - - - - swap.avail - (size_t) - r- - [] - - Number of swap file bytes that are currently not - associated with any chunk (i.e. mapped, but otherwise completely - unmanaged). - - - - - swap.prezeroed - (bool) - rw - [] - - If true, the allocator assumes that the swap file(s) - contain nothing but nil bytes. If this assumption is violated, - allocator behavior is undefined. This value becomes read-only after - swap.fds is - successfully written to. - - - - - swap.nfds - (size_t) - r- - [] - - Number of file descriptors in use for swap. - - - - - - swap.fds - (int *) - rw - [] - - When written to, the files associated with the - specified file descriptors are contiguously mapped via - mmap - 2. The resulting virtual memory - region is preferred over anonymous - mmap - 2 and - sbrk - 2 memory. Note that if a file's - size is not a multiple of the page size, it is automatically truncated - to the nearest page size multiple. See the - swap.prezeroed - mallctl for specifying that the files are pre-zeroed. - @@ -2065,10 +1863,11 @@ malloc_conf = "xmalloc:true";]]> This implementation does not provide much detail about the problems it detects, because the performance impact for storing such information - would be prohibitive. There are a number of allocator implementations - available on the Internet which focus on detecting and pinpointing problems - by trading performance for extra sanity checks and detailed - diagnostics. + would be prohibitive. However, jemalloc does integrate with the most + excellent Valgrind tool if the + configuration option is enabled and the + opt.valgrind option + is enabled. DIAGNOSTIC MESSAGES @@ -2124,6 +1923,27 @@ malloc_conf = "xmalloc:true";]]> + The aligned_alloc function returns + a pointer to the allocated memory if successful; otherwise a + NULL pointer is returned and + errno is set. The + aligned_alloc function will fail if: + + + EINVAL + + The alignment parameter is + not a power of 2. + + + + ENOMEM + + Memory allocation error. + + + + The realloc function returns a pointer, possibly identical to ptr, to the allocated memory if successful; otherwise a NULL @@ -2196,11 +2016,13 @@ malloc_conf = "xmalloc:true";]]> Experimental API The allocm, rallocm, - sallocm, and - dallocm functions return + sallocm, + dallocm, and + nallocm functions return ALLOCM_SUCCESS on success; otherwise they return an - error value. The allocm and - rallocm functions will fail if: + error value. The allocm, + rallocm, and + nallocm functions will fail if: ALLOCM_ERR_OOM @@ -2259,6 +2081,8 @@ malloc_conf = "lg_chunk:24";]]> 2, sbrk 2, + utrace + 2, alloca 3, atexit diff --git a/deps/jemalloc/include/jemalloc/internal/arena.h b/deps/jemalloc/include/jemalloc/internal/arena.h index b80c118d811..0b0f640a459 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena.h +++ b/deps/jemalloc/include/jemalloc/internal/arena.h @@ -1,41 +1,6 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -/* - * Subpages are an artificially designated partitioning of pages. Their only - * purpose is to support subpage-spaced size classes. - * - * There must be at least 4 subpages per page, due to the way size classes are - * handled. - */ -#define LG_SUBPAGE 8 -#define SUBPAGE ((size_t)(1U << LG_SUBPAGE)) -#define SUBPAGE_MASK (SUBPAGE - 1) - -/* Return the smallest subpage multiple that is >= s. */ -#define SUBPAGE_CEILING(s) \ - (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK) - -#ifdef JEMALLOC_TINY - /* Smallest size class to support. */ -# define LG_TINY_MIN LG_SIZEOF_PTR -# define TINY_MIN (1U << LG_TINY_MIN) -#endif - -/* - * Maximum size class that is a multiple of the quantum, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ -#define LG_QSPACE_MAX_DEFAULT 7 - -/* - * Maximum size class that is a multiple of the cacheline, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ -#define LG_CSPACE_MAX_DEFAULT 9 - /* * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized * as small as possible such that this setting is still honored, without @@ -51,7 +16,7 @@ * constraint is relaxed (ignored) for runs that are so small that the * per-region overhead is greater than: * - * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) + * (RUN_MAX_OVRHD / (reg_interval << (3+RUN_BFP)) */ #define RUN_BFP 12 /* \/ Implicit binary fixed point. */ @@ -62,6 +27,12 @@ #define LG_RUN_MAXREGS 11 #define RUN_MAXREGS (1U << LG_RUN_MAXREGS) +/* + * Minimum redzone size. Redzones may be larger than this if necessary to + * preserve region alignment. + */ +#define REDZONE_MINSIZE 16 + /* * The minimum ratio of active:dirty pages per arena is computed as: * @@ -85,6 +56,15 @@ typedef struct arena_s arena_t; /* Each element of the chunk map corresponds to one page within the chunk. */ struct arena_chunk_map_s { +#ifndef JEMALLOC_PROF + /* + * Overlay prof_ctx in order to allow it to be referenced by dead code. + * Such antics aren't warranted for per arena data structures, but + * chunk map overhead accounts for a percentage of memory, rather than + * being just a fixed cost. + */ + union { +#endif union { /* * Linkage for run trees. There are two disjoint uses: @@ -103,22 +83,23 @@ struct arena_chunk_map_s { ql_elm(arena_chunk_map_t) ql_link; } u; -#ifdef JEMALLOC_PROF /* Profile counters, used for large object runs. */ prof_ctx_t *prof_ctx; +#ifndef JEMALLOC_PROF + }; /* union { ... }; */ #endif /* * Run address (or size) and various flags are stored together. The bit * layout looks like (assuming 32-bit system): * - * ???????? ???????? ????---- ----dula + * ???????? ???????? ????nnnn nnnndula * * ? : Unallocated: Run address for first/last pages, unset for internal * pages. * Small: Run page offset. * Large: Run size for first page, unset for trailing pages. - * - : Unused. + * n : binind for small size class, BININD_INVALID for large size class. * d : dirty? * u : unzeroed? * l : large? @@ -128,7 +109,8 @@ struct arena_chunk_map_s { * * p : run page offset * s : run size - * c : (binind+1) for size class (used only if prof_promote is true) + * n : binind for size class; large objects set these to BININD_INVALID + * except for promoted allocations (see prof_promote) * x : don't care * - : 0 * + : 1 @@ -136,37 +118,38 @@ struct arena_chunk_map_s { * [dula] : bit unset * * Unallocated (clean): - * ssssssss ssssssss ssss---- ----du-a - * xxxxxxxx xxxxxxxx xxxx---- -----Uxx - * ssssssss ssssssss ssss---- ----dU-a + * ssssssss ssssssss ssss++++ ++++du-a + * xxxxxxxx xxxxxxxx xxxxxxxx xxxx-Uxx + * ssssssss ssssssss ssss++++ ++++dU-a * * Unallocated (dirty): - * ssssssss ssssssss ssss---- ----D--a - * xxxxxxxx xxxxxxxx xxxx---- ----xxxx - * ssssssss ssssssss ssss---- ----D--a + * ssssssss ssssssss ssss++++ ++++D--a + * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + * ssssssss ssssssss ssss++++ ++++D--a * * Small: - * pppppppp pppppppp pppp---- ----d--A - * pppppppp pppppppp pppp---- -------A - * pppppppp pppppppp pppp---- ----d--A + * pppppppp pppppppp ppppnnnn nnnnd--A + * pppppppp pppppppp ppppnnnn nnnn---A + * pppppppp pppppppp ppppnnnn nnnnd--A * * Large: - * ssssssss ssssssss ssss---- ----D-LA - * xxxxxxxx xxxxxxxx xxxx---- ----xxxx - * -------- -------- -------- ----D-LA + * ssssssss ssssssss ssss++++ ++++D-LA + * xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx + * -------- -------- ----++++ ++++D-LA * - * Large (sampled, size <= PAGE_SIZE): - * ssssssss ssssssss sssscccc ccccD-LA + * Large (sampled, size <= PAGE): + * ssssssss ssssssss ssssnnnn nnnnD-LA * - * Large (not sampled, size == PAGE_SIZE): - * ssssssss ssssssss ssss---- ----D-LA + * Large (not sampled, size == PAGE): + * ssssssss ssssssss ssss++++ ++++D-LA */ size_t bits; -#ifdef JEMALLOC_PROF -#define CHUNK_MAP_CLASS_SHIFT 4 -#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U) -#endif -#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU) +#define CHUNK_MAP_BININD_SHIFT 4 +#define BININD_INVALID ((size_t)0xffU) +/* CHUNK_MAP_BININD_MASK == (BININD_INVALID << CHUNK_MAP_BININD_SHIFT) */ +#define CHUNK_MAP_BININD_MASK ((size_t)0xff0U) +#define CHUNK_MAP_BININD_INVALID CHUNK_MAP_BININD_MASK +#define CHUNK_MAP_FLAGS_MASK ((size_t)0xcU) #define CHUNK_MAP_DIRTY ((size_t)0x8U) #define CHUNK_MAP_UNZEROED ((size_t)0x4U) #define CHUNK_MAP_LARGE ((size_t)0x2U) @@ -205,11 +188,6 @@ struct arena_chunk_s { typedef rb_tree(arena_chunk_t) arena_chunk_tree_t; struct arena_run_s { -#ifdef JEMALLOC_DEBUG - uint32_t magic; -# define ARENA_RUN_MAGIC 0x384adf93 -#endif - /* Bin this run is associated with. */ arena_bin_t *bin; @@ -224,11 +202,50 @@ struct arena_run_s { * Read-only information associated with each element of arena_t's bins array * is stored separately, partly to reduce memory usage (only one copy, rather * than one per arena), but mainly to avoid false cacheline sharing. + * + * Each run has the following layout: + * + * /--------------------\ + * | arena_run_t header | + * | ... | + * bitmap_offset | bitmap | + * | ... | + * ctx0_offset | ctx map | + * | ... | + * |--------------------| + * | redzone | + * reg0_offset | region 0 | + * | redzone | + * |--------------------| \ + * | redzone | | + * | region 1 | > reg_interval + * | redzone | / + * |--------------------| + * | ... | + * | ... | + * | ... | + * |--------------------| + * | redzone | + * | region nregs-1 | + * | redzone | + * |--------------------| + * | alignment pad? | + * \--------------------/ + * + * reg_interval has at least the same minimum alignment as reg_size; this + * preserves the alignment constraint that sa2u() depends on. Alignment pad is + * either 0 or redzone_size; it is present only if needed to align reg0_offset. */ struct arena_bin_info_s { /* Size of regions in a run for this bin's size class. */ size_t reg_size; + /* Redzone size. */ + size_t redzone_size; + + /* Interval between regions (reg_size + (redzone_size << 1)). */ + size_t reg_interval; + /* Total size of a run for this bin's size class. */ size_t run_size; @@ -247,13 +264,11 @@ struct arena_bin_info_s { */ bitmap_info_t bitmap_info; -#ifdef JEMALLOC_PROF /* * Offset of first (prof_ctx_t *) in a run header for this bin's size - * class, or 0 if (opt_prof == false). + * class, or 0 if (config_prof == false || opt_prof == false). */ uint32_t ctx0_offset; -#endif /* Offset of first region in a run for this bin's size class. */ uint32_t reg0_offset; @@ -283,18 +298,11 @@ struct arena_bin_s { */ arena_run_tree_t runs; -#ifdef JEMALLOC_STATS /* Bin statistics. */ malloc_bin_stats_t stats; -#endif }; struct arena_s { -#ifdef JEMALLOC_DEBUG - uint32_t magic; -# define ARENA_MAGIC 0x947d3d24 -#endif - /* This arena's index within the arenas array. */ unsigned ind; @@ -314,20 +322,14 @@ struct arena_s { */ malloc_mutex_t lock; -#ifdef JEMALLOC_STATS arena_stats_t stats; -# ifdef JEMALLOC_TCACHE /* * List of tcaches for extant threads associated with this arena. * Stats from these are merged incrementally, and at exit. */ ql_head(tcache_t) tcache_ql; -# endif -#endif -#ifdef JEMALLOC_PROF uint64_t prof_accumbytes; -#endif /* List of dirty-page-containing chunks this arena manages. */ ql_head(arena_chunk_t) chunks_dirty; @@ -378,140 +380,334 @@ struct arena_s { arena_avail_tree_t runs_avail_clean; arena_avail_tree_t runs_avail_dirty; - /* - * bins is used to store trees of free regions of the following sizes, - * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and - * default MALLOC_CONF. - * - * bins[i] | size | - * --------+--------+ - * 0 | 8 | - * --------+--------+ - * 1 | 16 | - * 2 | 32 | - * 3 | 48 | - * : : - * 6 | 96 | - * 7 | 112 | - * 8 | 128 | - * --------+--------+ - * 9 | 192 | - * 10 | 256 | - * 11 | 320 | - * 12 | 384 | - * 13 | 448 | - * 14 | 512 | - * --------+--------+ - * 15 | 768 | - * 16 | 1024 | - * 17 | 1280 | - * : : - * 25 | 3328 | - * 26 | 3584 | - * 27 | 3840 | - * --------+--------+ - */ - arena_bin_t bins[1]; /* Dynamically sized. */ + /* bins is used to store trees of free regions. */ + arena_bin_t bins[NBINS]; }; #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -extern size_t opt_lg_qspace_max; -extern size_t opt_lg_cspace_max; extern ssize_t opt_lg_dirty_mult; /* * small_size2bin is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, * and all accesses are via the SMALL_SIZE2BIN macro. */ -extern uint8_t const *small_size2bin; +extern uint8_t const small_size2bin[]; #define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) -extern arena_bin_info_t *arena_bin_info; - -/* Various bin-related settings. */ -#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ -# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN)) -#else -# define ntbins 0 -#endif -extern unsigned nqbins; /* Number of quantum-spaced bins. */ -extern unsigned ncbins; /* Number of cacheline-spaced bins. */ -extern unsigned nsbins; /* Number of subpage-spaced bins. */ -extern unsigned nbins; -#ifdef JEMALLOC_TINY -# define tspace_max ((size_t)(QUANTUM >> 1)) -#endif -#define qspace_min QUANTUM -extern size_t qspace_max; -extern size_t cspace_min; -extern size_t cspace_max; -extern size_t sspace_min; -extern size_t sspace_max; -#define small_maxclass sspace_max +extern arena_bin_info_t arena_bin_info[NBINS]; +/* Number of large size classes. */ #define nlclasses (chunk_npages - map_bias) void arena_purge_all(arena_t *arena); -#ifdef JEMALLOC_PROF void arena_prof_accum(arena_t *arena, uint64_t accumbytes); -#endif -#ifdef JEMALLOC_TCACHE void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, - size_t binind -# ifdef JEMALLOC_PROF - , uint64_t prof_accumbytes -# endif - ); -#endif + size_t binind, uint64_t prof_accumbytes); +void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, + bool zero); +void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); void *arena_malloc_small(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero); -void *arena_malloc(size_t size, bool zero); -void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size, - size_t alignment, bool zero); -size_t arena_salloc(const void *ptr); -#ifdef JEMALLOC_PROF +void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); void arena_prof_promoted(const void *ptr, size_t size); -size_t arena_salloc_demote(const void *ptr); -#endif -void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, +void arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_t *mapelm); +void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind, arena_chunk_map_t *mapelm); +void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind); +void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, + void *ptr); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); -#ifdef JEMALLOC_STATS void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats); -#endif void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); + size_t alignment, bool zero, bool try_tcache); bool arena_new(arena_t *arena, unsigned ind); -bool arena_boot(void); +void arena_boot(void); +void arena_prefork(arena_t *arena); +void arena_postfork_parent(arena_t *arena); +void arena_postfork_child(arena_t *arena); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); +size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, + size_t pageind); +size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); +void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, + size_t size, size_t flags); +void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, + size_t size); +void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, + size_t size, size_t flags); +void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, + size_t binind); +void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, + size_t runind, size_t binind, size_t flags); +void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, + size_t unzeroed); +size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); -# ifdef JEMALLOC_PROF prof_ctx_t *arena_prof_ctx_get(const void *ptr); void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -# endif -void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr); +void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); +size_t arena_salloc(const void *ptr, bool demote); +void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, + bool try_tcache); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) +# ifdef JEMALLOC_ARENA_INLINE_A +JEMALLOC_INLINE arena_chunk_map_t * +arena_mapp_get(arena_chunk_t *chunk, size_t pageind) +{ + + assert(pageind >= map_bias); + assert(pageind < chunk_npages); + + return (&chunk->map[pageind-map_bias]); +} + +JEMALLOC_INLINE size_t * +arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) +{ + + return (&arena_mapp_get(chunk, pageind)->bits); +} + +JEMALLOC_INLINE size_t +arena_mapbits_get(arena_chunk_t *chunk, size_t pageind) +{ + + return (*arena_mapbitsp_get(chunk, pageind)); +} + +JEMALLOC_INLINE size_t +arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); + return (mapbits & ~PAGE_MASK); +} + +JEMALLOC_INLINE size_t +arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == + (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)); + return (mapbits & ~PAGE_MASK); +} + +JEMALLOC_INLINE size_t +arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == + CHUNK_MAP_ALLOCATED); + return (mapbits >> LG_PAGE); +} + +JEMALLOC_INLINE size_t +arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + size_t binind; + + mapbits = arena_mapbits_get(chunk, pageind); + binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; + assert(binind < NBINS || binind == BININD_INVALID); + return (binind); +} + +JEMALLOC_INLINE size_t +arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_DIRTY); +} + +JEMALLOC_INLINE size_t +arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_UNZEROED); +} + +JEMALLOC_INLINE size_t +arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_LARGE); +} + +JEMALLOC_INLINE size_t +arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) +{ + size_t mapbits; + + mapbits = arena_mapbits_get(chunk, pageind); + return (mapbits & CHUNK_MAP_ALLOCATED); +} + +JEMALLOC_INLINE void +arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, + size_t flags) +{ + size_t *mapbitsp; + + mapbitsp = arena_mapbitsp_get(chunk, pageind); + assert((size & PAGE_MASK) == 0); + assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); + assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); + *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags; +} + +JEMALLOC_INLINE void +arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, + size_t size) +{ + size_t *mapbitsp; + + mapbitsp = arena_mapbitsp_get(chunk, pageind); + assert((size & PAGE_MASK) == 0); + assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); + *mapbitsp = size | (*mapbitsp & PAGE_MASK); +} + +JEMALLOC_INLINE void +arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, + size_t flags) +{ + size_t *mapbitsp; + size_t unzeroed; + + mapbitsp = arena_mapbitsp_get(chunk, pageind); + assert((size & PAGE_MASK) == 0); + assert((flags & CHUNK_MAP_DIRTY) == flags); + unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ + *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | unzeroed | + CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; +} + +JEMALLOC_INLINE void +arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, + size_t binind) +{ + size_t *mapbitsp; + + assert(binind <= BININD_INVALID); + mapbitsp = arena_mapbitsp_get(chunk, pageind); + assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); + *mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind << + CHUNK_MAP_BININD_SHIFT); +} + +JEMALLOC_INLINE void +arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, + size_t binind, size_t flags) +{ + size_t *mapbitsp; + size_t unzeroed; + + assert(binind < BININD_INVALID); + mapbitsp = arena_mapbitsp_get(chunk, pageind); + assert(pageind - runind >= map_bias); + assert((flags & CHUNK_MAP_DIRTY) == flags); + unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ + *mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) | + flags | unzeroed | CHUNK_MAP_ALLOCATED; +} + +JEMALLOC_INLINE void +arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, + size_t unzeroed) +{ + size_t *mapbitsp; + + mapbitsp = arena_mapbitsp_get(chunk, pageind); + *mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed; +} + +JEMALLOC_INLINE size_t +arena_ptr_small_binind_get(const void *ptr, size_t mapbits) +{ + size_t binind; + + binind = (mapbits & CHUNK_MAP_BININD_MASK) >> CHUNK_MAP_BININD_SHIFT; + + if (config_debug) { + arena_chunk_t *chunk; + arena_t *arena; + size_t pageind; + size_t actual_mapbits; + arena_run_t *run; + arena_bin_t *bin; + size_t actual_binind; + arena_bin_info_t *bin_info; + + assert(binind != BININD_INVALID); + assert(binind < NBINS); + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + arena = chunk->arena; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + actual_mapbits = arena_mapbits_get(chunk, pageind); + assert(mapbits == actual_mapbits); + assert(arena_mapbits_large_get(chunk, pageind) == 0); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + (actual_mapbits >> LG_PAGE)) << LG_PAGE)); + bin = run->bin; + actual_binind = bin - arena->bins; + assert(binind == actual_binind); + bin_info = &arena_bin_info[actual_binind]; + assert(((uintptr_t)ptr - ((uintptr_t)run + + (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_interval + == 0); + } + + return (binind); +} +# endif /* JEMALLOC_ARENA_INLINE_A */ + +# ifdef JEMALLOC_ARENA_INLINE_B JEMALLOC_INLINE size_t arena_bin_index(arena_t *arena, arena_bin_t *bin) { size_t binind = bin - arena->bins; - assert(binind < nbins); + assert(binind < NBINS); return (binind); } @@ -519,9 +715,8 @@ JEMALLOC_INLINE unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) { unsigned shift, diff, regind; - size_t size; + size_t interval; - dassert(run->magic == ARENA_RUN_MAGIC); /* * Freeing a pointer lower than region zero can cause assertion * failure. @@ -537,12 +732,12 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) bin_info->reg0_offset); /* Rescale (factor powers of 2 out of the numerator and denominator). */ - size = bin_info->reg_size; - shift = ffs(size) - 1; + interval = bin_info->reg_interval; + shift = ffs(interval) - 1; diff >>= shift; - size >>= shift; + interval >>= shift; - if (size == 1) { + if (interval == 1) { /* The divisor was a power of 2. */ regind = diff; } else { @@ -554,7 +749,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) * * becomes * - * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT + * (X * interval_invs[D - 3]) >> SIZE_INV_SHIFT * * We can omit the first three elements, because we never * divide by 0, and 1 and 2 are both powers of two, which are @@ -562,7 +757,7 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) */ #define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) #define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) - static const unsigned size_invs[] = { + static const unsigned interval_invs[] = { SIZE_INV(3), SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), @@ -573,20 +768,21 @@ arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) }; - if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) - regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; - else - regind = diff / size; + if (interval <= ((sizeof(interval_invs) / sizeof(unsigned)) + + 2)) { + regind = (diff * interval_invs[interval - 3]) >> + SIZE_INV_SHIFT; + } else + regind = diff / interval; #undef SIZE_INV #undef SIZE_INV_SHIFT } - assert(diff == regind * size); + assert(diff == regind * interval); assert(regind < bin_info->nregs); return (regind); } -#ifdef JEMALLOC_PROF JEMALLOC_INLINE prof_ctx_t * arena_prof_ctx_get(const void *ptr) { @@ -594,32 +790,33 @@ arena_prof_ctx_get(const void *ptr) arena_chunk_t *chunk; size_t pageind, mapbits; + cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); if ((mapbits & CHUNK_MAP_LARGE) == 0) { if (prof_promote) ret = (prof_ctx_t *)(uintptr_t)1U; else { arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - size_t binind = arena_bin_index(chunk->arena, run->bin); + (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << + LG_PAGE)); + size_t binind = arena_ptr_small_binind_get(ptr, + mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind; - dassert(run->magic == ARENA_RUN_MAGIC); regind = arena_run_regind(run, bin_info, ptr); ret = *(prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset + (regind * sizeof(prof_ctx_t *))); } } else - ret = chunk->map[pageind-map_bias].prof_ctx; + ret = arena_mapp_get(chunk, pageind)->prof_ctx; return (ret); } @@ -630,25 +827,24 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) arena_chunk_t *chunk; size_t pageind, mapbits; + cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + mapbits = arena_mapbits_get(chunk, pageind); assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); if ((mapbits & CHUNK_MAP_LARGE) == 0) { if (prof_promote == false) { arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - arena_bin_t *bin = run->bin; + (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << + LG_PAGE)); size_t binind; arena_bin_info_t *bin_info; unsigned regind; - dassert(run->magic == ARENA_RUN_MAGIC); - binind = arena_bin_index(chunk->arena, bin); + binind = arena_ptr_small_binind_get(ptr, mapbits); bin_info = &arena_bin_info[binind]; regind = arena_run_regind(run, bin_info, ptr); @@ -657,86 +853,122 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) } else assert((uintptr_t)ctx == (uintptr_t)1U); } else - chunk->map[pageind-map_bias].prof_ctx = ctx; + arena_mapp_get(chunk, pageind)->prof_ctx = ctx; +} + +JEMALLOC_INLINE void * +arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) +{ + tcache_t *tcache; + + assert(size != 0); + assert(size <= arena_maxclass); + + if (size <= SMALL_MAXCLASS) { + if (try_tcache && (tcache = tcache_get(true)) != NULL) + return (tcache_alloc_small(tcache, size, zero)); + else { + return (arena_malloc_small(choose_arena(arena), size, + zero)); + } + } else { + /* + * Initialize tcache after checking size in order to avoid + * infinite recursion during tcache initialization. + */ + if (try_tcache && size <= tcache_maxclass && (tcache = + tcache_get(true)) != NULL) + return (tcache_alloc_large(tcache, size, zero)); + else { + return (arena_malloc_large(choose_arena(arena), size, + zero)); + } + } +} + +/* Return the size of the allocation pointed to by ptr. */ +JEMALLOC_INLINE size_t +arena_salloc(const void *ptr, bool demote) +{ + size_t ret; + arena_chunk_t *chunk; + size_t pageind, binind; + + assert(ptr != NULL); + assert(CHUNK_ADDR2BASE(ptr) != ptr); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + binind = arena_mapbits_binind_get(chunk, pageind); + if (binind == BININD_INVALID || (config_prof && demote == false && + prof_promote && arena_mapbits_large_get(chunk, pageind) != 0)) { + /* + * Large allocation. In the common case (demote == true), and + * as this is an inline function, most callers will only end up + * looking at binind to determine that ptr is a small + * allocation. + */ + assert(((uintptr_t)ptr & PAGE_MASK) == 0); + ret = arena_mapbits_large_size_get(chunk, pageind); + assert(ret != 0); + assert(pageind + (ret>>LG_PAGE) <= chunk_npages); + assert(ret == PAGE || arena_mapbits_large_size_get(chunk, + pageind+(ret>>LG_PAGE)-1) == 0); + assert(binind == arena_mapbits_binind_get(chunk, + pageind+(ret>>LG_PAGE)-1)); + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+(ret>>LG_PAGE)-1)); + } else { + /* + * Small allocation (possibly promoted to a large object due to + * prof_promote). + */ + assert(arena_mapbits_large_get(chunk, pageind) != 0 || + arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, + pageind)) == binind); + ret = arena_bin_info[binind].reg_size; + } + + return (ret); } -#endif JEMALLOC_INLINE void -arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) +arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) { - size_t pageind; - arena_chunk_map_t *mapelm; + size_t pageind, mapbits; + tcache_t *tcache; assert(arena != NULL); - dassert(arena->magic == ARENA_MAGIC); assert(chunk->arena == arena); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapelm = &chunk->map[pageind-map_bias]; - assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + mapbits = arena_mapbits_get(chunk, pageind); + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + if ((mapbits & CHUNK_MAP_LARGE) == 0) { /* Small allocation. */ -#ifdef JEMALLOC_TCACHE - tcache_t *tcache; + if (try_tcache && (tcache = tcache_get(false)) != NULL) { + size_t binind; - if ((tcache = tcache_get()) != NULL) - tcache_dalloc_small(tcache, ptr); - else { -#endif - arena_run_t *run; - arena_bin_t *bin; - - run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapelm->bits >> - PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; -#ifdef JEMALLOC_DEBUG - { - size_t binind = arena_bin_index(arena, bin); - arena_bin_info_t *bin_info = - &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % - bin_info->reg_size == 0); - } -#endif - malloc_mutex_lock(&bin->lock); - arena_dalloc_bin(arena, chunk, ptr, mapelm); - malloc_mutex_unlock(&bin->lock); -#ifdef JEMALLOC_TCACHE - } -#endif + binind = arena_ptr_small_binind_get(ptr, mapbits); + tcache_dalloc_small(tcache, ptr, binind); + } else + arena_dalloc_small(arena, chunk, ptr, pageind); } else { -#ifdef JEMALLOC_TCACHE - size_t size = mapelm->bits & ~PAGE_MASK; + size_t size = arena_mapbits_large_size_get(chunk, pageind); assert(((uintptr_t)ptr & PAGE_MASK) == 0); - if (size <= tcache_maxclass) { - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - tcache_dalloc_large(tcache, ptr, size); - else { - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); - } - } else { - malloc_mutex_lock(&arena->lock); + + if (try_tcache && size <= tcache_maxclass && (tcache = + tcache_get(false)) != NULL) { + tcache_dalloc_large(tcache, ptr, size); + } else arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); - } -#else - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); -#endif } } +# endif /* JEMALLOC_ARENA_INLINE_B */ #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/deps/jemalloc/include/jemalloc/internal/atomic.h b/deps/jemalloc/include/jemalloc/internal/atomic.h index 9a298623f8a..11a7b47fe0f 100644 --- a/deps/jemalloc/include/jemalloc/internal/atomic.h +++ b/deps/jemalloc/include/jemalloc/internal/atomic.h @@ -11,22 +11,8 @@ #define atomic_read_uint64(p) atomic_add_uint64(p, 0) #define atomic_read_uint32(p) atomic_add_uint32(p, 0) - -#if (LG_SIZEOF_PTR == 3) -# define atomic_read_z(p) \ - (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0) -# define atomic_add_z(p, x) \ - (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x) -# define atomic_sub_z(p, x) \ - (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x) -#elif (LG_SIZEOF_PTR == 2) -# define atomic_read_z(p) \ - (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0) -# define atomic_add_z(p, x) \ - (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x) -# define atomic_sub_z(p, x) \ - (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x) -#endif +#define atomic_read_z(p) atomic_add_z(p, 0) +#define atomic_read_u(p) atomic_add_u(p, 0) #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -37,12 +23,17 @@ uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); +size_t atomic_add_z(size_t *p, size_t x); +size_t atomic_sub_z(size_t *p, size_t x); +unsigned atomic_add_u(unsigned *p, unsigned x); +unsigned atomic_sub_u(unsigned *p, unsigned x); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) /******************************************************************************/ /* 64-bit operations. */ -#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 +#if (LG_SIZEOF_PTR == 3 || LG_SIZEOF_INT == 3) +# ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -56,6 +47,20 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (__sync_sub_and_fetch(p, x)); } +#elif (defined(_MSC_VER)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (InterlockedExchangeAdd64(p, x)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (InterlockedExchangeAdd64(p, -((int64_t)x))); +} #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) @@ -70,7 +75,7 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); } -#elif (defined(__amd64_) || defined(__x86_64__)) +# elif (defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE uint64_t atomic_add_uint64(uint64_t *p, uint64_t x) { @@ -97,8 +102,43 @@ atomic_sub_uint64(uint64_t *p, uint64_t x) return (x); } -#else -# if (LG_SIZEOF_PTR == 3) +# elif (defined(JEMALLOC_ATOMIC9)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + /* + * atomic_fetchadd_64() doesn't exist, but we only ever use this + * function on LP64 systems, so atomic_fetchadd_long() will do. + */ + assert(sizeof(uint64_t) == sizeof(unsigned long)); + + return (atomic_fetchadd_long(p, (unsigned long)x) + x); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + assert(sizeof(uint64_t) == sizeof(unsigned long)); + + return (atomic_fetchadd_long(p, (unsigned long)(-(long)x)) - x); +} +# elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_8)) +JEMALLOC_INLINE uint64_t +atomic_add_uint64(uint64_t *p, uint64_t x) +{ + + return (__sync_add_and_fetch(p, x)); +} + +JEMALLOC_INLINE uint64_t +atomic_sub_uint64(uint64_t *p, uint64_t x) +{ + + return (__sync_sub_and_fetch(p, x)); +} +# else # error "Missing implementation for 64-bit atomic operations" # endif #endif @@ -119,6 +159,20 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (__sync_sub_and_fetch(p, x)); } +#elif (defined(_MSC_VER)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (InterlockedExchangeAdd(p, x)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (InterlockedExchangeAdd(p, -((int32_t)x))); +} #elif (defined(JEMALLOC_OSATOMIC)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) @@ -133,7 +187,7 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); } -#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__)) +#elif (defined(__i386__) || defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE uint32_t atomic_add_uint32(uint32_t *p, uint32_t x) { @@ -160,9 +214,90 @@ atomic_sub_uint32(uint32_t *p, uint32_t x) return (x); } +#elif (defined(JEMALLOC_ATOMIC9)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (atomic_fetchadd_32(p, x) + x); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (atomic_fetchadd_32(p, (uint32_t)(-(int32_t)x)) - x); +} +#elif (defined(JE_FORCE_SYNC_COMPARE_AND_SWAP_4)) +JEMALLOC_INLINE uint32_t +atomic_add_uint32(uint32_t *p, uint32_t x) +{ + + return (__sync_add_and_fetch(p, x)); +} + +JEMALLOC_INLINE uint32_t +atomic_sub_uint32(uint32_t *p, uint32_t x) +{ + + return (__sync_sub_and_fetch(p, x)); +} #else # error "Missing implementation for 32-bit atomic operations" #endif + +/******************************************************************************/ +/* size_t operations. */ +JEMALLOC_INLINE size_t +atomic_add_z(size_t *p, size_t x) +{ + +#if (LG_SIZEOF_PTR == 3) + return ((size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); +#elif (LG_SIZEOF_PTR == 2) + return ((size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); +#endif +} + +JEMALLOC_INLINE size_t +atomic_sub_z(size_t *p, size_t x) +{ + +#if (LG_SIZEOF_PTR == 3) + return ((size_t)atomic_add_uint64((uint64_t *)p, + (uint64_t)-((int64_t)x))); +#elif (LG_SIZEOF_PTR == 2) + return ((size_t)atomic_add_uint32((uint32_t *)p, + (uint32_t)-((int32_t)x))); +#endif +} + +/******************************************************************************/ +/* unsigned operations. */ +JEMALLOC_INLINE unsigned +atomic_add_u(unsigned *p, unsigned x) +{ + +#if (LG_SIZEOF_INT == 3) + return ((unsigned)atomic_add_uint64((uint64_t *)p, (uint64_t)x)); +#elif (LG_SIZEOF_INT == 2) + return ((unsigned)atomic_add_uint32((uint32_t *)p, (uint32_t)x)); +#endif +} + +JEMALLOC_INLINE unsigned +atomic_sub_u(unsigned *p, unsigned x) +{ + +#if (LG_SIZEOF_INT == 3) + return ((unsigned)atomic_add_uint64((uint64_t *)p, + (uint64_t)-((int64_t)x))); +#elif (LG_SIZEOF_INT == 2) + return ((unsigned)atomic_add_uint32((uint32_t *)p, + (uint32_t)-((int32_t)x))); +#endif +} +/******************************************************************************/ #endif #endif /* JEMALLOC_H_INLINES */ diff --git a/deps/jemalloc/include/jemalloc/internal/base.h b/deps/jemalloc/include/jemalloc/internal/base.h index e353f309bd2..9cf75ffb0b3 100644 --- a/deps/jemalloc/include/jemalloc/internal/base.h +++ b/deps/jemalloc/include/jemalloc/internal/base.h @@ -9,12 +9,14 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -extern malloc_mutex_t base_mtx; - void *base_alloc(size_t size); +void *base_calloc(size_t number, size_t size); extent_node_t *base_node_alloc(void); void base_node_dealloc(extent_node_t *node); bool base_boot(void); +void base_prefork(void); +void base_postfork_parent(void); +void base_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk.h b/deps/jemalloc/include/jemalloc/internal/chunk.h index 54b6a3ec886..8fb1fe6d15c 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk.h @@ -28,20 +28,13 @@ #ifdef JEMALLOC_H_EXTERNS extern size_t opt_lg_chunk; -#ifdef JEMALLOC_SWAP -extern bool opt_overcommit; -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) /* Protects stats_chunks; currently not used for any other purpose. */ extern malloc_mutex_t chunks_mtx; /* Chunk statistics. */ extern chunk_stats_t stats_chunks; -#endif -#ifdef JEMALLOC_IVSALLOC extern rtree_t *chunks_rtree; -#endif extern size_t chunksize; extern size_t chunksize_mask; /* (chunksize - 1). */ @@ -49,7 +42,7 @@ extern size_t chunk_npages; extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t arena_maxclass; /* Max size class for arenas. */ -void *chunk_alloc(size_t size, bool base, bool *zero); +void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero); void chunk_dealloc(void *chunk, size_t size, bool unmap); bool chunk_boot(void); @@ -60,6 +53,5 @@ bool chunk_boot(void); #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#include "jemalloc/internal/chunk_swap.h" #include "jemalloc/internal/chunk_dss.h" #include "jemalloc/internal/chunk_mmap.h" diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h index 6f005222181..6e2643b24c3 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h @@ -1,4 +1,3 @@ -#ifdef JEMALLOC_DSS /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -10,16 +9,12 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -/* - * Protects sbrk() calls. This avoids malloc races among threads, though it - * does not protect against races with threads that call sbrk() directly. - */ -extern malloc_mutex_t dss_mtx; - -void *chunk_alloc_dss(size_t size, bool *zero); +void *chunk_alloc_dss(size_t size, size_t alignment, bool *zero); bool chunk_in_dss(void *chunk); -bool chunk_dealloc_dss(void *chunk, size_t size); bool chunk_dss_boot(void); +void chunk_dss_prefork(void); +void chunk_dss_postfork_parent(void); +void chunk_dss_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -27,4 +22,3 @@ bool chunk_dss_boot(void); #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h b/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h index 07b50a4dc37..b29f39e9eaf 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h @@ -9,11 +9,10 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -void *chunk_alloc_mmap(size_t size); -void *chunk_alloc_mmap_noreserve(size_t size); -void chunk_dealloc_mmap(void *chunk, size_t size); +void pages_purge(void *addr, size_t length); -bool chunk_mmap_boot(void); +void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero); +bool chunk_dealloc_mmap(void *chunk, size_t size); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/ckh.h b/deps/jemalloc/include/jemalloc/internal/ckh.h index 3e4ad4c85f9..05d1fc03e7a 100644 --- a/deps/jemalloc/include/jemalloc/internal/ckh.h +++ b/deps/jemalloc/include/jemalloc/internal/ckh.h @@ -30,11 +30,6 @@ struct ckhc_s { }; struct ckh_s { -#ifdef JEMALLOC_DEBUG -#define CKH_MAGIC 0x3af2489d - uint32_t magic; -#endif - #ifdef CKH_COUNT /* Counters used to get an idea of performance. */ uint64_t ngrows; @@ -47,7 +42,7 @@ struct ckh_s { /* Used for pseudo-random number generation. */ #define CKH_A 1103515241 #define CKH_C 12347 - uint32_t prn_state; + uint32_t prng_state; /* Total number of items. */ size_t count; diff --git a/deps/jemalloc/include/jemalloc/internal/ctl.h b/deps/jemalloc/include/jemalloc/internal/ctl.h index f1f5eb70a2a..adf3827f094 100644 --- a/deps/jemalloc/include/jemalloc/internal/ctl.h +++ b/deps/jemalloc/include/jemalloc/internal/ctl.h @@ -2,6 +2,8 @@ #ifdef JEMALLOC_H_TYPES typedef struct ctl_node_s ctl_node_t; +typedef struct ctl_named_node_s ctl_named_node_t; +typedef struct ctl_indexed_node_s ctl_indexed_node_t; typedef struct ctl_arena_stats_s ctl_arena_stats_t; typedef struct ctl_stats_s ctl_stats_t; @@ -11,20 +13,21 @@ typedef struct ctl_stats_s ctl_stats_t; struct ctl_node_s { bool named; - union { - struct { - const char *name; - /* If (nchildren == 0), this is a terminal node. */ - unsigned nchildren; - const ctl_node_t *children; - } named; - struct { - const ctl_node_t *(*index)(const size_t *, size_t, - size_t); - } indexed; - } u; - int (*ctl)(const size_t *, size_t, void *, size_t *, void *, - size_t); +}; + +struct ctl_named_node_s { + struct ctl_node_s node; + const char *name; + /* If (nchildren == 0), this is a terminal node. */ + unsigned nchildren; + const ctl_node_t *children; + int (*ctl)(const size_t *, size_t, void *, size_t *, + void *, size_t); +}; + +struct ctl_indexed_node_s { + struct ctl_node_s node; + const ctl_named_node_t *(*index)(const size_t *, size_t, size_t); }; struct ctl_arena_stats_s { @@ -32,7 +35,6 @@ struct ctl_arena_stats_s { unsigned nthreads; size_t pactive; size_t pdirty; -#ifdef JEMALLOC_STATS arena_stats_t astats; /* Aggregate stats for small size classes, based on bin stats. */ @@ -41,13 +43,11 @@ struct ctl_arena_stats_s { uint64_t ndalloc_small; uint64_t nrequests_small; - malloc_bin_stats_t *bstats; /* nbins elements. */ + malloc_bin_stats_t bstats[NBINS]; malloc_large_stats_t *lstats; /* nlclasses elements. */ -#endif }; struct ctl_stats_s { -#ifdef JEMALLOC_STATS size_t allocated; size_t active; size_t mapped; @@ -61,11 +61,7 @@ struct ctl_stats_s { uint64_t nmalloc; /* huge_nmalloc */ uint64_t ndalloc; /* huge_ndalloc */ } huge; -#endif ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ -#ifdef JEMALLOC_SWAP - size_t swap_avail; -#endif }; #endif /* JEMALLOC_H_STRUCTS */ @@ -81,27 +77,25 @@ int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, bool ctl_boot(void); #define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ - if (JEMALLOC_P(mallctl)(name, oldp, oldlenp, newp, newlen) \ + if (je_mallctl(name, oldp, oldlenp, newp, newlen) \ != 0) { \ - malloc_write(": Failure in xmallctl(\""); \ - malloc_write(name); \ - malloc_write("\", ...)\n"); \ + malloc_printf( \ + ": Failure in xmallctl(\"%s\", ...)\n", \ + name); \ abort(); \ } \ } while (0) #define xmallctlnametomib(name, mibp, miblenp) do { \ - if (JEMALLOC_P(mallctlnametomib)(name, mibp, miblenp) != 0) { \ - malloc_write( \ - ": Failure in xmallctlnametomib(\""); \ - malloc_write(name); \ - malloc_write("\", ...)\n"); \ + if (je_mallctlnametomib(name, mibp, miblenp) != 0) { \ + malloc_printf(": Failure in " \ + "xmallctlnametomib(\"%s\", ...)\n", name); \ abort(); \ } \ } while (0) #define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \ - if (JEMALLOC_P(mallctlbymib)(mib, miblen, oldp, oldlenp, newp, \ + if (je_mallctlbymib(mib, miblen, oldp, oldlenp, newp, \ newlen) != 0) { \ malloc_write( \ ": Failure in xmallctlbymib()\n"); \ diff --git a/deps/jemalloc/include/jemalloc/internal/extent.h b/deps/jemalloc/include/jemalloc/internal/extent.h index 6fe9702b5f6..36af8be8995 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent.h +++ b/deps/jemalloc/include/jemalloc/internal/extent.h @@ -9,18 +9,14 @@ typedef struct extent_node_s extent_node_t; /* Tree of extents. */ struct extent_node_s { -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) /* Linkage for the size/address-ordered tree. */ rb_node(extent_node_t) link_szad; -#endif /* Linkage for the address-ordered tree. */ rb_node(extent_node_t) link_ad; -#ifdef JEMALLOC_PROF /* Profile counters, used for huge objects. */ prof_ctx_t *prof_ctx; -#endif /* Pointer to the extent that this tree node is responsible for. */ void *addr; @@ -34,9 +30,7 @@ typedef rb_tree(extent_node_t) extent_tree_t; /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t) -#endif rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) diff --git a/deps/jemalloc/include/jemalloc/internal/hash.h b/deps/jemalloc/include/jemalloc/internal/hash.h index 8a46ce30803..2f501f5d4c9 100644 --- a/deps/jemalloc/include/jemalloc/internal/hash.h +++ b/deps/jemalloc/include/jemalloc/internal/hash.h @@ -26,7 +26,7 @@ uint64_t hash(const void *key, size_t len, uint64_t seed); JEMALLOC_INLINE uint64_t hash(const void *key, size_t len, uint64_t seed) { - const uint64_t m = 0xc6a4a7935bd1e995LLU; + const uint64_t m = UINT64_C(0xc6a4a7935bd1e995); const int r = 47; uint64_t h = seed ^ (len * m); const uint64_t *data = (const uint64_t *)key; @@ -48,14 +48,14 @@ hash(const void *key, size_t len, uint64_t seed) data2 = (const unsigned char *)data; switch(len & 7) { - case 7: h ^= ((uint64_t)(data2[6])) << 48; - case 6: h ^= ((uint64_t)(data2[5])) << 40; - case 5: h ^= ((uint64_t)(data2[4])) << 32; - case 4: h ^= ((uint64_t)(data2[3])) << 24; - case 3: h ^= ((uint64_t)(data2[2])) << 16; - case 2: h ^= ((uint64_t)(data2[1])) << 8; - case 1: h ^= ((uint64_t)(data2[0])); - h *= m; + case 7: h ^= ((uint64_t)(data2[6])) << 48; + case 6: h ^= ((uint64_t)(data2[5])) << 40; + case 5: h ^= ((uint64_t)(data2[4])) << 32; + case 4: h ^= ((uint64_t)(data2[3])) << 24; + case 3: h ^= ((uint64_t)(data2[2])) << 16; + case 2: h ^= ((uint64_t)(data2[1])) << 8; + case 1: h ^= ((uint64_t)(data2[0])); + h *= m; } h ^= h >> r; diff --git a/deps/jemalloc/include/jemalloc/internal/huge.h b/deps/jemalloc/include/jemalloc/internal/huge.h index 66544cf8d97..e8513c933e2 100644 --- a/deps/jemalloc/include/jemalloc/internal/huge.h +++ b/deps/jemalloc/include/jemalloc/internal/huge.h @@ -9,12 +9,10 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -#ifdef JEMALLOC_STATS /* Huge allocation statistics. */ extern uint64_t huge_nmalloc; extern uint64_t huge_ndalloc; extern size_t huge_allocated; -#endif /* Protects chunk-related data structures. */ extern malloc_mutex_t huge_mtx; @@ -27,11 +25,12 @@ void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero); void huge_dalloc(void *ptr, bool unmap); size_t huge_salloc(const void *ptr); -#ifdef JEMALLOC_PROF prof_ctx_t *huge_prof_ctx_get(const void *ptr); void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -#endif bool huge_boot(void); +void huge_prefork(void); +void huge_postfork_parent(void); +void huge_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in index a44f0978ae4..268cd146fd4 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in @@ -1,17 +1,33 @@ -#include -#include -#include +#ifndef JEMALLOC_INTERNAL_H +#define JEMALLOC_INTERNAL_H +#include +#ifdef _WIN32 +# include +# define ENOENT ERROR_PATH_NOT_FOUND +# define EINVAL ERROR_BAD_ARGUMENTS +# define EAGAIN ERROR_OUTOFMEMORY +# define EPERM ERROR_WRITE_FAULT +# define EFAULT ERROR_INVALID_ADDRESS +# define ENOMEM ERROR_NOT_ENOUGH_MEMORY +# undef ERANGE +# define ERANGE ERROR_INVALID_DATA +#else +# include +# include +# include +# if !defined(SYS_write) && defined(__NR_write) +# define SYS_write __NR_write +# endif +# include +# include +# include +#endif #include -#include -#include -#include #include #ifndef SIZE_T_MAX # define SIZE_T_MAX SIZE_MAX #endif -#include -#include #include #include #include @@ -25,16 +41,156 @@ #include #include #include -#include +#ifdef _MSC_VER +# include +typedef intptr_t ssize_t; +# define PATH_MAX 1024 +# define STDERR_FILENO 2 +# define __func__ __FUNCTION__ +/* Disable warnings about deprecated system functions */ +# pragma warning(disable: 4996) +#else +# include +#endif #include -#include -#include -#define JEMALLOC_MANGLE +#define JEMALLOC_NO_DEMANGLE #include "../jemalloc@install_suffix@.h" +#ifdef JEMALLOC_UTRACE +#include +#endif + +#ifdef JEMALLOC_VALGRIND +#include +#include +#endif + #include "jemalloc/internal/private_namespace.h" +#ifdef JEMALLOC_CC_SILENCE +#define UNUSED JEMALLOC_ATTR(unused) +#else +#define UNUSED +#endif + +static const bool config_debug = +#ifdef JEMALLOC_DEBUG + true +#else + false +#endif + ; +static const bool config_dss = +#ifdef JEMALLOC_DSS + true +#else + false +#endif + ; +static const bool config_fill = +#ifdef JEMALLOC_FILL + true +#else + false +#endif + ; +static const bool config_lazy_lock = +#ifdef JEMALLOC_LAZY_LOCK + true +#else + false +#endif + ; +static const bool config_prof = +#ifdef JEMALLOC_PROF + true +#else + false +#endif + ; +static const bool config_prof_libgcc = +#ifdef JEMALLOC_PROF_LIBGCC + true +#else + false +#endif + ; +static const bool config_prof_libunwind = +#ifdef JEMALLOC_PROF_LIBUNWIND + true +#else + false +#endif + ; +static const bool config_mremap = +#ifdef JEMALLOC_MREMAP + true +#else + false +#endif + ; +static const bool config_munmap = +#ifdef JEMALLOC_MUNMAP + true +#else + false +#endif + ; +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; +static const bool config_tcache = +#ifdef JEMALLOC_TCACHE + true +#else + false +#endif + ; +static const bool config_tls = +#ifdef JEMALLOC_TLS + true +#else + false +#endif + ; +static const bool config_utrace = +#ifdef JEMALLOC_UTRACE + true +#else + false +#endif + ; +static const bool config_valgrind = +#ifdef JEMALLOC_VALGRIND + true +#else + false +#endif + ; +static const bool config_xmalloc = +#ifdef JEMALLOC_XMALLOC + true +#else + false +#endif + ; +static const bool config_ivsalloc = +#ifdef JEMALLOC_IVSALLOC + true +#else + false +#endif + ; + +#ifdef JEMALLOC_ATOMIC9 +#include +#endif + #if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) #include #endif @@ -46,48 +202,11 @@ #include #endif -#ifdef JEMALLOC_LAZY_LOCK -#include -#endif - #define RB_COMPACT #include "jemalloc/internal/rb.h" #include "jemalloc/internal/qr.h" #include "jemalloc/internal/ql.h" -extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); - -/* - * Define a custom assert() in order to reduce the chances of deadlock during - * assertion failure. - */ -#ifndef assert -# ifdef JEMALLOC_DEBUG -# define assert(e) do { \ - if (!(e)) { \ - char line_buf[UMAX2S_BUFSIZE]; \ - malloc_write(": "); \ - malloc_write(__FILE__); \ - malloc_write(":"); \ - malloc_write(u2s(__LINE__, 10, line_buf)); \ - malloc_write(": Failed assertion: "); \ - malloc_write("\""); \ - malloc_write(#e); \ - malloc_write("\"\n"); \ - abort(); \ - } \ -} while (0) -# else -# define assert(e) -# endif -#endif - -#ifdef JEMALLOC_DEBUG -# define dassert(e) assert(e) -#else -# define dassert(e) -#endif - /* * jemalloc can conceptually be broken into components (arena, tcache, etc.), * but there are circular dependencies that cannot be broken without @@ -119,38 +238,56 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #else # define JEMALLOC_ENABLE_INLINE # define JEMALLOC_INLINE static inline +# ifdef _MSC_VER +# define inline _inline +# endif #endif -/* Size of stack-allocated buffer passed to buferror(). */ -#define BUFERROR_BUF 64 +/* Smallest size class to support. */ +#define LG_TINY_MIN 3 +#define TINY_MIN (1U << LG_TINY_MIN) -/* Minimum alignment of allocations is 2^LG_QUANTUM bytes. */ -#ifdef __i386__ -# define LG_QUANTUM 4 -#endif -#ifdef __ia64__ -# define LG_QUANTUM 4 -#endif -#ifdef __alpha__ -# define LG_QUANTUM 4 -#endif -#ifdef __sparc64__ -# define LG_QUANTUM 4 -#endif -#if (defined(__amd64__) || defined(__x86_64__)) -# define LG_QUANTUM 4 -#endif -#ifdef __arm__ -# define LG_QUANTUM 3 -#endif -#ifdef __mips__ -# define LG_QUANTUM 3 -#endif -#ifdef __powerpc__ -# define LG_QUANTUM 4 -#endif -#ifdef __s390x__ -# define LG_QUANTUM 4 +/* + * Minimum alignment of allocations is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# ifdef __sparc64__ +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# ifdef __s390x__ +# define LG_QUANTUM 4 +# endif +# ifdef __SH4__ +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "No LG_QUANTUM definition for architecture; specify via CPPFLAGS" +# endif #endif #define QUANTUM ((size_t)(1U << LG_QUANTUM)) @@ -164,67 +301,149 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #define LONG_MASK (LONG - 1) /* Return the smallest long multiple that is >= a. */ -#define LONG_CEILING(a) \ +#define LONG_CEILING(a) \ (((a) + LONG_MASK) & ~LONG_MASK) #define SIZEOF_PTR (1U << LG_SIZEOF_PTR) #define PTR_MASK (SIZEOF_PTR - 1) /* Return the smallest (void *) multiple that is >= a. */ -#define PTR_CEILING(a) \ +#define PTR_CEILING(a) \ (((a) + PTR_MASK) & ~PTR_MASK) /* * Maximum size of L1 cache line. This is used to avoid cache line aliasing. * In addition, this controls the spacing of cacheline-spaced size classes. + * + * CACHELINE cannot be based on LG_CACHELINE because __declspec(align()) can + * only handle raw constants. */ #define LG_CACHELINE 6 -#define CACHELINE ((size_t)(1U << LG_CACHELINE)) +#define CACHELINE 64 #define CACHELINE_MASK (CACHELINE - 1) /* Return the smallest cacheline multiple that is >= s. */ #define CACHELINE_CEILING(s) \ (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) -/* - * Page size. STATIC_PAGE_SHIFT is determined by the configure script. If - * DYNAMIC_PAGE_SHIFT is enabled, only use the STATIC_PAGE_* macros where - * compile-time values are required for the purposes of defining data - * structures. - */ -#define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT)) -#define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1)) - -#ifdef PAGE_SHIFT -# undef PAGE_SHIFT -#endif -#ifdef PAGE_SIZE -# undef PAGE_SIZE -#endif +/* Page size. STATIC_PAGE_SHIFT is determined by the configure script. */ #ifdef PAGE_MASK # undef PAGE_MASK #endif - -#ifdef DYNAMIC_PAGE_SHIFT -# define PAGE_SHIFT lg_pagesize -# define PAGE_SIZE pagesize -# define PAGE_MASK pagesize_mask -#else -# define PAGE_SHIFT STATIC_PAGE_SHIFT -# define PAGE_SIZE STATIC_PAGE_SIZE -# define PAGE_MASK STATIC_PAGE_MASK -#endif +#define LG_PAGE STATIC_PAGE_SHIFT +#define PAGE ((size_t)(1U << STATIC_PAGE_SHIFT)) +#define PAGE_MASK ((size_t)(PAGE - 1)) /* Return the smallest pagesize multiple that is >= s. */ #define PAGE_CEILING(s) \ (((s) + PAGE_MASK) & ~PAGE_MASK) +/* Return the nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2BASE(a, alignment) \ + ((void *)((uintptr_t)(a) & (-(alignment)))) + +/* Return the offset between a and the nearest aligned address at or below a. */ +#define ALIGNMENT_ADDR2OFFSET(a, alignment) \ + ((size_t)((uintptr_t)(a) & (alignment - 1))) + +/* Return the smallest alignment multiple that is >= s. */ +#define ALIGNMENT_CEILING(s, alignment) \ + (((s) + (alignment - 1)) & (-(alignment))) + +/* Declare a variable length array */ +#if __STDC_VERSION__ < 199901L +# ifdef _MSC_VER +# include +# define alloca _alloca +# else +# include +# endif +# define VARIABLE_ARRAY(type, name, count) \ + type *name = alloca(sizeof(type) * count) +#else +# define VARIABLE_ARRAY(type, name, count) type name[count] +#endif + +#ifdef JEMALLOC_VALGRIND +/* + * The JEMALLOC_VALGRIND_*() macros must be macros rather than functions + * so that when Valgrind reports errors, there are no extra stack frames + * in the backtraces. + * + * The size that is reported to valgrind must be consistent through a chain of + * malloc..realloc..realloc calls. Request size isn't recorded anywhere in + * jemalloc, so it is critical that all callers of these macros provide usize + * rather than request size. As a result, buffer overflow detection is + * technically weakened for the standard API, though it is generally accepted + * practice to consider any extra bytes reported by malloc_usable_size() as + * usable space. + */ +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do { \ + if (config_valgrind && opt_valgrind && cond) \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, p2rz(ptr), zero); \ +} while (0) +#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ + old_rzsize, zero) do { \ + if (config_valgrind && opt_valgrind) { \ + size_t rzsize = p2rz(ptr); \ + \ + if (ptr == old_ptr) { \ + VALGRIND_RESIZEINPLACE_BLOCK(ptr, old_usize, \ + usize, rzsize); \ + if (zero && old_usize < usize) { \ + VALGRIND_MAKE_MEM_DEFINED( \ + (void *)((uintptr_t)ptr + \ + old_usize), usize - old_usize); \ + } \ + } else { \ + if (old_ptr != NULL) { \ + VALGRIND_FREELIKE_BLOCK(old_ptr, \ + old_rzsize); \ + } \ + if (ptr != NULL) { \ + size_t copy_size = (old_usize < usize) \ + ? old_usize : usize; \ + size_t tail_size = usize - copy_size; \ + VALGRIND_MALLOCLIKE_BLOCK(ptr, usize, \ + rzsize, false); \ + if (copy_size > 0) { \ + VALGRIND_MAKE_MEM_DEFINED(ptr, \ + copy_size); \ + } \ + if (zero && tail_size > 0) { \ + VALGRIND_MAKE_MEM_DEFINED( \ + (void *)((uintptr_t)ptr + \ + copy_size), tail_size); \ + } \ + } \ + } \ + } \ +} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do { \ + if (config_valgrind && opt_valgrind) \ + VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ +} while (0) +#else +#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) +#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) +#define VALGRIND_FREELIKE_BLOCK(addr, rzB) +#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) +#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) +#define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ + old_rzsize, zero) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) +#endif + +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/arena.h" @@ -235,21 +454,22 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_TYPES /******************************************************************************/ #define JEMALLOC_H_STRUCTS +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" @@ -260,66 +480,37 @@ extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" -#ifdef JEMALLOC_STATS typedef struct { uint64_t allocated; uint64_t deallocated; } thread_allocated_t; -#endif +/* + * The JEMALLOC_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro + * argument. + */ +#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_CONCAT({0, 0}) #undef JEMALLOC_H_STRUCTS /******************************************************************************/ #define JEMALLOC_H_EXTERNS extern bool opt_abort; -#ifdef JEMALLOC_FILL extern bool opt_junk; -#endif -#ifdef JEMALLOC_SYSV -extern bool opt_sysv; -#endif -#ifdef JEMALLOC_XMALLOC +extern size_t opt_quarantine; +extern bool opt_redzone; +extern bool opt_utrace; +extern bool opt_valgrind; extern bool opt_xmalloc; -#endif -#ifdef JEMALLOC_FILL extern bool opt_zero; -#endif extern size_t opt_narenas; -#ifdef DYNAMIC_PAGE_SHIFT -extern size_t pagesize; -extern size_t pagesize_mask; -extern size_t lg_pagesize; -#endif - /* Number of CPUs. */ extern unsigned ncpus; extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ -extern pthread_key_t arenas_tsd; -#ifndef NO_TLS -/* - * Map of pthread_self() --> arenas[???], used for selecting an arena to use - * for allocations. - */ -extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); -# define ARENA_GET() arenas_tls -# define ARENA_SET(v) do { \ - arenas_tls = (v); \ - pthread_setspecific(arenas_tsd, (void *)(v)); \ -} while (0) -#else -# define ARENA_GET() ((arena_t *)pthread_getspecific(arenas_tsd)) -# define ARENA_SET(v) do { \ - pthread_setspecific(arenas_tsd, (void *)(v)); \ -} while (0) -#endif - /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. @@ -327,45 +518,22 @@ extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); extern arena_t **arenas; extern unsigned narenas; -#ifdef JEMALLOC_STATS -# ifndef NO_TLS -extern __thread thread_allocated_t thread_allocated_tls; -# define ALLOCATED_GET() (thread_allocated_tls.allocated) -# define ALLOCATEDP_GET() (&thread_allocated_tls.allocated) -# define DEALLOCATED_GET() (thread_allocated_tls.deallocated) -# define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated) -# define ALLOCATED_ADD(a, d) do { \ - thread_allocated_tls.allocated += a; \ - thread_allocated_tls.deallocated += d; \ -} while (0) -# else -extern pthread_key_t thread_allocated_tsd; -thread_allocated_t *thread_allocated_get_hard(void); - -# define ALLOCATED_GET() (thread_allocated_get()->allocated) -# define ALLOCATEDP_GET() (&thread_allocated_get()->allocated) -# define DEALLOCATED_GET() (thread_allocated_get()->deallocated) -# define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated) -# define ALLOCATED_ADD(a, d) do { \ - thread_allocated_t *thread_allocated = thread_allocated_get(); \ - thread_allocated->allocated += (a); \ - thread_allocated->deallocated += (d); \ -} while (0) -# endif -#endif - arena_t *arenas_extend(unsigned ind); +void arenas_cleanup(void *arg); arena_t *choose_arena_hard(void); -int buferror(int errnum, char *buf, size_t buflen); void jemalloc_prefork(void); -void jemalloc_postfork(void); +void jemalloc_postfork_parent(void); +void jemalloc_postfork_child(void); +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent.h" @@ -376,21 +544,22 @@ void jemalloc_postfork(void); #include "jemalloc/internal/rtree.h" #include "jemalloc/internal/tcache.h" #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_EXTERNS /******************************************************************************/ #define JEMALLOC_H_INLINES +#include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" +#include "jemalloc/internal/prng.h" #include "jemalloc/internal/ckh.h" +#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/tsd.h" #include "jemalloc/internal/mb.h" #include "jemalloc/internal/extent.h" #include "jemalloc/internal/base.h" @@ -398,34 +567,20 @@ void jemalloc_postfork(void); #include "jemalloc/internal/huge.h" #ifndef JEMALLOC_ENABLE_INLINE -size_t pow2_ceil(size_t x); +malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *) + size_t s2u(size_t size); -size_t sa2u(size_t size, size_t alignment, size_t *run_size_p); -void malloc_write(const char *s); -arena_t *choose_arena(void); -# if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -thread_allocated_t *thread_allocated_get(void); -# endif +size_t sa2u(size_t size, size_t alignment); +arena_t *choose_arena(arena_t *arena); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -/* Compute the smallest power of 2 that is >= x. */ -JEMALLOC_INLINE size_t -pow2_ceil(size_t x) -{ - - x--; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - x |= x >> 8; - x |= x >> 16; -#if (LG_SIZEOF_PTR == 3) - x |= x >> 32; -#endif - x++; - return (x); -} +/* + * Map of pthread_self() --> arenas[???], used for selecting an arena to use + * for allocations. + */ +malloc_tsd_externs(arenas, arena_t *) +malloc_tsd_funcs(JEMALLOC_INLINE, arenas, arena_t *, NULL, arenas_cleanup) /* * Compute usable size that would result from allocating an object with the @@ -435,7 +590,7 @@ JEMALLOC_INLINE size_t s2u(size_t size) { - if (size <= small_maxclass) + if (size <= SMALL_MAXCLASS) return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); if (size <= arena_maxclass) return (PAGE_CEILING(size)); @@ -447,10 +602,12 @@ s2u(size_t size) * specified size and alignment. */ JEMALLOC_INLINE size_t -sa2u(size_t size, size_t alignment, size_t *run_size_p) +sa2u(size_t size, size_t alignment) { size_t usize; + assert(alignment != 0 && ((alignment - 1) & alignment) == 0); + /* * Round size up to the nearest multiple of alignment. * @@ -464,12 +621,8 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) * 96 | 1100000 | 32 * 144 | 10100000 | 32 * 192 | 11000000 | 64 - * - * Depending on runtime settings, it is possible that arena_malloc() - * will further round up to a power of two, but that never causes - * correctness issues. */ - usize = (size + (alignment - 1)) & (-alignment); + usize = ALIGNMENT_CEILING(size, alignment); /* * (usize < size) protects against the combination of maximal * alignment and size greater than maximal alignment. @@ -479,8 +632,8 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) return (0); } - if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { - if (usize <= small_maxclass) + if (usize <= arena_maxclass && alignment <= PAGE) { + if (usize <= SMALL_MAXCLASS) return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); return (PAGE_CEILING(usize)); } else { @@ -494,7 +647,7 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) usize = PAGE_CEILING(size); /* * (usize < size) protects against very large sizes within - * PAGE_SIZE of SIZE_T_MAX. + * PAGE of SIZE_T_MAX. * * (usize + alignment < usize) protects against the * combination of maximal alignment and usize large enough @@ -512,93 +665,63 @@ sa2u(size_t size, size_t alignment, size_t *run_size_p) /* * Calculate the size of the over-size run that arena_palloc() * would need to allocate in order to guarantee the alignment. + * If the run wouldn't fit within a chunk, round up to a huge + * allocation size. */ - if (usize >= alignment) - run_size = usize + alignment - PAGE_SIZE; - else { - /* - * It is possible that (alignment << 1) will cause - * overflow, but it doesn't matter because we also - * subtract PAGE_SIZE, which in the case of overflow - * leaves us with a very large run_size. That causes - * the first conditional below to fail, which means - * that the bogus run_size value never gets used for - * anything important. - */ - run_size = (alignment << 1) - PAGE_SIZE; - } - if (run_size_p != NULL) - *run_size_p = run_size; - + run_size = usize + alignment - PAGE; if (run_size <= arena_maxclass) return (PAGE_CEILING(usize)); return (CHUNK_CEILING(usize)); } } -/* - * Wrapper around malloc_message() that avoids the need for - * JEMALLOC_P(malloc_message)(...) throughout the code. - */ -JEMALLOC_INLINE void -malloc_write(const char *s) -{ - - JEMALLOC_P(malloc_message)(NULL, s); -} - -/* - * Choose an arena based on a per-thread value (fast-path code, calls slow-path - * code if necessary). - */ +/* Choose an arena based on a per-thread value. */ JEMALLOC_INLINE arena_t * -choose_arena(void) +choose_arena(arena_t *arena) { arena_t *ret; - ret = ARENA_GET(); - if (ret == NULL) { + if (arena != NULL) + return (arena); + + if ((ret = *arenas_tsd_get()) == NULL) { ret = choose_arena_hard(); assert(ret != NULL); } return (ret); } - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -JEMALLOC_INLINE thread_allocated_t * -thread_allocated_get(void) -{ - thread_allocated_t *thread_allocated = (thread_allocated_t *) - pthread_getspecific(thread_allocated_tsd); - - if (thread_allocated == NULL) - return (thread_allocated_get_hard()); - return (thread_allocated); -} -#endif #endif #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/rtree.h" +/* + * Include arena.h twice in order to resolve circular dependencies with + * tcache.h. + */ +#define JEMALLOC_ARENA_INLINE_A +#include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_A #include "jemalloc/internal/tcache.h" +#define JEMALLOC_ARENA_INLINE_B #include "jemalloc/internal/arena.h" +#undef JEMALLOC_ARENA_INLINE_B #include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif +#include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE void *imalloc(size_t size); void *icalloc(size_t size); void *ipalloc(size_t usize, size_t alignment, bool zero); -size_t isalloc(const void *ptr); -# ifdef JEMALLOC_IVSALLOC -size_t ivsalloc(const void *ptr); -# endif +size_t isalloc(const void *ptr, bool demote); +size_t ivsalloc(const void *ptr, bool demote); +size_t u2rz(size_t usize); +size_t p2rz(const void *ptr); void idalloc(void *ptr); +void iqalloc(void *ptr); void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move); +malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) @@ -609,7 +732,7 @@ imalloc(size_t size) assert(size != 0); if (size <= arena_maxclass) - return (arena_malloc(size, false)); + return (arena_malloc(NULL, size, false, true)); else return (huge_malloc(size, false)); } @@ -619,7 +742,7 @@ icalloc(size_t size) { if (size <= arena_maxclass) - return (arena_malloc(size, true)); + return (arena_malloc(NULL, size, true, true)); else return (huge_malloc(size, true)); } @@ -630,75 +753,80 @@ ipalloc(size_t usize, size_t alignment, bool zero) void *ret; assert(usize != 0); - assert(usize == sa2u(usize, alignment, NULL)); + assert(usize == sa2u(usize, alignment)); - if (usize <= arena_maxclass && alignment <= PAGE_SIZE) - ret = arena_malloc(usize, zero); + if (usize <= arena_maxclass && alignment <= PAGE) + ret = arena_malloc(NULL, usize, zero, true); else { - size_t run_size -#ifdef JEMALLOC_CC_SILENCE - = 0 -#endif - ; - - /* - * Ideally we would only ever call sa2u() once per aligned - * allocation request, and the caller of this function has - * already done so once. However, it's rather burdensome to - * require every caller to pass in run_size, especially given - * that it's only relevant to large allocations. Therefore, - * just call it again here in order to get run_size. - */ - sa2u(usize, alignment, &run_size); - if (run_size <= arena_maxclass) { - ret = arena_palloc(choose_arena(), usize, run_size, - alignment, zero); + if (usize <= arena_maxclass) { + ret = arena_palloc(choose_arena(NULL), usize, alignment, + zero); } else if (alignment <= chunksize) ret = huge_malloc(usize, zero); else ret = huge_palloc(usize, alignment, zero); } - assert(((uintptr_t)ret & (alignment - 1)) == 0); + assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); return (ret); } +/* + * Typical usage: + * void *ptr = [...] + * size_t sz = isalloc(ptr, config_prof); + */ JEMALLOC_INLINE size_t -isalloc(const void *ptr) +isalloc(const void *ptr, bool demote) { size_t ret; arena_chunk_t *chunk; assert(ptr != NULL); + /* Demotion only makes sense if config_prof is true. */ + assert(config_prof || demote == false); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { - /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - -#ifdef JEMALLOC_PROF - ret = arena_salloc_demote(ptr); -#else - ret = arena_salloc(ptr); -#endif - } else + if (chunk != ptr) + ret = arena_salloc(ptr, demote); + else ret = huge_salloc(ptr); return (ret); } -#ifdef JEMALLOC_IVSALLOC JEMALLOC_INLINE size_t -ivsalloc(const void *ptr) +ivsalloc(const void *ptr, bool demote) { /* Return 0 if ptr is not within a chunk managed by jemalloc. */ if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) return (0); - return (isalloc(ptr)); + return (isalloc(ptr, demote)); +} + +JEMALLOC_INLINE size_t +u2rz(size_t usize) +{ + size_t ret; + + if (usize <= SMALL_MAXCLASS) { + size_t binind = SMALL_SIZE2BIN(usize); + ret = arena_bin_info[binind].redzone_size; + } else + ret = 0; + + return (ret); +} + +JEMALLOC_INLINE size_t +p2rz(const void *ptr) +{ + size_t usize = isalloc(ptr, false); + + return (u2rz(usize)); } -#endif JEMALLOC_INLINE void idalloc(void *ptr) @@ -709,11 +837,21 @@ idalloc(void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk->arena, chunk, ptr); + arena_dalloc(chunk->arena, chunk, ptr, true); else huge_dalloc(ptr, true); } +JEMALLOC_INLINE void +iqalloc(void *ptr) +{ + + if (config_fill && opt_quarantine) + quarantine(ptr); + else + idalloc(ptr); +} + JEMALLOC_INLINE void * iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move) @@ -724,19 +862,19 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, assert(ptr != NULL); assert(size != 0); - oldsize = isalloc(ptr); + oldsize = isalloc(ptr, config_prof); if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { size_t usize, copysize; /* - * Existing object alignment is inadquate; allocate new space + * Existing object alignment is inadequate; allocate new space * and copy. */ if (no_move) return (NULL); - usize = sa2u(size + extra, alignment, NULL); + usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); ret = ipalloc(usize, alignment, zero); @@ -744,7 +882,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, if (extra == 0) return (NULL); /* Try again, without extra this time. */ - usize = sa2u(size, alignment, NULL); + usize = sa2u(size, alignment); if (usize == 0) return (NULL); ret = ipalloc(usize, alignment, zero); @@ -758,7 +896,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloc(ptr); return (ret); } @@ -773,16 +911,21 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, } else { if (size + extra <= arena_maxclass) { return (arena_ralloc(ptr, oldsize, size, extra, - alignment, zero)); + alignment, zero, true)); } else { return (huge_ralloc(ptr, oldsize, size, extra, alignment, zero)); } } } + +malloc_tsd_externs(thread_allocated, thread_allocated_t) +malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t, + THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup) #endif #include "jemalloc/internal/prof.h" #undef JEMALLOC_H_INLINES /******************************************************************************/ +#endif /* JEMALLOC_INTERNAL_H */ diff --git a/deps/jemalloc/include/jemalloc/internal/mb.h b/deps/jemalloc/include/jemalloc/internal/mb.h index dc9f2a54262..3cfa7872942 100644 --- a/deps/jemalloc/include/jemalloc/internal/mb.h +++ b/deps/jemalloc/include/jemalloc/internal/mb.h @@ -54,7 +54,7 @@ mb_write(void) ); #endif } -#elif (defined(__amd64_) || defined(__x86_64__)) +#elif (defined(__amd64__) || defined(__x86_64__)) JEMALLOC_INLINE void mb_write(void) { @@ -87,6 +87,13 @@ mb_write(void) : "memory" /* Clobbers. */ ); } +#elif defined(__tile__) +JEMALLOC_INLINE void +mb_write(void) +{ + + __sync_synchronize(); +} #else /* * This is much slower than a simple memory barrier, but the semantics of mutex diff --git a/deps/jemalloc/include/jemalloc/internal/mutex.h b/deps/jemalloc/include/jemalloc/internal/mutex.h index 62947ced55e..de44e1435ad 100644 --- a/deps/jemalloc/include/jemalloc/internal/mutex.h +++ b/deps/jemalloc/include/jemalloc/internal/mutex.h @@ -1,22 +1,42 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -#ifdef JEMALLOC_OSSPIN -typedef OSSpinLock malloc_mutex_t; -#else -typedef pthread_mutex_t malloc_mutex_t; -#endif +typedef struct malloc_mutex_s malloc_mutex_t; -#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP -# define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP +#ifdef _WIN32 +# define MALLOC_MUTEX_INITIALIZER +#elif (defined(JEMALLOC_OSSPIN)) +# define MALLOC_MUTEX_INITIALIZER {0} +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER, NULL} #else -# define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER +# if (defined(PTHREAD_MUTEX_ADAPTIVE_NP) && \ + defined(PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP)) +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_ADAPTIVE_NP +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP} +# else +# define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT +# define MALLOC_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER} +# endif #endif #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +struct malloc_mutex_s { +#ifdef _WIN32 + CRITICAL_SECTION lock; +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLock lock; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + pthread_mutex_t lock; + malloc_mutex_t *postponed_next; +#else + pthread_mutex_t lock; +#endif +}; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS @@ -24,11 +44,15 @@ typedef pthread_mutex_t malloc_mutex_t; #ifdef JEMALLOC_LAZY_LOCK extern bool isthreaded; #else +# undef isthreaded /* Undo private_namespace.h definition. */ # define isthreaded true #endif bool malloc_mutex_init(malloc_mutex_t *mutex); -void malloc_mutex_destroy(malloc_mutex_t *mutex); +void malloc_mutex_prefork(malloc_mutex_t *mutex); +void malloc_mutex_postfork_parent(malloc_mutex_t *mutex); +void malloc_mutex_postfork_child(malloc_mutex_t *mutex); +bool mutex_boot(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -36,7 +60,6 @@ void malloc_mutex_destroy(malloc_mutex_t *mutex); #ifndef JEMALLOC_ENABLE_INLINE void malloc_mutex_lock(malloc_mutex_t *mutex); -bool malloc_mutex_trylock(malloc_mutex_t *mutex); void malloc_mutex_unlock(malloc_mutex_t *mutex); #endif @@ -46,37 +69,27 @@ malloc_mutex_lock(malloc_mutex_t *mutex) { if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - OSSpinLockLock(mutex); +#ifdef _WIN32 + EnterCriticalSection(&mutex->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockLock(&mutex->lock); #else - pthread_mutex_lock(mutex); + pthread_mutex_lock(&mutex->lock); #endif } } -JEMALLOC_INLINE bool -malloc_mutex_trylock(malloc_mutex_t *mutex) -{ - - if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - return (OSSpinLockTry(mutex) == false); -#else - return (pthread_mutex_trylock(mutex) != 0); -#endif - } else - return (false); -} - JEMALLOC_INLINE void malloc_mutex_unlock(malloc_mutex_t *mutex) { if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - OSSpinLockUnlock(mutex); +#ifdef _WIN32 + LeaveCriticalSection(&mutex->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockUnlock(&mutex->lock); #else - pthread_mutex_unlock(mutex); + pthread_mutex_unlock(&mutex->lock); #endif } } diff --git a/deps/jemalloc/include/jemalloc/internal/private_namespace.h b/deps/jemalloc/include/jemalloc/internal/private_namespace.h index d4f5f96d7b2..b8166470dc3 100644 --- a/deps/jemalloc/include/jemalloc/internal/private_namespace.h +++ b/deps/jemalloc/include/jemalloc/internal/private_namespace.h @@ -1,36 +1,84 @@ +#define a0calloc JEMALLOC_N(a0calloc) +#define a0free JEMALLOC_N(a0free) +#define a0malloc JEMALLOC_N(a0malloc) +#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small) #define arena_bin_index JEMALLOC_N(arena_bin_index) +#define arena_bin_info JEMALLOC_N(arena_bin_info) #define arena_boot JEMALLOC_N(arena_boot) #define arena_dalloc JEMALLOC_N(arena_dalloc) #define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) +#define arena_dalloc_bin_locked JEMALLOC_N(arena_dalloc_bin_locked) +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) #define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) +#define arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked) +#define arena_dalloc_small JEMALLOC_N(arena_dalloc_small) #define arena_malloc JEMALLOC_N(arena_malloc) #define arena_malloc_large JEMALLOC_N(arena_malloc_large) #define arena_malloc_small JEMALLOC_N(arena_malloc_small) +#define arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get) +#define arena_mapbits_binind_get JEMALLOC_N(arena_mapbits_binind_get) +#define arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get) +#define arena_mapbits_get JEMALLOC_N(arena_mapbits_get) +#define arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set) +#define arena_mapbits_large_get JEMALLOC_N(arena_mapbits_large_get) +#define arena_mapbits_large_set JEMALLOC_N(arena_mapbits_large_set) +#define arena_mapbits_large_size_get JEMALLOC_N(arena_mapbits_large_size_get) +#define arena_mapbits_small_runind_get JEMALLOC_N(arena_mapbits_small_runind_get) +#define arena_mapbits_small_set JEMALLOC_N(arena_mapbits_small_set) +#define arena_mapbits_unallocated_set JEMALLOC_N(arena_mapbits_unallocated_set) +#define arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get) +#define arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set) +#define arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get) +#define arena_mapbits_unzeroed_set JEMALLOC_N(arena_mapbits_unzeroed_set) +#define arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get) +#define arena_mapp_get JEMALLOC_N(arena_mapp_get) +#define arena_maxclass JEMALLOC_N(arena_maxclass) #define arena_new JEMALLOC_N(arena_new) #define arena_palloc JEMALLOC_N(arena_palloc) +#define arena_postfork_child JEMALLOC_N(arena_postfork_child) +#define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) +#define arena_prefork JEMALLOC_N(arena_prefork) #define arena_prof_accum JEMALLOC_N(arena_prof_accum) #define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) #define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) #define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) +#define arena_ptr_small_binind_get JEMALLOC_N(arena_ptr_small_binind_get) #define arena_purge_all JEMALLOC_N(arena_purge_all) #define arena_ralloc JEMALLOC_N(arena_ralloc) #define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) #define arena_run_regind JEMALLOC_N(arena_run_regind) #define arena_salloc JEMALLOC_N(arena_salloc) -#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote) #define arena_stats_merge JEMALLOC_N(arena_stats_merge) #define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) +#define arenas JEMALLOC_N(arenas) #define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index) +#define arenas_booted JEMALLOC_N(arenas_booted) +#define arenas_cleanup JEMALLOC_N(arenas_cleanup) #define arenas_extend JEMALLOC_N(arenas_extend) +#define arenas_initialized JEMALLOC_N(arenas_initialized) +#define arenas_lock JEMALLOC_N(arenas_lock) #define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index) +#define arenas_tls JEMALLOC_N(arenas_tls) +#define arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot) +#define arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper) +#define arenas_tsd_get JEMALLOC_N(arenas_tsd_get) +#define arenas_tsd_set JEMALLOC_N(arenas_tsd_set) +#define atomic_add_u JEMALLOC_N(atomic_add_u) #define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32) #define atomic_add_uint64 JEMALLOC_N(atomic_add_uint64) +#define atomic_add_z JEMALLOC_N(atomic_add_z) +#define atomic_sub_u JEMALLOC_N(atomic_sub_u) #define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) #define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) +#define atomic_sub_z JEMALLOC_N(atomic_sub_z) #define base_alloc JEMALLOC_N(base_alloc) #define base_boot JEMALLOC_N(base_boot) +#define base_calloc JEMALLOC_N(base_calloc) #define base_node_alloc JEMALLOC_N(base_node_alloc) #define base_node_dealloc JEMALLOC_N(base_node_dealloc) +#define base_postfork_child JEMALLOC_N(base_postfork_child) +#define base_postfork_parent JEMALLOC_N(base_postfork_parent) +#define base_prefork JEMALLOC_N(base_prefork) #define bitmap_full JEMALLOC_N(bitmap_full) #define bitmap_get JEMALLOC_N(bitmap_get) #define bitmap_info_init JEMALLOC_N(bitmap_info_init) @@ -47,19 +95,19 @@ #define chunk_alloc JEMALLOC_N(chunk_alloc) #define chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss) #define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) -#define chunk_alloc_mmap_noreserve JEMALLOC_N(chunk_alloc_mmap_noreserve) -#define chunk_alloc_swap JEMALLOC_N(chunk_alloc_swap) #define chunk_boot JEMALLOC_N(chunk_boot) #define chunk_dealloc JEMALLOC_N(chunk_dealloc) -#define chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss) #define chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap) -#define chunk_dealloc_swap JEMALLOC_N(chunk_dealloc_swap) #define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) +#define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child) +#define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent) +#define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork) #define chunk_in_dss JEMALLOC_N(chunk_in_dss) -#define chunk_in_swap JEMALLOC_N(chunk_in_swap) -#define chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot) -#define chunk_swap_boot JEMALLOC_N(chunk_swap_boot) -#define chunk_swap_enable JEMALLOC_N(chunk_swap_enable) +#define chunk_npages JEMALLOC_N(chunk_npages) +#define chunks_mtx JEMALLOC_N(chunks_mtx) +#define chunks_rtree JEMALLOC_N(chunks_rtree) +#define chunksize JEMALLOC_N(chunksize) +#define chunksize_mask JEMALLOC_N(chunksize_mask) #define ckh_bucket_search JEMALLOC_N(ckh_bucket_search) #define ckh_count JEMALLOC_N(ckh_count) #define ckh_delete JEMALLOC_N(ckh_delete) @@ -77,7 +125,6 @@ #define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) #define ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert) #define ckh_try_insert JEMALLOC_N(ckh_try_insert) -#define create_zone JEMALLOC_N(create_zone) #define ctl_boot JEMALLOC_N(ctl_boot) #define ctl_bymib JEMALLOC_N(ctl_bymib) #define ctl_byname JEMALLOC_N(ctl_byname) @@ -115,10 +162,17 @@ #define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) #define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) #define hash JEMALLOC_N(hash) +#define huge_allocated JEMALLOC_N(huge_allocated) #define huge_boot JEMALLOC_N(huge_boot) #define huge_dalloc JEMALLOC_N(huge_dalloc) #define huge_malloc JEMALLOC_N(huge_malloc) +#define huge_mtx JEMALLOC_N(huge_mtx) +#define huge_ndalloc JEMALLOC_N(huge_ndalloc) +#define huge_nmalloc JEMALLOC_N(huge_nmalloc) #define huge_palloc JEMALLOC_N(huge_palloc) +#define huge_postfork_child JEMALLOC_N(huge_postfork_child) +#define huge_postfork_parent JEMALLOC_N(huge_postfork_parent) +#define huge_prefork JEMALLOC_N(huge_prefork) #define huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get) #define huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set) #define huge_ralloc JEMALLOC_N(huge_ralloc) @@ -129,21 +183,63 @@ #define idalloc JEMALLOC_N(idalloc) #define imalloc JEMALLOC_N(imalloc) #define ipalloc JEMALLOC_N(ipalloc) +#define iqalloc JEMALLOC_N(iqalloc) #define iralloc JEMALLOC_N(iralloc) #define isalloc JEMALLOC_N(isalloc) +#define isthreaded JEMALLOC_N(isthreaded) #define ivsalloc JEMALLOC_N(ivsalloc) -#define jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init) -#define jemalloc_postfork JEMALLOC_N(jemalloc_postfork) +#define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) +#define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) #define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) #define malloc_cprintf JEMALLOC_N(malloc_cprintf) -#define malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy) #define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) #define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) -#define malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock) +#define malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child) +#define malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent) +#define malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork) #define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) #define malloc_printf JEMALLOC_N(malloc_printf) +#define malloc_snprintf JEMALLOC_N(malloc_snprintf) +#define malloc_strtoumax JEMALLOC_N(malloc_strtoumax) +#define malloc_tsd_boot JEMALLOC_N(malloc_tsd_boot) +#define malloc_tsd_cleanup_register JEMALLOC_N(malloc_tsd_cleanup_register) +#define malloc_tsd_dalloc JEMALLOC_N(malloc_tsd_dalloc) +#define malloc_tsd_malloc JEMALLOC_N(malloc_tsd_malloc) +#define malloc_tsd_no_cleanup JEMALLOC_N(malloc_tsd_no_cleanup) +#define malloc_vcprintf JEMALLOC_N(malloc_vcprintf) +#define malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf) #define malloc_write JEMALLOC_N(malloc_write) +#define map_bias JEMALLOC_N(map_bias) #define mb_write JEMALLOC_N(mb_write) +#define mutex_boot JEMALLOC_N(mutex_boot) +#define narenas JEMALLOC_N(narenas) +#define ncpus JEMALLOC_N(ncpus) +#define nhbins JEMALLOC_N(nhbins) +#define opt_abort JEMALLOC_N(opt_abort) +#define opt_junk JEMALLOC_N(opt_junk) +#define opt_lg_chunk JEMALLOC_N(opt_lg_chunk) +#define opt_lg_dirty_mult JEMALLOC_N(opt_lg_dirty_mult) +#define opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval) +#define opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample) +#define opt_lg_tcache_max JEMALLOC_N(opt_lg_tcache_max) +#define opt_narenas JEMALLOC_N(opt_narenas) +#define opt_prof JEMALLOC_N(opt_prof) +#define opt_prof_accum JEMALLOC_N(opt_prof_accum) +#define opt_prof_active JEMALLOC_N(opt_prof_active) +#define opt_prof_final JEMALLOC_N(opt_prof_final) +#define opt_prof_gdump JEMALLOC_N(opt_prof_gdump) +#define opt_prof_leak JEMALLOC_N(opt_prof_leak) +#define opt_prof_prefix JEMALLOC_N(opt_prof_prefix) +#define opt_quarantine JEMALLOC_N(opt_quarantine) +#define opt_redzone JEMALLOC_N(opt_redzone) +#define opt_stats_print JEMALLOC_N(opt_stats_print) +#define opt_tcache JEMALLOC_N(opt_tcache) +#define opt_utrace JEMALLOC_N(opt_utrace) +#define opt_valgrind JEMALLOC_N(opt_valgrind) +#define opt_xmalloc JEMALLOC_N(opt_xmalloc) +#define opt_zero JEMALLOC_N(opt_zero) +#define p2rz JEMALLOC_N(p2rz) +#define pages_purge JEMALLOC_N(pages_purge) #define pow2_ceil JEMALLOC_N(pow2_ceil) #define prof_backtrace JEMALLOC_N(prof_backtrace) #define prof_boot0 JEMALLOC_N(prof_boot0) @@ -154,14 +250,31 @@ #define prof_free JEMALLOC_N(prof_free) #define prof_gdump JEMALLOC_N(prof_gdump) #define prof_idump JEMALLOC_N(prof_idump) +#define prof_interval JEMALLOC_N(prof_interval) #define prof_lookup JEMALLOC_N(prof_lookup) #define prof_malloc JEMALLOC_N(prof_malloc) #define prof_mdump JEMALLOC_N(prof_mdump) +#define prof_promote JEMALLOC_N(prof_promote) #define prof_realloc JEMALLOC_N(prof_realloc) #define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) #define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) +#define prof_tdata_booted JEMALLOC_N(prof_tdata_booted) +#define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) +#define prof_tdata_get JEMALLOC_N(prof_tdata_get) #define prof_tdata_init JEMALLOC_N(prof_tdata_init) -#define pthread_create JEMALLOC_N(pthread_create) +#define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized) +#define prof_tdata_tls JEMALLOC_N(prof_tdata_tls) +#define prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot) +#define prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper) +#define prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get) +#define prof_tdata_tsd_set JEMALLOC_N(prof_tdata_tsd_set) +#define quarantine JEMALLOC_N(quarantine) +#define quarantine_boot JEMALLOC_N(quarantine_boot) +#define quarantine_tsd_boot JEMALLOC_N(quarantine_tsd_boot) +#define quarantine_tsd_cleanup_wrapper JEMALLOC_N(quarantine_tsd_cleanup_wrapper) +#define quarantine_tsd_get JEMALLOC_N(quarantine_tsd_get) +#define quarantine_tsd_set JEMALLOC_N(quarantine_tsd_set) +#define register_zone JEMALLOC_N(register_zone) #define rtree_get JEMALLOC_N(rtree_get) #define rtree_get_locked JEMALLOC_N(rtree_get_locked) #define rtree_new JEMALLOC_N(rtree_new) @@ -171,25 +284,56 @@ #define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index) #define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index) #define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index) +#define stats_cactive JEMALLOC_N(stats_cactive) #define stats_cactive_add JEMALLOC_N(stats_cactive_add) #define stats_cactive_get JEMALLOC_N(stats_cactive_get) #define stats_cactive_sub JEMALLOC_N(stats_cactive_sub) +#define stats_chunks JEMALLOC_N(stats_chunks) #define stats_print JEMALLOC_N(stats_print) -#define szone2ozone JEMALLOC_N(szone2ozone) #define tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy) #define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) #define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) #define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) +#define tcache_arena_associate JEMALLOC_N(tcache_arena_associate) +#define tcache_arena_dissociate JEMALLOC_N(tcache_arena_dissociate) #define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) #define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) -#define tcache_boot JEMALLOC_N(tcache_boot) +#define tcache_bin_info JEMALLOC_N(tcache_bin_info) +#define tcache_boot0 JEMALLOC_N(tcache_boot0) +#define tcache_boot1 JEMALLOC_N(tcache_boot1) +#define tcache_booted JEMALLOC_N(tcache_booted) #define tcache_create JEMALLOC_N(tcache_create) #define tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large) #define tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small) #define tcache_destroy JEMALLOC_N(tcache_destroy) +#define tcache_enabled_booted JEMALLOC_N(tcache_enabled_booted) +#define tcache_enabled_get JEMALLOC_N(tcache_enabled_get) +#define tcache_enabled_initialized JEMALLOC_N(tcache_enabled_initialized) +#define tcache_enabled_set JEMALLOC_N(tcache_enabled_set) +#define tcache_enabled_tls JEMALLOC_N(tcache_enabled_tls) +#define tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot) +#define tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper) +#define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get) +#define tcache_enabled_tsd_set JEMALLOC_N(tcache_enabled_tsd_set) #define tcache_event JEMALLOC_N(tcache_event) +#define tcache_event_hard JEMALLOC_N(tcache_event_hard) +#define tcache_flush JEMALLOC_N(tcache_flush) #define tcache_get JEMALLOC_N(tcache_get) +#define tcache_initialized JEMALLOC_N(tcache_initialized) +#define tcache_maxclass JEMALLOC_N(tcache_maxclass) +#define tcache_salloc JEMALLOC_N(tcache_salloc) #define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) -#define thread_allocated_get JEMALLOC_N(thread_allocated_get) -#define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard) -#define u2s JEMALLOC_N(u2s) +#define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup) +#define tcache_tls JEMALLOC_N(tcache_tls) +#define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot) +#define tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper) +#define tcache_tsd_get JEMALLOC_N(tcache_tsd_get) +#define tcache_tsd_set JEMALLOC_N(tcache_tsd_set) +#define thread_allocated_booted JEMALLOC_N(thread_allocated_booted) +#define thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized) +#define thread_allocated_tls JEMALLOC_N(thread_allocated_tls) +#define thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot) +#define thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper) +#define thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get) +#define thread_allocated_tsd_set JEMALLOC_N(thread_allocated_tsd_set) +#define u2rz JEMALLOC_N(u2rz) diff --git a/deps/jemalloc/include/jemalloc/internal/prng.h b/deps/jemalloc/include/jemalloc/internal/prng.h new file mode 100644 index 00000000000..83a5462b4dd --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/prng.h @@ -0,0 +1,60 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* + * Simple linear congruential pseudo-random number generator: + * + * prng(y) = (a*x + c) % m + * + * where the following constants ensure maximal period: + * + * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4. + * c == Odd number (relatively prime to 2^n). + * m == 2^32 + * + * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints. + * + * This choice of m has the disadvantage that the quality of the bits is + * proportional to bit position. For example. the lowest bit has a cycle of 2, + * the next has a cycle of 4, etc. For this reason, we prefer to use the upper + * bits. + * + * Macro parameters: + * uint32_t r : Result. + * unsigned lg_range : (0..32], number of least significant bits to return. + * uint32_t state : Seed value. + * const uint32_t a, c : See above discussion. + */ +#define prng32(r, lg_range, state, a, c) do { \ + assert(lg_range > 0); \ + assert(lg_range <= 32); \ + \ + r = (state * (a)) + (c); \ + state = r; \ + r >>= (32 - lg_range); \ +} while (false) + +/* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */ +#define prng64(r, lg_range, state, a, c) do { \ + assert(lg_range > 0); \ + assert(lg_range <= 64); \ + \ + r = (state * (a)) + (c); \ + state = r; \ + r >>= (64 - lg_range); \ +} while (false) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/prof.h b/deps/jemalloc/include/jemalloc/internal/prof.h index e9064ba6e73..c3e3f9e4bcd 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof.h +++ b/deps/jemalloc/include/jemalloc/internal/prof.h @@ -1,4 +1,3 @@ -#ifdef JEMALLOC_PROF /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -10,28 +9,41 @@ typedef struct prof_tdata_s prof_tdata_t; /* Option defaults. */ #define PROF_PREFIX_DEFAULT "jeprof" -#define LG_PROF_BT_MAX_DEFAULT 7 -#define LG_PROF_SAMPLE_DEFAULT 0 +#define LG_PROF_SAMPLE_DEFAULT 19 #define LG_PROF_INTERVAL_DEFAULT -1 -#define LG_PROF_TCMAX_DEFAULT -1 /* - * Hard limit on stack backtrace depth. Note that the version of - * prof_backtrace() that is based on __builtin_return_address() necessarily has - * a hard-coded number of backtrace frame handlers. + * Hard limit on stack backtrace depth. The version of prof_backtrace() that + * is based on __builtin_return_address() necessarily has a hard-coded number + * of backtrace frame handlers, and should be kept in sync with this setting. */ -#if (defined(JEMALLOC_PROF_LIBGCC) || defined(JEMALLOC_PROF_LIBUNWIND)) -# define LG_PROF_BT_MAX ((ZU(1) << (LG_SIZEOF_PTR+3)) - 1) -#else -# define LG_PROF_BT_MAX 7 /* >= LG_PROF_BT_MAX_DEFAULT */ -#endif -#define PROF_BT_MAX (1U << LG_PROF_BT_MAX) +#define PROF_BT_MAX 128 + +/* Maximum number of backtraces to store in each per thread LRU cache. */ +#define PROF_TCMAX 1024 /* Initial hash table size. */ -#define PROF_CKH_MINITEMS 64 +#define PROF_CKH_MINITEMS 64 /* Size of memory buffer to use when writing dump files. */ -#define PROF_DUMP_BUF_SIZE 65536 +#define PROF_DUMP_BUFSIZE 65536 + +/* Size of stack-allocated buffer used by prof_printf(). */ +#define PROF_PRINTF_BUFSIZE 128 + +/* + * Number of mutexes shared among all ctx's. No space is allocated for these + * unless profiling is enabled, so it's okay to over-provision. + */ +#define PROF_NCTX_LOCKS 1024 + +/* + * prof_tdata pointers close to NULL are used to encode state information that + * is used for cleaning up during thread shutdown. + */ +#define PROF_TDATA_STATE_REINCARNATED ((prof_tdata_t *)(uintptr_t)1) +#define PROF_TDATA_STATE_PURGATORY ((prof_tdata_t *)(uintptr_t)2) +#define PROF_TDATA_STATE_MAX PROF_TDATA_STATE_PURGATORY #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -109,8 +121,18 @@ struct prof_ctx_s { /* Associated backtrace. */ prof_bt_t *bt; - /* Protects cnt_merged and cnts_ql. */ - malloc_mutex_t lock; + /* Protects nlimbo, cnt_merged, and cnts_ql. */ + malloc_mutex_t *lock; + + /* + * Number of threads that currently cause this ctx to be in a state of + * limbo due to one of: + * - Initializing per thread counters associated with this ctx. + * - Preparing to destroy this ctx. + * nlimbo must be 1 (single destroyer) in order to safely destroy the + * ctx. + */ + unsigned nlimbo; /* Temporary storage for summation during dump. */ prof_cnt_t cnt_summed; @@ -145,9 +167,14 @@ struct prof_tdata_s { void **vec; /* Sampling state. */ - uint64_t prn_state; + uint64_t prng_state; uint64_t threshold; uint64_t accum; + + /* State used to avoid dumping while operating on prof internals. */ + bool enq; + bool enq_idump; + bool enq_gdump; }; #endif /* JEMALLOC_H_STRUCTS */ @@ -162,13 +189,12 @@ extern bool opt_prof; * to notice state changes. */ extern bool opt_prof_active; -extern size_t opt_lg_prof_bt_max; /* Maximum backtrace depth. */ extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ extern bool opt_prof_gdump; /* High-water memory dumping. */ +extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ -extern ssize_t opt_lg_prof_tcmax; /* lg(max per thread bactrace cache) */ extern char opt_prof_prefix[PATH_MAX + 1]; /* @@ -186,39 +212,14 @@ extern uint64_t prof_interval; */ extern bool prof_promote; -/* (1U << opt_lg_prof_bt_max). */ -extern unsigned prof_bt_max; - -/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ -#ifndef NO_TLS -extern __thread prof_tdata_t *prof_tdata_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -# define PROF_TCACHE_GET() prof_tdata_tls -# define PROF_TCACHE_SET(v) do { \ - prof_tdata_tls = (v); \ - pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ -} while (0) -#else -# define PROF_TCACHE_GET() \ - ((prof_tdata_t *)pthread_getspecific(prof_tdata_tsd)) -# define PROF_TCACHE_SET(v) do { \ - pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ -} while (0) -#endif -/* - * Same contents as b2cnt_tls, but initialized such that the TSD destructor is - * called when a thread exits, so that prof_tdata_tls contents can be merged, - * unlinked, and deallocated. - */ -extern pthread_key_t prof_tdata_tsd; - void bt_init(prof_bt_t *bt, void **vec); -void prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max); +void prof_backtrace(prof_bt_t *bt, unsigned nignore); prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); void prof_idump(void); bool prof_mdump(const char *filename); void prof_gdump(void); prof_tdata_t *prof_tdata_init(void); +void prof_tdata_cleanup(void *arg); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); @@ -233,13 +234,13 @@ bool prof_boot2(void); \ assert(size == s2u(size)); \ \ - prof_tdata = PROF_TCACHE_GET(); \ - if (prof_tdata == NULL) { \ - prof_tdata = prof_tdata_init(); \ - if (prof_tdata == NULL) { \ + prof_tdata = prof_tdata_get(); \ + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \ + if (prof_tdata != NULL) \ + ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ + else \ ret = NULL; \ - break; \ - } \ + break; \ } \ \ if (opt_prof_active == false) { \ @@ -249,13 +250,13 @@ bool prof_boot2(void); /* Don't bother with sampling logic, since sampling */\ /* interval is 1. */\ bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore, prof_bt_max); \ + prof_backtrace(&bt, nignore); \ ret = prof_lookup(&bt); \ } else { \ if (prof_tdata->threshold == 0) { \ /* Initialize. Seed the prng differently for */\ /* each thread. */\ - prof_tdata->prn_state = \ + prof_tdata->prng_state = \ (uint64_t)(uintptr_t)&size; \ prof_sample_threshold_update(prof_tdata); \ } \ @@ -272,7 +273,7 @@ bool prof_boot2(void); if (size >= prof_tdata->threshold - \ prof_tdata->accum) { \ bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore, prof_bt_max); \ + prof_backtrace(&bt, nignore); \ ret = prof_lookup(&bt); \ } else \ ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ @@ -280,6 +281,9 @@ bool prof_boot2(void); } while (0) #ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) + +prof_tdata_t *prof_tdata_get(void); void prof_sample_threshold_update(prof_tdata_t *prof_tdata); prof_ctx_t *prof_ctx_get(const void *ptr); void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); @@ -291,12 +295,35 @@ void prof_free(const void *ptr, size_t size); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) +/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ +malloc_tsd_externs(prof_tdata, prof_tdata_t *) +malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, + prof_tdata_cleanup) + +JEMALLOC_INLINE prof_tdata_t * +prof_tdata_get(void) +{ + prof_tdata_t *prof_tdata; + + cassert(config_prof); + + prof_tdata = *prof_tdata_tsd_get(); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { + if (prof_tdata == NULL) + prof_tdata = prof_tdata_init(); + } + + return (prof_tdata); +} + JEMALLOC_INLINE void prof_sample_threshold_update(prof_tdata_t *prof_tdata) { uint64_t r; double u; + cassert(config_prof); + /* * Compute sample threshold as a geometrically distributed random * variable with mean (2^opt_lg_prof_sample). @@ -315,8 +342,8 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata) * pp 500 * (http://cg.scs.carleton.ca/~luc/rnbookindex.html) */ - prn64(r, 53, prof_tdata->prn_state, - (uint64_t)6364136223846793005LLU, (uint64_t)1442695040888963407LLU); + prng64(r, 53, prof_tdata->prng_state, + UINT64_C(6364136223846793005), UINT64_C(1442695040888963407)); u = (double)r * (1.0/9007199254740992.0L); prof_tdata->threshold = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) @@ -329,13 +356,12 @@ prof_ctx_get(const void *ptr) prof_ctx_t *ret; arena_chunk_t *chunk; + cassert(config_prof); assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - ret = arena_prof_ctx_get(ptr); } else ret = huge_prof_ctx_get(ptr); @@ -348,13 +374,12 @@ prof_ctx_set(const void *ptr, prof_ctx_t *ctx) { arena_chunk_t *chunk; + cassert(config_prof); assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - arena_prof_ctx_set(ptr, ctx); } else huge_prof_ctx_set(ptr, ctx); @@ -365,11 +390,13 @@ prof_sample_accum_update(size_t size) { prof_tdata_t *prof_tdata; + cassert(config_prof); /* Sampling logic is unnecessary if the interval is 1. */ assert(opt_lg_prof_sample != 0); - prof_tdata = PROF_TCACHE_GET(); - assert(prof_tdata != NULL); + prof_tdata = *prof_tdata_tsd_get(); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (true); /* Take care to avoid integer overflow. */ if (size >= prof_tdata->threshold - prof_tdata->accum) { @@ -391,8 +418,9 @@ JEMALLOC_INLINE void prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) { + cassert(config_prof); assert(ptr != NULL); - assert(size == isalloc(ptr)); + assert(size == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { if (prof_sample_accum_update(size)) { @@ -437,10 +465,11 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, { prof_thr_cnt_t *told_cnt; + cassert(config_prof); assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); if (ptr != NULL) { - assert(size == isalloc(ptr)); + assert(size == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { if (prof_sample_accum_update(size)) { /* @@ -463,10 +492,10 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, * It's too late to propagate OOM for this realloc(), * so operate directly on old_cnt->ctx->cnt_merged. */ - malloc_mutex_lock(&old_ctx->lock); + malloc_mutex_lock(old_ctx->lock); old_ctx->cnt_merged.curobjs--; old_ctx->cnt_merged.curbytes -= old_size; - malloc_mutex_unlock(&old_ctx->lock); + malloc_mutex_unlock(old_ctx->lock); told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; } } else @@ -510,9 +539,12 @@ prof_free(const void *ptr, size_t size) { prof_ctx_t *ctx = prof_ctx_get(ptr); + cassert(config_prof); + if ((uintptr_t)ctx > (uintptr_t)1) { - assert(size == isalloc(ptr)); - prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); + prof_thr_cnt_t *tcnt; + assert(size == isalloc(ptr, true)); + tcnt = prof_lookup(ctx->bt); if (tcnt != NULL) { tcnt->epoch++; @@ -533,10 +565,10 @@ prof_free(const void *ptr, size_t size) * OOM during free() cannot be propagated, so operate * directly on cnt->ctx->cnt_merged. */ - malloc_mutex_lock(&ctx->lock); + malloc_mutex_lock(ctx->lock); ctx->cnt_merged.curobjs--; ctx->cnt_merged.curbytes -= size; - malloc_mutex_unlock(&ctx->lock); + malloc_mutex_unlock(ctx->lock); } } } @@ -544,4 +576,3 @@ prof_free(const void *ptr, size_t size) #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc/include/jemalloc/internal/quarantine.h b/deps/jemalloc/include/jemalloc/internal/quarantine.h new file mode 100644 index 00000000000..38f3d696e93 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/quarantine.h @@ -0,0 +1,24 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Default per thread quarantine size if valgrind is enabled. */ +#define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void quarantine(void *ptr); +bool quarantine_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ + diff --git a/deps/jemalloc/include/jemalloc/internal/rb.h b/deps/jemalloc/include/jemalloc/internal/rb.h index ee9b009d235..7b675f09051 100644 --- a/deps/jemalloc/include/jemalloc/internal/rb.h +++ b/deps/jemalloc/include/jemalloc/internal/rb.h @@ -223,88 +223,88 @@ a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ * The following API is generated: * * static void - * ex_new(ex_t *extree); + * ex_new(ex_t *tree); * Description: Initialize a red-black tree structure. * Args: - * extree: Pointer to an uninitialized red-black tree object. + * tree: Pointer to an uninitialized red-black tree object. * * static ex_node_t * - * ex_first(ex_t *extree); + * ex_first(ex_t *tree); * static ex_node_t * - * ex_last(ex_t *extree); - * Description: Get the first/last node in extree. + * ex_last(ex_t *tree); + * Description: Get the first/last node in tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * Ret: First/last node in extree, or NULL if extree is empty. + * tree: Pointer to an initialized red-black tree object. + * Ret: First/last node in tree, or NULL if tree is empty. * * static ex_node_t * - * ex_next(ex_t *extree, ex_node_t *node); + * ex_next(ex_t *tree, ex_node_t *node); * static ex_node_t * - * ex_prev(ex_t *extree, ex_node_t *node); + * ex_prev(ex_t *tree, ex_node_t *node); * Description: Get node's successor/predecessor. * Args: - * extree: Pointer to an initialized red-black tree object. - * node : A node in extree. - * Ret: node's successor/predecessor in extree, or NULL if node is + * tree: Pointer to an initialized red-black tree object. + * node: A node in tree. + * Ret: node's successor/predecessor in tree, or NULL if node is * last/first. * * static ex_node_t * - * ex_search(ex_t *extree, ex_node_t *key); + * ex_search(ex_t *tree, ex_node_t *key); * Description: Search for node that matches key. * Args: - * extree: Pointer to an initialized red-black tree object. - * key : Search key. - * Ret: Node in extree that matches key, or NULL if no match. + * tree: Pointer to an initialized red-black tree object. + * key : Search key. + * Ret: Node in tree that matches key, or NULL if no match. * * static ex_node_t * - * ex_nsearch(ex_t *extree, ex_node_t *key); + * ex_nsearch(ex_t *tree, ex_node_t *key); * static ex_node_t * - * ex_psearch(ex_t *extree, ex_node_t *key); + * ex_psearch(ex_t *tree, ex_node_t *key); * Description: Search for node that matches key. If no match is found, * return what would be key's successor/predecessor, were - * key in extree. + * key in tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * key : Search key. - * Ret: Node in extree that matches key, or if no match, hypothetical - * node's successor/predecessor (NULL if no successor/predecessor). + * tree: Pointer to an initialized red-black tree object. + * key : Search key. + * Ret: Node in tree that matches key, or if no match, hypothetical node's + * successor/predecessor (NULL if no successor/predecessor). * * static void - * ex_insert(ex_t *extree, ex_node_t *node); - * Description: Insert node into extree. + * ex_insert(ex_t *tree, ex_node_t *node); + * Description: Insert node into tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * node : Node to be inserted into extree. + * tree: Pointer to an initialized red-black tree object. + * node: Node to be inserted into tree. * * static void - * ex_remove(ex_t *extree, ex_node_t *node); - * Description: Remove node from extree. + * ex_remove(ex_t *tree, ex_node_t *node); + * Description: Remove node from tree. * Args: - * extree: Pointer to an initialized red-black tree object. - * node : Node in extree to be removed. + * tree: Pointer to an initialized red-black tree object. + * node: Node in tree to be removed. * * static ex_node_t * - * ex_iter(ex_t *extree, ex_node_t *start, ex_node_t *(*cb)(ex_t *, + * ex_iter(ex_t *tree, ex_node_t *start, ex_node_t *(*cb)(ex_t *, * ex_node_t *, void *), void *arg); * static ex_node_t * - * ex_reverse_iter(ex_t *extree, ex_node_t *start, ex_node *(*cb)(ex_t *, + * ex_reverse_iter(ex_t *tree, ex_node_t *start, ex_node *(*cb)(ex_t *, * ex_node_t *, void *), void *arg); - * Description: Iterate forward/backward over extree, starting at node. - * If extree is modified, iteration must be immediately + * Description: Iterate forward/backward over tree, starting at node. If + * tree is modified, iteration must be immediately * terminated by the callback function that causes the * modification. * Args: - * extree: Pointer to an initialized red-black tree object. - * start : Node at which to start iteration, or NULL to start at - * first/last node. - * cb : Callback function, which is called for each node during - * iteration. Under normal circumstances the callback function - * should return NULL, which causes iteration to continue. If a - * callback function returns non-NULL, iteration is immediately - * terminated and the non-NULL return value is returned by the - * iterator. This is useful for re-starting iteration after - * modifying extree. - * arg : Opaque pointer passed to cb(). + * tree : Pointer to an initialized red-black tree object. + * start: Node at which to start iteration, or NULL to start at + * first/last node. + * cb : Callback function, which is called for each node during + * iteration. Under normal circumstances the callback function + * should return NULL, which causes iteration to continue. If a + * callback function returns non-NULL, iteration is immediately + * terminated and the non-NULL return value is returned by the + * iterator. This is useful for re-starting iteration after + * modifying tree. + * arg : Opaque pointer passed to cb(). * Ret: NULL if iteration completed, or the non-NULL callback return value * that caused termination of the iteration. */ diff --git a/deps/jemalloc/include/jemalloc/internal/size_classes.sh b/deps/jemalloc/include/jemalloc/internal/size_classes.sh new file mode 100755 index 00000000000..29c80c1fb8d --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/size_classes.sh @@ -0,0 +1,122 @@ +#!/bin/sh + +# The following limits are chosen such that they cover all supported platforms. + +# Range of quanta. +lg_qmin=3 +lg_qmax=4 + +# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. +lg_tmin=3 + +# Range of page sizes. +lg_pmin=12 +lg_pmax=16 + +pow2() { + e=$1 + pow2_result=1 + while [ ${e} -gt 0 ] ; do + pow2_result=$((${pow2_result} + ${pow2_result})) + e=$((${e} - 1)) + done +} + +cat < 255) +# error "Too many small size classes" +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ +EOF diff --git a/deps/jemalloc/include/jemalloc/internal/stats.h b/deps/jemalloc/include/jemalloc/internal/stats.h index 2a9b31d9ffc..27f68e3681c 100644 --- a/deps/jemalloc/include/jemalloc/internal/stats.h +++ b/deps/jemalloc/include/jemalloc/internal/stats.h @@ -1,25 +1,16 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES -#define UMAX2S_BUFSIZE 65 - -#ifdef JEMALLOC_STATS typedef struct tcache_bin_stats_s tcache_bin_stats_t; typedef struct malloc_bin_stats_s malloc_bin_stats_t; typedef struct malloc_large_stats_s malloc_large_stats_t; typedef struct arena_stats_s arena_stats_t; -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) typedef struct chunk_stats_s chunk_stats_t; -#endif #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS -#ifdef JEMALLOC_STATS - -#ifdef JEMALLOC_TCACHE struct tcache_bin_stats_s { /* * Number of allocation requests that corresponded to the size of this @@ -27,7 +18,6 @@ struct tcache_bin_stats_s { */ uint64_t nrequests; }; -#endif struct malloc_bin_stats_s { /* @@ -52,13 +42,11 @@ struct malloc_bin_stats_s { */ uint64_t nrequests; -#ifdef JEMALLOC_TCACHE /* Number of tcache fills from this bin. */ uint64_t nfills; /* Number of tcache flushes to this bin. */ uint64_t nflushes; -#endif /* Total number of runs created for this bin's size class. */ uint64_t nruns; @@ -69,9 +57,6 @@ struct malloc_bin_stats_s { */ uint64_t reruns; - /* High-water mark for this bin. */ - size_t highruns; - /* Current number of runs in this bin. */ size_t curruns; }; @@ -93,9 +78,6 @@ struct malloc_large_stats_s { */ uint64_t nrequests; - /* High-water mark for this size class. */ - size_t highruns; - /* Current number of runs of this size class. */ size_t curruns; }; @@ -127,14 +109,10 @@ struct arena_stats_s { */ malloc_large_stats_t *lstats; }; -#endif /* JEMALLOC_STATS */ -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) struct chunk_stats_s { -# ifdef JEMALLOC_STATS /* Number of chunks that were allocated. */ uint64_t nchunks; -# endif /* High-water mark for number of chunks allocated. */ size_t highchunks; @@ -146,7 +124,6 @@ struct chunk_stats_s { */ size_t curchunks; }; -#endif /* JEMALLOC_STATS */ #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ @@ -154,24 +131,14 @@ struct chunk_stats_s { extern bool opt_stats_print; -#ifdef JEMALLOC_STATS extern size_t stats_cactive; -#endif -char *u2s(uint64_t x, unsigned base, char *s); -#ifdef JEMALLOC_STATS -void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, - const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); -void malloc_printf(const char *format, ...) - JEMALLOC_ATTR(format(printf, 1, 2)); -#endif void stats_print(void (*write)(void *, const char *), void *cbopaque, const char *opts); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES -#ifdef JEMALLOC_STATS #ifndef JEMALLOC_ENABLE_INLINE size_t stats_cactive_get(void); @@ -202,6 +169,5 @@ stats_cactive_sub(size_t size) } #endif -#endif /* JEMALLOC_STATS */ #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/tcache.h b/deps/jemalloc/include/jemalloc/internal/tcache.h index da3c68c5770..38d735c86c8 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache.h @@ -1,4 +1,3 @@ -#ifdef JEMALLOC_TCACHE /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -6,6 +5,16 @@ typedef struct tcache_bin_info_s tcache_bin_info_t; typedef struct tcache_bin_s tcache_bin_t; typedef struct tcache_s tcache_t; +/* + * tcache pointers close to NULL are used to encode state information that is + * used for two purposes: preventing thread caching on a per thread basis and + * cleaning up during thread shutdown. + */ +#define TCACHE_STATE_DISABLED ((tcache_t *)(uintptr_t)1) +#define TCACHE_STATE_REINCARNATED ((tcache_t *)(uintptr_t)2) +#define TCACHE_STATE_PURGATORY ((tcache_t *)(uintptr_t)3) +#define TCACHE_STATE_MAX TCACHE_STATE_PURGATORY + /* * Absolute maximum number of cache slots for each small bin in the thread * cache. This is an additional constraint beyond that imposed as: twice the @@ -22,17 +31,26 @@ typedef struct tcache_s tcache_t; #define LG_TCACHE_MAXCLASS_DEFAULT 15 /* - * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation - * events between full GC sweeps (-1: disabled). Integer rounding may cause - * the actual number to be slightly higher, since GC is performed - * incrementally. + * TCACHE_GC_SWEEP is the approximate number of allocation events between + * full GC sweeps. Integer rounding may cause the actual number to be + * slightly higher, since GC is performed incrementally. */ -#define LG_TCACHE_GC_SWEEP_DEFAULT 13 +#define TCACHE_GC_SWEEP 8192 + +/* Number of tcache allocation/deallocation events between incremental GCs. */ +#define TCACHE_GC_INCR \ + ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +typedef enum { + tcache_enabled_false = 0, /* Enable cast to/from bool. */ + tcache_enabled_true = 1, + tcache_enabled_default = 2 +} tcache_enabled_t; + /* * Read-only information associated with each element of tcache_t's tbins array * is stored separately, mainly to reduce memory usage. @@ -42,9 +60,7 @@ struct tcache_bin_info_s { }; struct tcache_bin_s { -# ifdef JEMALLOC_STATS tcache_bin_stats_t tstats; -# endif int low_water; /* Min # cached since last GC. */ unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */ unsigned ncached; /* # of cached objects. */ @@ -52,12 +68,8 @@ struct tcache_bin_s { }; struct tcache_s { -# ifdef JEMALLOC_STATS ql_elm(tcache_t) link; /* Used for aggregating stats. */ -# endif -# ifdef JEMALLOC_PROF uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */ -# endif arena_t *arena; /* This thread's arena. */ unsigned ev_cnt; /* Event count since incremental GC. */ unsigned next_gc_bin; /* Next bin to GC. */ @@ -76,29 +88,11 @@ struct tcache_s { extern bool opt_tcache; extern ssize_t opt_lg_tcache_max; -extern ssize_t opt_lg_tcache_gc_sweep; extern tcache_bin_info_t *tcache_bin_info; -/* Map of thread-specific caches. */ -#ifndef NO_TLS -extern __thread tcache_t *tcache_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -# define TCACHE_GET() tcache_tls -# define TCACHE_SET(v) do { \ - tcache_tls = (tcache_t *)(v); \ - pthread_setspecific(tcache_tsd, (void *)(v)); \ -} while (0) -#else -# define TCACHE_GET() ((tcache_t *)pthread_getspecific(tcache_tsd)) -# define TCACHE_SET(v) do { \ - pthread_setspecific(tcache_tsd, (void *)(v)); \ -} while (0) -#endif -extern pthread_key_t tcache_tsd; - /* - * Number of tcache bins. There are nbins small-object bins, plus 0 or more + * Number of tcache bins. There are NBINS small-object bins, plus 0 or more * large-object bins. */ extern size_t nhbins; @@ -106,68 +100,159 @@ extern size_t nhbins; /* Maximum cached size class. */ extern size_t tcache_maxclass; -/* Number of tcache allocation/deallocation events between incremental GCs. */ -extern unsigned tcache_gc_incr; - -void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ); -void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ); -tcache_t *tcache_create(arena_t *arena); +size_t tcache_salloc(const void *ptr); +void tcache_event_hard(tcache_t *tcache); void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind); +void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache); +void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache); +void tcache_arena_associate(tcache_t *tcache, arena_t *arena); +void tcache_arena_dissociate(tcache_t *tcache); +tcache_t *tcache_create(arena_t *arena); void tcache_destroy(tcache_t *tcache); -#ifdef JEMALLOC_STATS +void tcache_thread_cleanup(void *arg); void tcache_stats_merge(tcache_t *tcache, arena_t *arena); -#endif -bool tcache_boot(void); +bool tcache_boot0(void); +bool tcache_boot1(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache, tcache_t *) +malloc_tsd_protos(JEMALLOC_ATTR(unused), tcache_enabled, tcache_enabled_t) + void tcache_event(tcache_t *tcache); -tcache_t *tcache_get(void); +void tcache_flush(void); +bool tcache_enabled_get(void); +tcache_t *tcache_get(bool create); +void tcache_enabled_set(bool enabled); void *tcache_alloc_easy(tcache_bin_t *tbin); void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); -void tcache_dalloc_small(tcache_t *tcache, void *ptr); +void tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind); void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) +/* Map of thread-specific caches. */ +malloc_tsd_externs(tcache, tcache_t *) +malloc_tsd_funcs(JEMALLOC_INLINE, tcache, tcache_t *, NULL, + tcache_thread_cleanup) +/* Per thread flag that allows thread caches to be disabled. */ +malloc_tsd_externs(tcache_enabled, tcache_enabled_t) +malloc_tsd_funcs(JEMALLOC_INLINE, tcache_enabled, tcache_enabled_t, + tcache_enabled_default, malloc_tsd_no_cleanup) + +JEMALLOC_INLINE void +tcache_flush(void) +{ + tcache_t *tcache; + + cassert(config_tcache); + + tcache = *tcache_tsd_get(); + if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) + return; + tcache_destroy(tcache); + tcache = NULL; + tcache_tsd_set(&tcache); +} + +JEMALLOC_INLINE bool +tcache_enabled_get(void) +{ + tcache_enabled_t tcache_enabled; + + cassert(config_tcache); + + tcache_enabled = *tcache_enabled_tsd_get(); + if (tcache_enabled == tcache_enabled_default) { + tcache_enabled = (tcache_enabled_t)opt_tcache; + tcache_enabled_tsd_set(&tcache_enabled); + } + + return ((bool)tcache_enabled); +} + +JEMALLOC_INLINE void +tcache_enabled_set(bool enabled) +{ + tcache_enabled_t tcache_enabled; + tcache_t *tcache; + + cassert(config_tcache); + + tcache_enabled = (tcache_enabled_t)enabled; + tcache_enabled_tsd_set(&tcache_enabled); + tcache = *tcache_tsd_get(); + if (enabled) { + if (tcache == TCACHE_STATE_DISABLED) { + tcache = NULL; + tcache_tsd_set(&tcache); + } + } else /* disabled */ { + if (tcache > TCACHE_STATE_MAX) { + tcache_destroy(tcache); + tcache = NULL; + } + if (tcache == NULL) { + tcache = TCACHE_STATE_DISABLED; + tcache_tsd_set(&tcache); + } + } +} + JEMALLOC_INLINE tcache_t * -tcache_get(void) +tcache_get(bool create) { tcache_t *tcache; - if ((isthreaded & opt_tcache) == false) + if (config_tcache == false) + return (NULL); + if (config_lazy_lock && isthreaded == false) return (NULL); - tcache = TCACHE_GET(); - if ((uintptr_t)tcache <= (uintptr_t)2) { + tcache = *tcache_tsd_get(); + if ((uintptr_t)tcache <= (uintptr_t)TCACHE_STATE_MAX) { + if (tcache == TCACHE_STATE_DISABLED) + return (NULL); if (tcache == NULL) { - tcache = tcache_create(choose_arena()); - if (tcache == NULL) - return (NULL); - } else { - if (tcache == (void *)(uintptr_t)1) { + if (create == false) { /* - * Make a note that an allocator function was - * called after the tcache_thread_cleanup() was - * called. + * Creating a tcache here would cause + * allocation as a side effect of free(). + * Ordinarily that would be okay since + * tcache_create() failure is a soft failure + * that doesn't propagate. However, if TLS + * data are freed via free() as in glibc, + * subtle corruption could result from setting + * a TLS variable after its backing memory is + * freed. */ - TCACHE_SET((uintptr_t)2); + return (NULL); + } + if (tcache_enabled_get() == false) { + tcache_enabled_set(false); /* Memoize. */ + return (NULL); } + return (tcache_create(choose_arena(NULL))); + } + if (tcache == TCACHE_STATE_PURGATORY) { + /* + * Make a note that an allocator function was called + * after tcache_thread_cleanup() was called. + */ + tcache = TCACHE_STATE_REINCARNATED; + tcache_tsd_set(&tcache); return (NULL); } + if (tcache == TCACHE_STATE_REINCARNATED) + return (NULL); + not_reached(); } return (tcache); @@ -177,60 +262,13 @@ JEMALLOC_INLINE void tcache_event(tcache_t *tcache) { - if (tcache_gc_incr == 0) + if (TCACHE_GC_INCR == 0) return; tcache->ev_cnt++; - assert(tcache->ev_cnt <= tcache_gc_incr); - if (tcache->ev_cnt == tcache_gc_incr) { - size_t binind = tcache->next_gc_bin; - tcache_bin_t *tbin = &tcache->tbins[binind]; - tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; - - if (tbin->low_water > 0) { - /* - * Flush (ceiling) 3/4 of the objects below the low - * water mark. - */ - if (binind < nbins) { - tcache_bin_flush_small(tbin, binind, - tbin->ncached - tbin->low_water + - (tbin->low_water >> 2) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } else { - tcache_bin_flush_large(tbin, binind, - tbin->ncached - tbin->low_water + - (tbin->low_water >> 2) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } - /* - * Reduce fill count by 2X. Limit lg_fill_div such that - * the fill count is always at least 1. - */ - if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) - >= 1) - tbin->lg_fill_div++; - } else if (tbin->low_water < 0) { - /* - * Increase fill count by 2X. Make sure lg_fill_div - * stays greater than 0. - */ - if (tbin->lg_fill_div > 1) - tbin->lg_fill_div--; - } - tbin->low_water = tbin->ncached; - - tcache->next_gc_bin++; - if (tcache->next_gc_bin == nhbins) - tcache->next_gc_bin = 0; - tcache->ev_cnt = 0; - } + assert(tcache->ev_cnt <= TCACHE_GC_INCR); + if (tcache->ev_cnt == TCACHE_GC_INCR) + tcache_event_hard(tcache); } JEMALLOC_INLINE void * @@ -257,7 +295,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) tcache_bin_t *tbin; binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); + assert(binind < NBINS); tbin = &tcache->tbins[binind]; ret = tcache_alloc_easy(tbin); if (ret == NULL) { @@ -265,24 +303,29 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) if (ret == NULL) return (NULL); } - assert(arena_salloc(ret) == arena_bin_info[binind].reg_size); + assert(tcache_salloc(ret) == arena_bin_info[binind].reg_size); if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else + if (config_fill) { + if (opt_junk) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (opt_zero) + memset(ret, 0, size); + } + } else { + if (config_fill && opt_junk) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } -#ifdef JEMALLOC_STATS - tbin->tstats.nrequests++; -#endif -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes += arena_bin_info[binind].reg_size; -#endif + if (config_stats) + tbin->tstats.nrequests++; + if (config_prof) + tcache->prof_accumbytes += arena_bin_info[binind].reg_size; tcache_event(tcache); return (ret); } @@ -296,7 +339,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) size = PAGE_CEILING(size); assert(size <= tcache_maxclass); - binind = nbins + (size >> PAGE_SHIFT) - 1; + binind = NBINS + (size >> LG_PAGE) - 1; assert(binind < nhbins); tbin = &tcache->tbins[binind]; ret = tcache_alloc_easy(tbin); @@ -309,28 +352,30 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) if (ret == NULL) return (NULL); } else { -#ifdef JEMALLOC_PROF - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); - size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> - PAGE_SHIFT); - chunk->map[pageind-map_bias].bits &= ~CHUNK_MAP_CLASS_MASK; -#endif + if (config_prof && prof_promote && size == PAGE) { + arena_chunk_t *chunk = + (arena_chunk_t *)CHUNK_ADDR2BASE(ret); + size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> + LG_PAGE); + arena_mapbits_large_binind_set(chunk, pageind, + BININD_INVALID); + } if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else + if (config_fill) { + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); + } + } else { + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } -#ifdef JEMALLOC_STATS - tbin->tstats.nrequests++; -#endif -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes += size; -#endif + if (config_stats) + tbin->tstats.nrequests++; + if (config_prof) + tcache->prof_accumbytes += size; } tcache_event(tcache); @@ -338,45 +383,21 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) } JEMALLOC_INLINE void -tcache_dalloc_small(tcache_t *tcache, void *ptr) +tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) { - arena_t *arena; - arena_chunk_t *chunk; - arena_run_t *run; - arena_bin_t *bin; tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; - size_t pageind, binind; - arena_chunk_map_t *mapelm; - - assert(arena_salloc(ptr) <= small_maxclass); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapelm = &chunk->map[pageind-map_bias]; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; - binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) / - sizeof(arena_bin_t); - assert(binind < nbins); - -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, arena_bin_info[binind].reg_size); -#endif + + assert(tcache_salloc(ptr) <= SMALL_MAXCLASS); + + if (config_fill && opt_junk) + arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (tbin->ncached == tbin_info->ncached_max) { tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >> - 1) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); + 1), tcache); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; @@ -388,35 +409,24 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr) JEMALLOC_INLINE void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) { - arena_t *arena; - arena_chunk_t *chunk; - size_t pageind, binind; + size_t binind; tcache_bin_t *tbin; tcache_bin_info_t *tbin_info; assert((size & PAGE_MASK) == 0); - assert(arena_salloc(ptr) > small_maxclass); - assert(arena_salloc(ptr) <= tcache_maxclass); + assert(tcache_salloc(ptr) > SMALL_MAXCLASS); + assert(tcache_salloc(ptr) <= tcache_maxclass); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - binind = nbins + (size >> PAGE_SHIFT) - 1; + binind = NBINS + (size >> LG_PAGE) - 1; -#ifdef JEMALLOC_FILL - if (opt_junk) + if (config_fill && opt_junk) memset(ptr, 0x5a, size); -#endif tbin = &tcache->tbins[binind]; tbin_info = &tcache_bin_info[binind]; if (tbin->ncached == tbin_info->ncached_max) { tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >> - 1) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); + 1), tcache); } assert(tbin->ncached < tbin_info->ncached_max); tbin->avail[tbin->ncached] = ptr; @@ -428,4 +438,3 @@ tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ -#endif /* JEMALLOC_TCACHE */ diff --git a/deps/jemalloc/include/jemalloc/internal/tsd.h b/deps/jemalloc/include/jemalloc/internal/tsd.h new file mode 100644 index 00000000000..0037cf35e70 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/tsd.h @@ -0,0 +1,397 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Maximum number of malloc_tsd users with cleanup functions. */ +#define MALLOC_TSD_CLEANUPS_MAX 8 + +typedef bool (*malloc_tsd_cleanup_t)(void); + +/* + * TLS/TSD-agnostic macro-based implementation of thread-specific data. There + * are four macros that support (at least) three use cases: file-private, + * library-private, and library-private inlined. Following is an example + * library-private tsd variable: + * + * In example.h: + * typedef struct { + * int x; + * int y; + * } example_t; + * #define EX_INITIALIZER JEMALLOC_CONCAT({0, 0}) + * malloc_tsd_protos(, example, example_t *) + * malloc_tsd_externs(example, example_t *) + * In example.c: + * malloc_tsd_data(, example, example_t *, EX_INITIALIZER) + * malloc_tsd_funcs(, example, example_t *, EX_INITIALIZER, + * example_tsd_cleanup) + * + * The result is a set of generated functions, e.g.: + * + * bool example_tsd_boot(void) {...} + * example_t **example_tsd_get() {...} + * void example_tsd_set(example_t **val) {...} + * + * Note that all of the functions deal in terms of (a_type *) rather than + * (a_type) so that it is possible to support non-pointer types (unlike + * pthreads TSD). example_tsd_cleanup() is passed an (a_type *) pointer that is + * cast to (void *). This means that the cleanup function needs to cast *and* + * dereference the function argument, e.g.: + * + * void + * example_tsd_cleanup(void *arg) + * { + * example_t *example = *(example_t **)arg; + * + * [...] + * if ([want the cleanup function to be called again]) { + * example_tsd_set(&example); + * } + * } + * + * If example_tsd_set() is called within example_tsd_cleanup(), it will be + * called again. This is similar to how pthreads TSD destruction works, except + * that pthreads only calls the cleanup function again if the value was set to + * non-NULL. + */ + +/* malloc_tsd_protos(). */ +#define malloc_tsd_protos(a_attr, a_name, a_type) \ +a_attr bool \ +a_name##_tsd_boot(void); \ +a_attr a_type * \ +a_name##_tsd_get(void); \ +a_attr void \ +a_name##_tsd_set(a_type *val); + +/* malloc_tsd_externs(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_externs(a_name, a_type) \ +extern __thread a_type a_name##_tls; \ +extern __thread bool a_name##_initialized; \ +extern bool a_name##_booted; +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_externs(a_name, a_type) \ +extern __thread a_type a_name##_tls; \ +extern pthread_key_t a_name##_tsd; \ +extern bool a_name##_booted; +#elif (defined(_WIN32)) +#define malloc_tsd_externs(a_name, a_type) \ +extern DWORD a_name##_tsd; \ +extern bool a_name##_booted; +#else +#define malloc_tsd_externs(a_name, a_type) \ +extern pthread_key_t a_name##_tsd; \ +extern bool a_name##_booted; +#endif + +/* malloc_tsd_data(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr __thread a_type JEMALLOC_TLS_MODEL \ + a_name##_tls = a_initializer; \ +a_attr __thread bool JEMALLOC_TLS_MODEL \ + a_name##_initialized = false; \ +a_attr bool a_name##_booted = false; +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr __thread a_type JEMALLOC_TLS_MODEL \ + a_name##_tls = a_initializer; \ +a_attr pthread_key_t a_name##_tsd; \ +a_attr bool a_name##_booted = false; +#elif (defined(_WIN32)) +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr DWORD a_name##_tsd; \ +a_attr bool a_name##_booted = false; +#else +#define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ +a_attr pthread_key_t a_name##_tsd; \ +a_attr bool a_name##_booted = false; +#endif + +/* malloc_tsd_funcs(). */ +#ifdef JEMALLOC_MALLOC_THREAD_CLEANUP +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Initialization/cleanup. */ \ +a_attr bool \ +a_name##_tsd_cleanup_wrapper(void) \ +{ \ + \ + if (a_name##_initialized) { \ + a_name##_initialized = false; \ + a_cleanup(&a_name##_tls); \ + } \ + return (a_name##_initialized); \ +} \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + malloc_tsd_cleanup_register( \ + &a_name##_tsd_cleanup_wrapper); \ + } \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + \ + assert(a_name##_booted); \ + return (&a_name##_tls); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + \ + assert(a_name##_booted); \ + a_name##_tls = (*val); \ + if (a_cleanup != malloc_tsd_no_cleanup) \ + a_name##_initialized = true; \ +} +#elif (defined(JEMALLOC_TLS)) +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Initialization/cleanup. */ \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + if (pthread_key_create(&a_name##_tsd, a_cleanup) != 0) \ + return (true); \ + } \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + \ + assert(a_name##_booted); \ + return (&a_name##_tls); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + \ + assert(a_name##_booted); \ + a_name##_tls = (*val); \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + if (pthread_setspecific(a_name##_tsd, \ + (void *)(&a_name##_tls))) { \ + malloc_write(": Error" \ + " setting TSD for "#a_name"\n"); \ + if (opt_abort) \ + abort(); \ + } \ + } \ +} +#elif (defined(_WIN32)) +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Data structure. */ \ +typedef struct { \ + bool initialized; \ + a_type val; \ +} a_name##_tsd_wrapper_t; \ +/* Initialization/cleanup. */ \ +a_attr bool \ +a_name##_tsd_cleanup_wrapper(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + wrapper = (a_name##_tsd_wrapper_t *) TlsGetValue(a_name##_tsd); \ + if (wrapper == NULL) \ + return (false); \ + if (a_cleanup != malloc_tsd_no_cleanup && \ + wrapper->initialized) { \ + a_type val = wrapper->val; \ + a_type tsd_static_data = a_initializer; \ + wrapper->initialized = false; \ + wrapper->val = tsd_static_data; \ + a_cleanup(&val); \ + if (wrapper->initialized) { \ + /* Trigger another cleanup round. */ \ + return (true); \ + } \ + } \ + malloc_tsd_dalloc(wrapper); \ + return (false); \ +} \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + a_name##_tsd = TlsAlloc(); \ + if (a_name##_tsd == TLS_OUT_OF_INDEXES) \ + return (true); \ + if (a_cleanup != malloc_tsd_no_cleanup) { \ + malloc_tsd_cleanup_register( \ + &a_name##_tsd_cleanup_wrapper); \ + } \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_name##_tsd_wrapper_t * \ +a_name##_tsd_get_wrapper(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ + TlsGetValue(a_name##_tsd); \ + \ + if (wrapper == NULL) { \ + wrapper = (a_name##_tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + if (wrapper == NULL) { \ + malloc_write(": Error allocating" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } else { \ + static a_type tsd_static_data = a_initializer; \ + wrapper->initialized = false; \ + wrapper->val = tsd_static_data; \ + } \ + if (!TlsSetValue(a_name##_tsd, (void *)wrapper)) { \ + malloc_write(": Error setting" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ + } \ + return (wrapper); \ +} \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + return (&wrapper->val); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + wrapper->val = *(val); \ + if (a_cleanup != malloc_tsd_no_cleanup) \ + wrapper->initialized = true; \ +} +#else +#define malloc_tsd_funcs(a_attr, a_name, a_type, a_initializer, \ + a_cleanup) \ +/* Data structure. */ \ +typedef struct { \ + bool initialized; \ + a_type val; \ +} a_name##_tsd_wrapper_t; \ +/* Initialization/cleanup. */ \ +a_attr void \ +a_name##_tsd_cleanup_wrapper(void *arg) \ +{ \ + a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *)arg;\ + \ + if (a_cleanup != malloc_tsd_no_cleanup && \ + wrapper->initialized) { \ + wrapper->initialized = false; \ + a_cleanup(&wrapper->val); \ + if (wrapper->initialized) { \ + /* Trigger another cleanup round. */ \ + if (pthread_setspecific(a_name##_tsd, \ + (void *)wrapper)) { \ + malloc_write(": Error" \ + " setting TSD for "#a_name"\n"); \ + if (opt_abort) \ + abort(); \ + } \ + return; \ + } \ + } \ + malloc_tsd_dalloc(wrapper); \ +} \ +a_attr bool \ +a_name##_tsd_boot(void) \ +{ \ + \ + if (pthread_key_create(&a_name##_tsd, \ + a_name##_tsd_cleanup_wrapper) != 0) \ + return (true); \ + a_name##_booted = true; \ + return (false); \ +} \ +/* Get/set. */ \ +a_attr a_name##_tsd_wrapper_t * \ +a_name##_tsd_get_wrapper(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper = (a_name##_tsd_wrapper_t *) \ + pthread_getspecific(a_name##_tsd); \ + \ + if (wrapper == NULL) { \ + wrapper = (a_name##_tsd_wrapper_t *) \ + malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + if (wrapper == NULL) { \ + malloc_write(": Error allocating" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } else { \ + static a_type tsd_static_data = a_initializer; \ + wrapper->initialized = false; \ + wrapper->val = tsd_static_data; \ + } \ + if (pthread_setspecific(a_name##_tsd, \ + (void *)wrapper)) { \ + malloc_write(": Error setting" \ + " TSD for "#a_name"\n"); \ + abort(); \ + } \ + } \ + return (wrapper); \ +} \ +a_attr a_type * \ +a_name##_tsd_get(void) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + return (&wrapper->val); \ +} \ +a_attr void \ +a_name##_tsd_set(a_type *val) \ +{ \ + a_name##_tsd_wrapper_t *wrapper; \ + \ + assert(a_name##_booted); \ + wrapper = a_name##_tsd_get_wrapper(); \ + wrapper->val = *(val); \ + if (a_cleanup != malloc_tsd_no_cleanup) \ + wrapper->initialized = true; \ +} +#endif + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +void malloc_tsd_no_cleanup(void *); +void malloc_tsd_cleanup_register(bool (*f)(void)); +void malloc_tsd_boot(void); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/util.h b/deps/jemalloc/include/jemalloc/internal/util.h new file mode 100644 index 00000000000..8479693631a --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/util.h @@ -0,0 +1,160 @@ +/******************************************************************************/ +#ifdef JEMALLOC_H_TYPES + +/* Size of stack-allocated buffer passed to buferror(). */ +#define BUFERROR_BUF 64 + +/* + * Size of stack-allocated buffer used by malloc_{,v,vc}printf(). This must be + * large enough for all possible uses within jemalloc. + */ +#define MALLOC_PRINTF_BUFSIZE 4096 + +/* + * Wrap a cpp argument that contains commas such that it isn't broken up into + * multiple arguments. + */ +#define JEMALLOC_CONCAT(...) __VA_ARGS__ + +/* + * Silence compiler warnings due to uninitialized values. This is used + * wherever the compiler fails to recognize that the variable is never used + * uninitialized. + */ +#ifdef JEMALLOC_CC_SILENCE +# define JEMALLOC_CC_SILENCE_INIT(v) = v +#else +# define JEMALLOC_CC_SILENCE_INIT(v) +#endif + +/* + * Define a custom assert() in order to reduce the chances of deadlock during + * assertion failure. + */ +#ifndef assert +#define assert(e) do { \ + if (config_debug && !(e)) { \ + malloc_printf( \ + ": %s:%d: Failed assertion: \"%s\"\n", \ + __FILE__, __LINE__, #e); \ + abort(); \ + } \ +} while (0) +#endif + +/* Use to assert a particular configuration, e.g., cassert(config_debug). */ +#define cassert(c) do { \ + if ((c) == false) \ + assert(false); \ +} while (0) + +#ifndef not_reached +#define not_reached() do { \ + if (config_debug) { \ + malloc_printf( \ + ": %s:%d: Unreachable code reached\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ +} while (0) +#endif + +#ifndef not_implemented +#define not_implemented() do { \ + if (config_debug) { \ + malloc_printf(": %s:%d: Not implemented\n", \ + __FILE__, __LINE__); \ + abort(); \ + } \ +} while (0) +#endif + +#define assert_not_implemented(e) do { \ + if (config_debug && !(e)) \ + not_implemented(); \ +} while (0) + +#endif /* JEMALLOC_H_TYPES */ +/******************************************************************************/ +#ifdef JEMALLOC_H_STRUCTS + +#endif /* JEMALLOC_H_STRUCTS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_EXTERNS + +int buferror(char *buf, size_t buflen); +uintmax_t malloc_strtoumax(const char *nptr, char **endptr, int base); +void malloc_write(const char *s); + +/* + * malloc_vsnprintf() supports a subset of snprintf(3) that avoids floating + * point math. + */ +int malloc_vsnprintf(char *str, size_t size, const char *format, + va_list ap); +int malloc_snprintf(char *str, size_t size, const char *format, ...) + JEMALLOC_ATTR(format(printf, 3, 4)); +void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, va_list ap); +void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, + const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); +void malloc_printf(const char *format, ...) + JEMALLOC_ATTR(format(printf, 1, 2)); + +#endif /* JEMALLOC_H_EXTERNS */ +/******************************************************************************/ +#ifdef JEMALLOC_H_INLINES + +#ifndef JEMALLOC_ENABLE_INLINE +size_t pow2_ceil(size_t x); +void malloc_write(const char *s); +void set_errno(int errnum); +int get_errno(void); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_UTIL_C_)) +/* Compute the smallest power of 2 that is >= x. */ +JEMALLOC_INLINE size_t +pow2_ceil(size_t x) +{ + + x--; + x |= x >> 1; + x |= x >> 2; + x |= x >> 4; + x |= x >> 8; + x |= x >> 16; +#if (LG_SIZEOF_PTR == 3) + x |= x >> 32; +#endif + x++; + return (x); +} + +/* Sets error code */ +JEMALLOC_INLINE void +set_errno(int errnum) +{ + +#ifdef _WIN32 + SetLastError(errnum); +#else + errno = errnum; +#endif +} + +/* Get last error code */ +JEMALLOC_INLINE int +get_errno(void) +{ + +#ifdef _WIN32 + return (GetLastError()); +#else + return (errno); +#endif +} +#endif + +#endif /* JEMALLOC_H_INLINES */ +/******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/jemalloc.h.in b/deps/jemalloc/include/jemalloc/jemalloc.h.in index 580a5ec5d5f..ad0694858da 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc.h.in @@ -15,10 +15,8 @@ extern "C" { #define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" #include "jemalloc_defs@install_suffix@.h" -#ifndef JEMALLOC_P -# define JEMALLOC_P(s) s -#endif +#ifdef JEMALLOC_EXPERIMENTAL #define ALLOCM_LG_ALIGN(la) (la) #if LG_SIZEOF_PTR == 2 #define ALLOCM_ALIGN(a) (ffs(a)-1) @@ -31,34 +29,124 @@ extern "C" { #define ALLOCM_SUCCESS 0 #define ALLOCM_ERR_OOM 1 #define ALLOCM_ERR_NOT_MOVED 2 +#endif -extern const char *JEMALLOC_P(malloc_conf); -extern void (*JEMALLOC_P(malloc_message))(void *, const char *); +/* + * The je_ prefix on the following public symbol declarations is an artifact of + * namespace management, and should be omitted in application code unless + * JEMALLOC_NO_DEMANGLE is defined (see below). + */ +extern JEMALLOC_EXPORT const char *je_malloc_conf; +extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, + const char *s); -void *JEMALLOC_P(malloc)(size_t size) JEMALLOC_ATTR(malloc); -void *JEMALLOC_P(calloc)(size_t num, size_t size) JEMALLOC_ATTR(malloc); -int JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) - JEMALLOC_ATTR(nonnull(1)); -void *JEMALLOC_P(realloc)(void *ptr, size_t size); -void JEMALLOC_P(free)(void *ptr); +JEMALLOC_EXPORT void *je_malloc(size_t size) JEMALLOC_ATTR(malloc); +JEMALLOC_EXPORT void *je_calloc(size_t num, size_t size) + JEMALLOC_ATTR(malloc); +JEMALLOC_EXPORT int je_posix_memalign(void **memptr, size_t alignment, + size_t size) JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT void *je_aligned_alloc(size_t alignment, size_t size) + JEMALLOC_ATTR(malloc); +JEMALLOC_EXPORT void *je_realloc(void *ptr, size_t size); +JEMALLOC_EXPORT void je_free(void *ptr); -size_t JEMALLOC_P(malloc_usable_size)(const void *ptr); -void JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), - void *cbopaque, const char *opts); -int JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, - void *newp, size_t newlen); -int JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, - size_t *miblenp); -int JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, +#ifdef JEMALLOC_OVERRIDE_MEMALIGN +JEMALLOC_EXPORT void * je_memalign(size_t alignment, size_t size) + JEMALLOC_ATTR(malloc); +#endif + +#ifdef JEMALLOC_OVERRIDE_VALLOC +JEMALLOC_EXPORT void * je_valloc(size_t size) JEMALLOC_ATTR(malloc); +#endif + +JEMALLOC_EXPORT size_t je_malloc_usable_size(const void *ptr); +JEMALLOC_EXPORT void je_malloc_stats_print(void (*write_cb)(void *, + const char *), void *je_cbopaque, const char *opts); +JEMALLOC_EXPORT int je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen); +JEMALLOC_EXPORT int je_mallctlnametomib(const char *name, size_t *mibp, + size_t *miblenp); +JEMALLOC_EXPORT int je_mallctlbymib(const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen); -int JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) - JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, +#ifdef JEMALLOC_EXPERIMENTAL +JEMALLOC_EXPORT int je_allocm(void **ptr, size_t *rsize, size_t size, + int flags) JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT int je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) +JEMALLOC_EXPORT int je_sallocm(const void *ptr, size_t *rsize, int flags) JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(dallocm)(void *ptr, int flags) JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT int je_dallocm(void *ptr, int flags) + JEMALLOC_ATTR(nonnull(1)); +JEMALLOC_EXPORT int je_nallocm(size_t *rsize, size_t size, int flags); +#endif + +/* + * By default application code must explicitly refer to mangled symbol names, + * so that it is possible to use jemalloc in conjunction with another allocator + * in the same application. Define JEMALLOC_MANGLE in order to cause automatic + * name mangling that matches the API prefixing that happened as a result of + * --with-mangling and/or --with-jemalloc-prefix configuration settings. + */ +#ifdef JEMALLOC_MANGLE +#ifndef JEMALLOC_NO_DEMANGLE +#define JEMALLOC_NO_DEMANGLE +#endif +#define malloc_conf je_malloc_conf +#define malloc_message je_malloc_message +#define malloc je_malloc +#define calloc je_calloc +#define posix_memalign je_posix_memalign +#define aligned_alloc je_aligned_alloc +#define realloc je_realloc +#define free je_free +#define malloc_usable_size je_malloc_usable_size +#define malloc_stats_print je_malloc_stats_print +#define mallctl je_mallctl +#define mallctlnametomib je_mallctlnametomib +#define mallctlbymib je_mallctlbymib +#define memalign je_memalign +#define valloc je_valloc +#ifdef JEMALLOC_EXPERIMENTAL +#define allocm je_allocm +#define rallocm je_rallocm +#define sallocm je_sallocm +#define dallocm je_dallocm +#define nallocm je_nallocm +#endif +#endif + +/* + * The je_* macros can be used as stable alternative names for the public + * jemalloc API if JEMALLOC_NO_DEMANGLE is defined. This is primarily meant + * for use in jemalloc itself, but it can be used by application code to + * provide isolation from the name mangling specified via --with-mangling + * and/or --with-jemalloc-prefix. + */ +#ifndef JEMALLOC_NO_DEMANGLE +#undef je_malloc_conf +#undef je_malloc_message +#undef je_malloc +#undef je_calloc +#undef je_posix_memalign +#undef je_aligned_alloc +#undef je_realloc +#undef je_free +#undef je_malloc_usable_size +#undef je_malloc_stats_print +#undef je_mallctl +#undef je_mallctlnametomib +#undef je_mallctlbymib +#undef je_memalign +#undef je_valloc +#ifdef JEMALLOC_EXPERIMENTAL +#undef je_allocm +#undef je_rallocm +#undef je_sallocm +#undef je_dallocm +#undef je_nallocm +#endif +#endif #ifdef __cplusplus }; diff --git a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in index 9ac7e1c2076..c469142a5d4 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -1,22 +1,36 @@ -#ifndef JEMALLOC_DEFS_H_ -#define JEMALLOC_DEFS_H_ - /* - * If JEMALLOC_PREFIX is defined, it will cause all public APIs to be prefixed. - * This makes it possible, with some care, to use multiple allocators - * simultaneously. - * - * In many cases it is more convenient to manually prefix allocator function - * calls than to let macros do it automatically, particularly when using - * multiple allocators simultaneously. Define JEMALLOC_MANGLE before - * #include'ing jemalloc.h in order to cause name mangling that corresponds to - * the API prefixing. + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. */ #undef JEMALLOC_PREFIX #undef JEMALLOC_CPREFIX -#if (defined(JEMALLOC_PREFIX) && defined(JEMALLOC_MANGLE)) -#undef JEMALLOC_P -#endif + +/* + * Name mangling for public symbols is controlled by --with-mangling and + * --with-jemalloc-prefix. With default settings the je_ prefix is stripped by + * these macro definitions. + */ +#undef je_malloc_conf +#undef je_malloc_message +#undef je_malloc +#undef je_calloc +#undef je_posix_memalign +#undef je_aligned_alloc +#undef je_realloc +#undef je_free +#undef je_malloc_usable_size +#undef je_malloc_stats_print +#undef je_mallctl +#undef je_mallctlnametomib +#undef je_mallctlbymib +#undef je_memalign +#undef je_valloc +#undef je_allocm +#undef je_rallocm +#undef je_sallocm +#undef je_dallocm +#undef je_nallocm /* * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. @@ -33,26 +47,92 @@ */ #undef CPU_SPINWAIT +/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */ +#undef JEMALLOC_ATOMIC9 + /* * Defined if OSAtomic*() functions are available, as provided by Darwin, and * documented in the atomic(3) manual page. */ #undef JEMALLOC_OSATOMIC +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines) + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines) + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 + /* * Defined if OSSpin*() functions are available, as provided by Darwin, and * documented in the spinlock(3) manual page. */ #undef JEMALLOC_OSSPIN +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +#undef JEMALLOC_MALLOC_THREAD_CLEANUP + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#undef JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +#undef JEMALLOC_MUTEX_INIT_CB + /* Defined if __attribute__((...)) syntax is supported. */ #undef JEMALLOC_HAVE_ATTR #ifdef JEMALLOC_HAVE_ATTR # define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +#elif _MSC_VER +# define JEMALLOC_ATTR(s) +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_NOINLINE __declspec(noinline) #else # define JEMALLOC_ATTR(s) +# define JEMALLOC_EXPORT +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_SECTION(s) +# define JEMALLOC_NOINLINE #endif +/* Defined if sbrk() is supported. */ +#undef JEMALLOC_HAVE_SBRK + +/* Non-empty if the tls_model attribute is supported. */ +#undef JEMALLOC_TLS_MODEL + /* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */ #undef JEMALLOC_CC_SILENCE @@ -77,12 +157,6 @@ /* Use gcc intrinsics for profile backtracing if defined. */ #undef JEMALLOC_PROF_GCC -/* - * JEMALLOC_TINY enables support for tiny objects, which are smaller than one - * quantum. - */ -#undef JEMALLOC_TINY - /* * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects. * This makes it possible to allocate/deallocate objects without any locking @@ -96,29 +170,43 @@ */ #undef JEMALLOC_DSS -/* JEMALLOC_SWAP enables mmap()ed swap file support. */ -#undef JEMALLOC_SWAP - -/* Support memory filling (junk/zero). */ +/* Support memory filling (junk/zero/quarantine/redzone). */ #undef JEMALLOC_FILL +/* Support the experimental API. */ +#undef JEMALLOC_EXPERIMENTAL + +/* Support utrace(2)-based tracing. */ +#undef JEMALLOC_UTRACE + +/* Support Valgrind. */ +#undef JEMALLOC_VALGRIND + /* Support optional abort() on OOM. */ #undef JEMALLOC_XMALLOC -/* Support SYSV semantics. */ -#undef JEMALLOC_SYSV - /* Support lazy locking (avoid locking unless a second thread is launched). */ #undef JEMALLOC_LAZY_LOCK -/* Determine page size at run time if defined. */ -#undef DYNAMIC_PAGE_SHIFT - /* One page is 2^STATIC_PAGE_SHIFT bytes. */ #undef STATIC_PAGE_SHIFT +/* + * If defined, use munmap() to unmap freed chunks, rather than storing them for + * later reuse. This is disabled by default on Linux because common sequences + * of mmap()/munmap() calls will cause virtual memory map holes. + */ +#undef JEMALLOC_MUNMAP + +/* + * If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). This is + * disabled by default because it is Linux-specific and it will cause virtual + * memory map holes, much like munmap(2) does. + */ +#undef JEMALLOC_MREMAP + /* TLS is used to map arenas and magazine caches to threads. */ -#undef NO_TLS +#undef JEMALLOC_TLS /* * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside @@ -139,9 +227,6 @@ #undef JEMALLOC_ZONE #undef JEMALLOC_ZONE_VERSION -/* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). */ -#undef JEMALLOC_MREMAP_FIXED - /* * Methods for purging unused pages differ between operating systems. * @@ -164,4 +249,5 @@ /* sizeof(long) == 2^LG_SIZEOF_LONG. */ #undef LG_SIZEOF_LONG -#endif /* JEMALLOC_DEFS_H_ */ +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#undef LG_SIZEOF_INTMAX_T diff --git a/deps/jemalloc/include/msvc_compat/inttypes.h b/deps/jemalloc/include/msvc_compat/inttypes.h new file mode 100644 index 00000000000..a4e6b75cb91 --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/inttypes.h @@ -0,0 +1,313 @@ +// ISO C9x compliant inttypes.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_INTTYPES_H_ // [ +#define _MSC_INTTYPES_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include "stdint.h" + +// 7.8 Format conversion of integer types + +typedef struct { + intmax_t quot; + intmax_t rem; +} imaxdiv_t; + +// 7.8.1 Macros for format specifiers + +#if !defined(__cplusplus) || defined(__STDC_FORMAT_MACROS) // [ See footnote 185 at page 198 + +#ifdef _WIN64 +# define __PRI64_PREFIX "l" +# define __PRIPTR_PREFIX "l" +#else +# define __PRI64_PREFIX "ll" +# define __PRIPTR_PREFIX +#endif + +// The fprintf macros for signed integers are: +#define PRId8 "d" +#define PRIi8 "i" +#define PRIdLEAST8 "d" +#define PRIiLEAST8 "i" +#define PRIdFAST8 "d" +#define PRIiFAST8 "i" + +#define PRId16 "hd" +#define PRIi16 "hi" +#define PRIdLEAST16 "hd" +#define PRIiLEAST16 "hi" +#define PRIdFAST16 "hd" +#define PRIiFAST16 "hi" + +#define PRId32 "d" +#define PRIi32 "i" +#define PRIdLEAST32 "d" +#define PRIiLEAST32 "i" +#define PRIdFAST32 "d" +#define PRIiFAST32 "i" + +#define PRId64 __PRI64_PREFIX "d" +#define PRIi64 __PRI64_PREFIX "i" +#define PRIdLEAST64 __PRI64_PREFIX "d" +#define PRIiLEAST64 __PRI64_PREFIX "i" +#define PRIdFAST64 __PRI64_PREFIX "d" +#define PRIiFAST64 __PRI64_PREFIX "i" + +#define PRIdMAX __PRI64_PREFIX "d" +#define PRIiMAX __PRI64_PREFIX "i" + +#define PRIdPTR __PRIPTR_PREFIX "d" +#define PRIiPTR __PRIPTR_PREFIX "i" + +// The fprintf macros for unsigned integers are: +#define PRIo8 "o" +#define PRIu8 "u" +#define PRIx8 "x" +#define PRIX8 "X" +#define PRIoLEAST8 "o" +#define PRIuLEAST8 "u" +#define PRIxLEAST8 "x" +#define PRIXLEAST8 "X" +#define PRIoFAST8 "o" +#define PRIuFAST8 "u" +#define PRIxFAST8 "x" +#define PRIXFAST8 "X" + +#define PRIo16 "ho" +#define PRIu16 "hu" +#define PRIx16 "hx" +#define PRIX16 "hX" +#define PRIoLEAST16 "ho" +#define PRIuLEAST16 "hu" +#define PRIxLEAST16 "hx" +#define PRIXLEAST16 "hX" +#define PRIoFAST16 "ho" +#define PRIuFAST16 "hu" +#define PRIxFAST16 "hx" +#define PRIXFAST16 "hX" + +#define PRIo32 "o" +#define PRIu32 "u" +#define PRIx32 "x" +#define PRIX32 "X" +#define PRIoLEAST32 "o" +#define PRIuLEAST32 "u" +#define PRIxLEAST32 "x" +#define PRIXLEAST32 "X" +#define PRIoFAST32 "o" +#define PRIuFAST32 "u" +#define PRIxFAST32 "x" +#define PRIXFAST32 "X" + +#define PRIo64 __PRI64_PREFIX "o" +#define PRIu64 __PRI64_PREFIX "u" +#define PRIx64 __PRI64_PREFIX "x" +#define PRIX64 __PRI64_PREFIX "X" +#define PRIoLEAST64 __PRI64_PREFIX "o" +#define PRIuLEAST64 __PRI64_PREFIX "u" +#define PRIxLEAST64 __PRI64_PREFIX "x" +#define PRIXLEAST64 __PRI64_PREFIX "X" +#define PRIoFAST64 __PRI64_PREFIX "o" +#define PRIuFAST64 __PRI64_PREFIX "u" +#define PRIxFAST64 __PRI64_PREFIX "x" +#define PRIXFAST64 __PRI64_PREFIX "X" + +#define PRIoMAX __PRI64_PREFIX "o" +#define PRIuMAX __PRI64_PREFIX "u" +#define PRIxMAX __PRI64_PREFIX "x" +#define PRIXMAX __PRI64_PREFIX "X" + +#define PRIoPTR __PRIPTR_PREFIX "o" +#define PRIuPTR __PRIPTR_PREFIX "u" +#define PRIxPTR __PRIPTR_PREFIX "x" +#define PRIXPTR __PRIPTR_PREFIX "X" + +// The fscanf macros for signed integers are: +#define SCNd8 "d" +#define SCNi8 "i" +#define SCNdLEAST8 "d" +#define SCNiLEAST8 "i" +#define SCNdFAST8 "d" +#define SCNiFAST8 "i" + +#define SCNd16 "hd" +#define SCNi16 "hi" +#define SCNdLEAST16 "hd" +#define SCNiLEAST16 "hi" +#define SCNdFAST16 "hd" +#define SCNiFAST16 "hi" + +#define SCNd32 "ld" +#define SCNi32 "li" +#define SCNdLEAST32 "ld" +#define SCNiLEAST32 "li" +#define SCNdFAST32 "ld" +#define SCNiFAST32 "li" + +#define SCNd64 "I64d" +#define SCNi64 "I64i" +#define SCNdLEAST64 "I64d" +#define SCNiLEAST64 "I64i" +#define SCNdFAST64 "I64d" +#define SCNiFAST64 "I64i" + +#define SCNdMAX "I64d" +#define SCNiMAX "I64i" + +#ifdef _WIN64 // [ +# define SCNdPTR "I64d" +# define SCNiPTR "I64i" +#else // _WIN64 ][ +# define SCNdPTR "ld" +# define SCNiPTR "li" +#endif // _WIN64 ] + +// The fscanf macros for unsigned integers are: +#define SCNo8 "o" +#define SCNu8 "u" +#define SCNx8 "x" +#define SCNX8 "X" +#define SCNoLEAST8 "o" +#define SCNuLEAST8 "u" +#define SCNxLEAST8 "x" +#define SCNXLEAST8 "X" +#define SCNoFAST8 "o" +#define SCNuFAST8 "u" +#define SCNxFAST8 "x" +#define SCNXFAST8 "X" + +#define SCNo16 "ho" +#define SCNu16 "hu" +#define SCNx16 "hx" +#define SCNX16 "hX" +#define SCNoLEAST16 "ho" +#define SCNuLEAST16 "hu" +#define SCNxLEAST16 "hx" +#define SCNXLEAST16 "hX" +#define SCNoFAST16 "ho" +#define SCNuFAST16 "hu" +#define SCNxFAST16 "hx" +#define SCNXFAST16 "hX" + +#define SCNo32 "lo" +#define SCNu32 "lu" +#define SCNx32 "lx" +#define SCNX32 "lX" +#define SCNoLEAST32 "lo" +#define SCNuLEAST32 "lu" +#define SCNxLEAST32 "lx" +#define SCNXLEAST32 "lX" +#define SCNoFAST32 "lo" +#define SCNuFAST32 "lu" +#define SCNxFAST32 "lx" +#define SCNXFAST32 "lX" + +#define SCNo64 "I64o" +#define SCNu64 "I64u" +#define SCNx64 "I64x" +#define SCNX64 "I64X" +#define SCNoLEAST64 "I64o" +#define SCNuLEAST64 "I64u" +#define SCNxLEAST64 "I64x" +#define SCNXLEAST64 "I64X" +#define SCNoFAST64 "I64o" +#define SCNuFAST64 "I64u" +#define SCNxFAST64 "I64x" +#define SCNXFAST64 "I64X" + +#define SCNoMAX "I64o" +#define SCNuMAX "I64u" +#define SCNxMAX "I64x" +#define SCNXMAX "I64X" + +#ifdef _WIN64 // [ +# define SCNoPTR "I64o" +# define SCNuPTR "I64u" +# define SCNxPTR "I64x" +# define SCNXPTR "I64X" +#else // _WIN64 ][ +# define SCNoPTR "lo" +# define SCNuPTR "lu" +# define SCNxPTR "lx" +# define SCNXPTR "lX" +#endif // _WIN64 ] + +#endif // __STDC_FORMAT_MACROS ] + +// 7.8.2 Functions for greatest-width integer types + +// 7.8.2.1 The imaxabs function +#define imaxabs _abs64 + +// 7.8.2.2 The imaxdiv function + +// This is modified version of div() function from Microsoft's div.c found +// in %MSVC.NET%\crt\src\div.c +#ifdef STATIC_IMAXDIV // [ +static +#else // STATIC_IMAXDIV ][ +_inline +#endif // STATIC_IMAXDIV ] +imaxdiv_t __cdecl imaxdiv(intmax_t numer, intmax_t denom) +{ + imaxdiv_t result; + + result.quot = numer / denom; + result.rem = numer % denom; + + if (numer < 0 && result.rem > 0) { + // did division wrong; must fix up + ++result.quot; + result.rem -= denom; + } + + return result; +} + +// 7.8.2.3 The strtoimax and strtoumax functions +#define strtoimax _strtoi64 +#define strtoumax _strtoui64 + +// 7.8.2.4 The wcstoimax and wcstoumax functions +#define wcstoimax _wcstoi64 +#define wcstoumax _wcstoui64 + + +#endif // _MSC_INTTYPES_H_ ] diff --git a/deps/jemalloc/include/msvc_compat/stdbool.h b/deps/jemalloc/include/msvc_compat/stdbool.h new file mode 100644 index 00000000000..da9ee8b809b --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/stdbool.h @@ -0,0 +1,16 @@ +#ifndef stdbool_h +#define stdbool_h + +#include + +/* MSVC doesn't define _Bool or bool in C, but does have BOOL */ +/* Note this doesn't pass autoconf's test because (bool) 0.5 != true */ +typedef BOOL _Bool; + +#define bool _Bool +#define true 1 +#define false 0 + +#define __bool_true_false_are_defined 1 + +#endif /* stdbool_h */ diff --git a/deps/jemalloc/include/msvc_compat/stdint.h b/deps/jemalloc/include/msvc_compat/stdint.h new file mode 100644 index 00000000000..d02608a5972 --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/stdint.h @@ -0,0 +1,247 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we should wrap include with 'extern "C++" {}' +// or compiler give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#ifdef __cplusplus +extern "C" { +#endif +# include +#ifdef __cplusplus +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +#define INTMAX_C INT64_C +#define UINTMAX_C UINT64_C + +#endif // __STDC_CONSTANT_MACROS ] + + +#endif // _MSC_STDINT_H_ ] diff --git a/deps/jemalloc/include/msvc_compat/strings.h b/deps/jemalloc/include/msvc_compat/strings.h new file mode 100644 index 00000000000..c84975b6b8e --- /dev/null +++ b/deps/jemalloc/include/msvc_compat/strings.h @@ -0,0 +1,23 @@ +#ifndef strings_h +#define strings_h + +/* MSVC doesn't define ffs/ffsl. This dummy strings.h header is provided + * for both */ +#include +#pragma intrinsic(_BitScanForward) +static __forceinline int ffsl(long x) +{ + unsigned long i; + + if (_BitScanForward(&i, x)) + return (i + 1); + return (0); +} + +static __forceinline int ffs(int x) +{ + + return (ffsl(x)); +} + +#endif diff --git a/deps/jemalloc/src/arena.c b/deps/jemalloc/src/arena.c index d166ca1ec4d..2a6150f3e8d 100644 --- a/deps/jemalloc/src/arena.c +++ b/deps/jemalloc/src/arena.c @@ -4,175 +4,60 @@ /******************************************************************************/ /* Data. */ -size_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; -size_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; -uint8_t const *small_size2bin; -arena_bin_info_t *arena_bin_info; - -/* Various bin-related settings. */ -unsigned nqbins; -unsigned ncbins; -unsigned nsbins; -unsigned nbins; -size_t qspace_max; -size_t cspace_min; -size_t cspace_max; -size_t sspace_min; -size_t sspace_max; - -size_t lg_mspace; -size_t mspace_mask; +arena_bin_info_t arena_bin_info[NBINS]; -/* - * const_small_size2bin is a static constant lookup table that in the common - * case can be used as-is for small_size2bin. - */ -#if (LG_TINY_MIN == 2) -#define S2B_4(i) i, -#define S2B_8(i) S2B_4(i) S2B_4(i) -#elif (LG_TINY_MIN == 3) +JEMALLOC_ALIGNED(CACHELINE) +const uint8_t small_size2bin[] = { #define S2B_8(i) i, -#else -# error "Unsupported LG_TINY_MIN" -#endif #define S2B_16(i) S2B_8(i) S2B_8(i) #define S2B_32(i) S2B_16(i) S2B_16(i) #define S2B_64(i) S2B_32(i) S2B_32(i) #define S2B_128(i) S2B_64(i) S2B_64(i) #define S2B_256(i) S2B_128(i) S2B_128(i) -/* - * The number of elements in const_small_size2bin is dependent on the - * definition for SUBPAGE. - */ -static JEMALLOC_ATTR(aligned(CACHELINE)) - const uint8_t const_small_size2bin[] = { -#if (LG_QUANTUM == 4) -/* 16-byte quantum **********************/ -# ifdef JEMALLOC_TINY -# if (LG_TINY_MIN == 2) - S2B_4(0) /* 4 */ - S2B_4(1) /* 8 */ - S2B_8(2) /* 16 */ -# define S2B_QMIN 2 -# elif (LG_TINY_MIN == 3) - S2B_8(0) /* 8 */ - S2B_8(1) /* 16 */ -# define S2B_QMIN 1 -# else -# error "Unsupported LG_TINY_MIN" -# endif -# else - S2B_16(0) /* 16 */ -# define S2B_QMIN 0 -# endif - S2B_16(S2B_QMIN + 1) /* 32 */ - S2B_16(S2B_QMIN + 2) /* 48 */ - S2B_16(S2B_QMIN + 3) /* 64 */ - S2B_16(S2B_QMIN + 4) /* 80 */ - S2B_16(S2B_QMIN + 5) /* 96 */ - S2B_16(S2B_QMIN + 6) /* 112 */ - S2B_16(S2B_QMIN + 7) /* 128 */ -# define S2B_CMIN (S2B_QMIN + 8) -#else -/* 8-byte quantum ***********************/ -# ifdef JEMALLOC_TINY -# if (LG_TINY_MIN == 2) - S2B_4(0) /* 4 */ - S2B_4(1) /* 8 */ -# define S2B_QMIN 1 -# else -# error "Unsupported LG_TINY_MIN" -# endif -# else - S2B_8(0) /* 8 */ -# define S2B_QMIN 0 -# endif - S2B_8(S2B_QMIN + 1) /* 16 */ - S2B_8(S2B_QMIN + 2) /* 24 */ - S2B_8(S2B_QMIN + 3) /* 32 */ - S2B_8(S2B_QMIN + 4) /* 40 */ - S2B_8(S2B_QMIN + 5) /* 48 */ - S2B_8(S2B_QMIN + 6) /* 56 */ - S2B_8(S2B_QMIN + 7) /* 64 */ - S2B_8(S2B_QMIN + 8) /* 72 */ - S2B_8(S2B_QMIN + 9) /* 80 */ - S2B_8(S2B_QMIN + 10) /* 88 */ - S2B_8(S2B_QMIN + 11) /* 96 */ - S2B_8(S2B_QMIN + 12) /* 104 */ - S2B_8(S2B_QMIN + 13) /* 112 */ - S2B_8(S2B_QMIN + 14) /* 120 */ - S2B_8(S2B_QMIN + 15) /* 128 */ -# define S2B_CMIN (S2B_QMIN + 16) -#endif -/****************************************/ - S2B_64(S2B_CMIN + 0) /* 192 */ - S2B_64(S2B_CMIN + 1) /* 256 */ - S2B_64(S2B_CMIN + 2) /* 320 */ - S2B_64(S2B_CMIN + 3) /* 384 */ - S2B_64(S2B_CMIN + 4) /* 448 */ - S2B_64(S2B_CMIN + 5) /* 512 */ -# define S2B_SMIN (S2B_CMIN + 6) - S2B_256(S2B_SMIN + 0) /* 768 */ - S2B_256(S2B_SMIN + 1) /* 1024 */ - S2B_256(S2B_SMIN + 2) /* 1280 */ - S2B_256(S2B_SMIN + 3) /* 1536 */ - S2B_256(S2B_SMIN + 4) /* 1792 */ - S2B_256(S2B_SMIN + 5) /* 2048 */ - S2B_256(S2B_SMIN + 6) /* 2304 */ - S2B_256(S2B_SMIN + 7) /* 2560 */ - S2B_256(S2B_SMIN + 8) /* 2816 */ - S2B_256(S2B_SMIN + 9) /* 3072 */ - S2B_256(S2B_SMIN + 10) /* 3328 */ - S2B_256(S2B_SMIN + 11) /* 3584 */ - S2B_256(S2B_SMIN + 12) /* 3840 */ -#if (STATIC_PAGE_SHIFT == 13) - S2B_256(S2B_SMIN + 13) /* 4096 */ - S2B_256(S2B_SMIN + 14) /* 4352 */ - S2B_256(S2B_SMIN + 15) /* 4608 */ - S2B_256(S2B_SMIN + 16) /* 4864 */ - S2B_256(S2B_SMIN + 17) /* 5120 */ - S2B_256(S2B_SMIN + 18) /* 5376 */ - S2B_256(S2B_SMIN + 19) /* 5632 */ - S2B_256(S2B_SMIN + 20) /* 5888 */ - S2B_256(S2B_SMIN + 21) /* 6144 */ - S2B_256(S2B_SMIN + 22) /* 6400 */ - S2B_256(S2B_SMIN + 23) /* 6656 */ - S2B_256(S2B_SMIN + 24) /* 6912 */ - S2B_256(S2B_SMIN + 25) /* 7168 */ - S2B_256(S2B_SMIN + 26) /* 7424 */ - S2B_256(S2B_SMIN + 27) /* 7680 */ - S2B_256(S2B_SMIN + 28) /* 7936 */ -#endif -}; -#undef S2B_1 -#undef S2B_2 -#undef S2B_4 +#define S2B_512(i) S2B_256(i) S2B_256(i) +#define S2B_1024(i) S2B_512(i) S2B_512(i) +#define S2B_2048(i) S2B_1024(i) S2B_1024(i) +#define S2B_4096(i) S2B_2048(i) S2B_2048(i) +#define S2B_8192(i) S2B_4096(i) S2B_4096(i) +#define SIZE_CLASS(bin, delta, size) \ + S2B_##delta(bin) + SIZE_CLASSES #undef S2B_8 #undef S2B_16 #undef S2B_32 #undef S2B_64 #undef S2B_128 #undef S2B_256 -#undef S2B_QMIN -#undef S2B_CMIN -#undef S2B_SMIN +#undef S2B_512 +#undef S2B_1024 +#undef S2B_2048 +#undef S2B_4096 +#undef S2B_8192 +#undef SIZE_CLASS +}; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, - bool large, bool zero); + bool large, size_t binind, bool zero); static arena_chunk_t *arena_chunk_alloc(arena_t *arena); static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); +static arena_run_t *arena_run_alloc_helper(arena_t *arena, size_t size, + bool large, size_t binind, bool zero); static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, - bool zero); + size_t binind, bool zero); static void arena_purge(arena_t *arena, bool all); static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize); static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); +static arena_run_t *arena_bin_runs_first(arena_bin_t *bin); +static void arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run); +static void arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run); +static arena_run_t *arena_bin_nonfull_run_tryget(arena_bin_t *bin); static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, @@ -187,14 +72,9 @@ static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); static bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); -static bool small_size2bin_init(void); -#ifdef JEMALLOC_DEBUG -static void small_size2bin_validate(void); -#endif -static bool small_size2bin_init_hard(void); static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size); -static bool bin_info_init(void); +static void bin_info_init(void); /******************************************************************************/ @@ -211,8 +91,8 @@ arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) } /* Generate red-black tree functions. */ -rb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, - arena_chunk_map_t, u.rb_link, arena_run_comp) +rb_gen(static UNUSED, arena_run_tree_, arena_run_tree_t, arena_chunk_map_t, + u.rb_link, arena_run_comp) static inline int arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) @@ -246,8 +126,8 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) } /* Generate red-black tree functions. */ -rb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, - arena_chunk_map_t, u.rb_link, arena_avail_comp) +rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, + u.rb_link, arena_avail_comp) static inline void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) @@ -257,13 +137,12 @@ arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); - dassert(run->magic == ARENA_RUN_MAGIC); assert(run->nfree > 0); assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + - (uintptr_t)(bin_info->reg_size * regind)); + (uintptr_t)(bin_info->reg_interval * regind)); run->nfree--; if (regind == run->nextind) run->nextind++; @@ -275,7 +154,9 @@ static inline void arena_run_reg_dalloc(arena_run_t *run, void *ptr) { arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - size_t binind = arena_bin_index(chunk->arena, run->bin); + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t mapbits = arena_mapbits_get(chunk, pageind); + size_t binind = arena_ptr_small_binind_get(ptr, mapbits); arena_bin_info_t *bin_info = &arena_bin_info[binind]; unsigned regind = arena_run_regind(run, bin_info, ptr); bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + @@ -284,8 +165,8 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) assert(run->nfree < bin_info->nregs); /* Freeing an interior pointer can cause assertion failure. */ assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size - == 0); + (uintptr_t)bin_info->reg0_offset)) % + (uintptr_t)bin_info->reg_interval == 0); assert((uintptr_t)ptr >= (uintptr_t)run + (uintptr_t)bin_info->reg0_offset); /* Freeing an unallocated pointer can cause assertion failure. */ @@ -295,75 +176,76 @@ arena_run_reg_dalloc(arena_run_t *run, void *ptr) run->nfree++; } -#ifdef JEMALLOC_DEBUG static inline void arena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) { size_t i; - size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << PAGE_SHIFT)); + UNUSED size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << LG_PAGE)); - for (i = 0; i < PAGE_SIZE / sizeof(size_t); i++) + for (i = 0; i < PAGE / sizeof(size_t); i++) assert(p[i] == 0); } -#endif static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, - bool zero) + size_t binind, bool zero) { arena_chunk_t *chunk; - size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; + size_t run_ind, total_pages, need_pages, rem_pages, i; size_t flag_dirty; arena_avail_tree_t *runs_avail; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif + + assert((large && binind == BININD_INVALID) || (large == false && binind + != BININD_INVALID)); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - old_ndirty = chunk->ndirty; - run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) - >> PAGE_SHIFT); - flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : &arena->runs_avail_clean; - total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> - PAGE_SHIFT; - assert((chunk->map[run_ind+total_pages-1-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty); - need_pages = (size >> PAGE_SHIFT); + total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> + LG_PAGE; + assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == + flag_dirty); + need_pages = (size >> LG_PAGE); assert(need_pages > 0); assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; - arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); -#ifdef JEMALLOC_STATS - /* Update stats_cactive if nactive is crossing a chunk multiple. */ - cactive_diff = CHUNK_CEILING((arena->nactive + need_pages) << - PAGE_SHIFT) - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); -#endif + arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, run_ind)); + if (config_stats) { + /* + * Update stats_cactive if nactive is crossing a chunk + * multiple. + */ + size_t cactive_diff = CHUNK_CEILING((arena->nactive + + need_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << + LG_PAGE); + if (cactive_diff != 0) + stats_cactive_add(cactive_diff); + } arena->nactive += need_pages; /* Keep track of trailing unused pages for later use. */ if (rem_pages > 0) { if (flag_dirty != 0) { - chunk->map[run_ind+need_pages-map_bias].bits = - (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; - chunk->map[run_ind+total_pages-1-map_bias].bits = - (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; + arena_mapbits_unallocated_set(chunk, run_ind+need_pages, + (rem_pages << LG_PAGE), CHUNK_MAP_DIRTY); + arena_mapbits_unallocated_set(chunk, + run_ind+total_pages-1, (rem_pages << LG_PAGE), + CHUNK_MAP_DIRTY); } else { - chunk->map[run_ind+need_pages-map_bias].bits = - (rem_pages << PAGE_SHIFT) | - (chunk->map[run_ind+need_pages-map_bias].bits & - CHUNK_MAP_UNZEROED); - chunk->map[run_ind+total_pages-1-map_bias].bits = - (rem_pages << PAGE_SHIFT) | - (chunk->map[run_ind+total_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED); + arena_mapbits_unallocated_set(chunk, run_ind+need_pages, + (rem_pages << LG_PAGE), + arena_mapbits_unzeroed_get(chunk, + run_ind+need_pages)); + arena_mapbits_unallocated_set(chunk, + run_ind+total_pages-1, (rem_pages << LG_PAGE), + arena_mapbits_unzeroed_get(chunk, + run_ind+total_pages-1)); } - arena_avail_tree_insert(runs_avail, - &chunk->map[run_ind+need_pages-map_bias]); + arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, + run_ind+need_pages)); } /* Update dirty page accounting. */ @@ -384,28 +266,34 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, * zeroed (i.e. never before touched). */ for (i = 0; i < need_pages; i++) { - if ((chunk->map[run_ind+i-map_bias].bits - & CHUNK_MAP_UNZEROED) != 0) { + if (arena_mapbits_unzeroed_get(chunk, + run_ind+i) != 0) { + VALGRIND_MAKE_MEM_UNDEFINED( + (void *)((uintptr_t) + chunk + ((run_ind+i) << + LG_PAGE)), PAGE); memset((void *)((uintptr_t) chunk + ((run_ind+i) << - PAGE_SHIFT)), 0, - PAGE_SIZE); - } -#ifdef JEMALLOC_DEBUG - else { + LG_PAGE)), 0, PAGE); + } else if (config_debug) { + VALGRIND_MAKE_MEM_DEFINED( + (void *)((uintptr_t) + chunk + ((run_ind+i) << + LG_PAGE)), PAGE); arena_chunk_validate_zeroed( chunk, run_ind+i); } -#endif } } else { /* * The run is dirty, so all pages must be * zeroed. */ + VALGRIND_MAKE_MEM_UNDEFINED((void + *)((uintptr_t)chunk + (run_ind << + LG_PAGE)), (need_pages << LG_PAGE)); memset((void *)((uintptr_t)chunk + (run_ind << - PAGE_SHIFT)), 0, (need_pages << - PAGE_SHIFT)); + LG_PAGE)), 0, (need_pages << LG_PAGE)); } } @@ -413,10 +301,9 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, * Set the last element first, in case the run only contains one * page (i.e. both statements set the same element). */ - chunk->map[run_ind+need_pages-1-map_bias].bits = - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; - chunk->map[run_ind-map_bias].bits = size | flag_dirty | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, + flag_dirty); + arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); } else { assert(zero == false); /* @@ -424,43 +311,29 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, * small run, so that arena_dalloc_bin_run() has the ability to * conditionally trim clean pages. */ - chunk->map[run_ind-map_bias].bits = - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | - CHUNK_MAP_ALLOCATED | flag_dirty; -#ifdef JEMALLOC_DEBUG + arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); /* * The first page will always be dirtied during small run * initialization, so a validation failure here would not * actually cause an observable failure. */ - if (flag_dirty == 0 && - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) - == 0) + if (config_debug && flag_dirty == 0 && + arena_mapbits_unzeroed_get(chunk, run_ind) == 0) arena_chunk_validate_zeroed(chunk, run_ind); -#endif for (i = 1; i < need_pages - 1; i++) { - chunk->map[run_ind+i-map_bias].bits = (i << PAGE_SHIFT) - | (chunk->map[run_ind+i-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; -#ifdef JEMALLOC_DEBUG - if (flag_dirty == 0 && - (chunk->map[run_ind+i-map_bias].bits & - CHUNK_MAP_UNZEROED) == 0) + arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); + if (config_debug && flag_dirty == 0 && + arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) arena_chunk_validate_zeroed(chunk, run_ind+i); -#endif } - chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages - - 1) << PAGE_SHIFT) | - (chunk->map[run_ind+need_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; -#ifdef JEMALLOC_DEBUG - if (flag_dirty == 0 && - (chunk->map[run_ind+need_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) == 0) { + arena_mapbits_small_set(chunk, run_ind+need_pages-1, + need_pages-1, binind, flag_dirty); + if (config_debug && flag_dirty == 0 && + arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == + 0) { arena_chunk_validate_zeroed(chunk, run_ind+need_pages-1); } -#endif } } @@ -476,31 +349,35 @@ arena_chunk_alloc(arena_t *arena) chunk = arena->spare; arena->spare = NULL; + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxclass); + assert(arena_mapbits_unallocated_size_get(chunk, + chunk_npages-1) == arena_maxclass); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); + /* Insert the run into the appropriate runs_avail_* tree. */ - if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) + if (arena_mapbits_dirty_get(chunk, map_bias) == 0) runs_avail = &arena->runs_avail_clean; else runs_avail = &arena->runs_avail_dirty; - assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); - assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) - == arena_maxclass); - assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == - (chunk->map[chunk_npages-1-map_bias].bits & - CHUNK_MAP_DIRTY)); - arena_avail_tree_insert(runs_avail, &chunk->map[0]); + arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, + map_bias)); } else { bool zero; size_t unzeroed; zero = false; malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); + chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, + false, &zero); malloc_mutex_lock(&arena->lock); if (chunk == NULL) return (NULL); -#ifdef JEMALLOC_STATS - arena->stats.mapped += chunksize; -#endif + if (config_stats) + arena->stats.mapped += chunksize; chunk->arena = arena; ql_elm_new(chunk, link_dirty); @@ -518,27 +395,27 @@ arena_chunk_alloc(arena_t *arena) * chunk. */ unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; - chunk->map[0].bits = arena_maxclass | unzeroed; + arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, + unzeroed); /* * There is no need to initialize the internal page map entries * unless the chunk is not zeroed. */ if (zero == false) { for (i = map_bias+1; i < chunk_npages-1; i++) - chunk->map[i-map_bias].bits = unzeroed; - } -#ifdef JEMALLOC_DEBUG - else { - for (i = map_bias+1; i < chunk_npages-1; i++) - assert(chunk->map[i-map_bias].bits == unzeroed); + arena_mapbits_unzeroed_set(chunk, i, unzeroed); + } else if (config_debug) { + for (i = map_bias+1; i < chunk_npages-1; i++) { + assert(arena_mapbits_unzeroed_get(chunk, i) == + unzeroed); + } } -#endif - chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | - unzeroed; + arena_mapbits_unallocated_set(chunk, chunk_npages-1, + arena_maxclass, unzeroed); /* Insert the run into the runs_avail_clean tree. */ arena_avail_tree_insert(&arena->runs_avail_clean, - &chunk->map[0]); + arena_mapp_get(chunk, map_bias)); } return (chunk); @@ -549,15 +426,24 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) { arena_avail_tree_t *runs_avail; + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxclass); + assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == + arena_maxclass); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); + /* * Remove run from the appropriate runs_avail_* tree, so that the arena * does not use it. */ - if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) + if (arena_mapbits_dirty_get(chunk, map_bias) == 0) runs_avail = &arena->runs_avail_clean; else runs_avail = &arena->runs_avail_dirty; - arena_avail_tree_remove(runs_avail, &chunk->map[0]); + arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, map_bias)); if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; @@ -571,24 +457,19 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) malloc_mutex_unlock(&arena->lock); chunk_dealloc((void *)spare, chunksize, true); malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_STATS - arena->stats.mapped -= chunksize; -#endif + if (config_stats) + arena->stats.mapped -= chunksize; } else arena->spare = chunk; } static arena_run_t * -arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) +arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, + bool zero) { - arena_chunk_t *chunk; arena_run_t *run; arena_chunk_map_t *mapelm, key; - assert(size <= arena_maxclass); - assert((size & PAGE_MASK) == 0); - - /* Search the arena's chunks for the lowest best fit. */ key.bits = size | CHUNK_MAP_KEY; mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); if (mapelm != NULL) { @@ -598,8 +479,8 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) + map_bias; run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + LG_PAGE)); + arena_run_split(arena, run, size, large, binind, zero); return (run); } mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); @@ -610,19 +491,38 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) + map_bias; run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + LG_PAGE)); + arena_run_split(arena, run, size, large, binind, zero); return (run); } + return (NULL); +} + +static arena_run_t * +arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, + bool zero) +{ + arena_chunk_t *chunk; + arena_run_t *run; + + assert(size <= arena_maxclass); + assert((size & PAGE_MASK) == 0); + assert((large && binind == BININD_INVALID) || (large == false && binind + != BININD_INVALID)); + + /* Search the arena's chunks for the lowest best fit. */ + run = arena_run_alloc_helper(arena, size, large, binind, zero); + if (run != NULL) + return (run); + /* * No usable runs. Create a new chunk from which to allocate the run. */ chunk = arena_chunk_alloc(arena); if (chunk != NULL) { - run = (arena_run_t *)((uintptr_t)chunk + (map_bias << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); + run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); + arena_run_split(arena, run, size, large, binind, zero); return (run); } @@ -631,32 +531,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) * sufficient memory available while this one dropped arena->lock in * arena_chunk_alloc(), so search one more time. */ - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - - return (NULL); + return (arena_run_alloc_helper(arena, size, large, binind, zero)); } static inline void @@ -677,12 +552,8 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) ql_head(arena_chunk_map_t) mapelms; arena_chunk_map_t *mapelm; size_t pageind, flag_unzeroed; -#ifdef JEMALLOC_DEBUG size_t ndirty; -#endif -#ifdef JEMALLOC_STATS size_t nmadvise; -#endif ql_new(&mapelms); @@ -692,13 +563,11 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous * mappings, but not for file-backed mappings. */ -# ifdef JEMALLOC_SWAP - swap_enabled ? CHUNK_MAP_UNZEROED : -# endif - 0; + 0 #else - CHUNK_MAP_UNZEROED; + CHUNK_MAP_UNZEROED #endif + ; /* * If chunk is the spare, temporarily re-allocate it, 1) so that its @@ -716,56 +585,61 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) * run. */ if (chunk == arena->spare) { - assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); + assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); + assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); + arena_chunk_alloc(arena); } /* Temporarily allocate all free dirty runs within chunk. */ for (pageind = map_bias; pageind < chunk_npages;) { - mapelm = &chunk->map[pageind-map_bias]; - if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { + mapelm = arena_mapp_get(chunk, pageind); + if (arena_mapbits_allocated_get(chunk, pageind) == 0) { size_t npages; - npages = mapelm->bits >> PAGE_SHIFT; + npages = arena_mapbits_unallocated_size_get(chunk, + pageind) >> LG_PAGE; assert(pageind + npages <= chunk_npages); - if (mapelm->bits & CHUNK_MAP_DIRTY) { + assert(arena_mapbits_dirty_get(chunk, pageind) == + arena_mapbits_dirty_get(chunk, pageind+npages-1)); + if (arena_mapbits_dirty_get(chunk, pageind) != 0) { size_t i; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif arena_avail_tree_remove( &arena->runs_avail_dirty, mapelm); - mapelm->bits = (npages << PAGE_SHIFT) | - flag_unzeroed | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED; + arena_mapbits_unzeroed_set(chunk, pageind, + flag_unzeroed); + arena_mapbits_large_set(chunk, pageind, + (npages << LG_PAGE), 0); /* * Update internal elements in the page map, so * that CHUNK_MAP_UNZEROED is properly set. */ for (i = 1; i < npages - 1; i++) { - chunk->map[pageind+i-map_bias].bits = - flag_unzeroed; + arena_mapbits_unzeroed_set(chunk, + pageind+i, flag_unzeroed); } if (npages > 1) { - chunk->map[ - pageind+npages-1-map_bias].bits = - flag_unzeroed | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED; + arena_mapbits_unzeroed_set(chunk, + pageind+npages-1, flag_unzeroed); + arena_mapbits_large_set(chunk, + pageind+npages-1, 0, 0); } -#ifdef JEMALLOC_STATS - /* - * Update stats_cactive if nactive is crossing a - * chunk multiple. - */ - cactive_diff = CHUNK_CEILING((arena->nactive + - npages) << PAGE_SHIFT) - - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); -#endif + if (config_stats) { + /* + * Update stats_cactive if nactive is + * crossing a chunk multiple. + */ + size_t cactive_diff = + CHUNK_CEILING((arena->nactive + + npages) << LG_PAGE) - + CHUNK_CEILING(arena->nactive << + LG_PAGE); + if (cactive_diff != 0) + stats_cactive_add(cactive_diff); + } arena->nactive += npages; /* Append to list for later processing. */ ql_elm_new(mapelm, u.ql_link); @@ -775,71 +649,57 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) pageind += npages; } else { /* Skip allocated run. */ - if (mapelm->bits & CHUNK_MAP_LARGE) - pageind += mapelm->bits >> PAGE_SHIFT; + if (arena_mapbits_large_get(chunk, pageind)) + pageind += arena_mapbits_large_size_get(chunk, + pageind) >> LG_PAGE; else { + size_t binind; + arena_bin_info_t *bin_info; arena_run_t *run = (arena_run_t *)((uintptr_t) - chunk + (uintptr_t)(pageind << PAGE_SHIFT)); - - assert((mapelm->bits >> PAGE_SHIFT) == 0); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(arena, - run->bin); - arena_bin_info_t *bin_info = - &arena_bin_info[binind]; - pageind += bin_info->run_size >> PAGE_SHIFT; + chunk + (uintptr_t)(pageind << LG_PAGE)); + + assert(arena_mapbits_small_runind_get(chunk, + pageind) == 0); + binind = arena_bin_index(arena, run->bin); + bin_info = &arena_bin_info[binind]; + pageind += bin_info->run_size >> LG_PAGE; } } } assert(pageind == chunk_npages); -#ifdef JEMALLOC_DEBUG - ndirty = chunk->ndirty; -#endif -#ifdef JEMALLOC_STATS - arena->stats.purged += chunk->ndirty; -#endif + if (config_debug) + ndirty = chunk->ndirty; + if (config_stats) + arena->stats.purged += chunk->ndirty; arena->ndirty -= chunk->ndirty; chunk->ndirty = 0; ql_remove(&arena->chunks_dirty, chunk, link_dirty); chunk->dirtied = false; malloc_mutex_unlock(&arena->lock); -#ifdef JEMALLOC_STATS - nmadvise = 0; -#endif + if (config_stats) + nmadvise = 0; ql_foreach(mapelm, &mapelms, u.ql_link) { size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t)) + map_bias; - size_t npages = mapelm->bits >> PAGE_SHIFT; + size_t npages = arena_mapbits_large_size_get(chunk, pageind) >> + LG_PAGE; assert(pageind + npages <= chunk_npages); -#ifdef JEMALLOC_DEBUG assert(ndirty >= npages); - ndirty -= npages; -#endif + if (config_debug) + ndirty -= npages; -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), - (npages << PAGE_SHIFT), MADV_DONTNEED); -#elif defined(JEMALLOC_PURGE_MADVISE_FREE) - madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), - (npages << PAGE_SHIFT), MADV_FREE); -#else -# error "No method defined for purging unused dirty pages." -#endif - -#ifdef JEMALLOC_STATS - nmadvise++; -#endif + pages_purge((void *)((uintptr_t)chunk + (pageind << LG_PAGE)), + (npages << LG_PAGE)); + if (config_stats) + nmadvise++; } -#ifdef JEMALLOC_DEBUG assert(ndirty == 0); -#endif malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_STATS - arena->stats.nmadvise += nmadvise; -#endif + if (config_stats) + arena->stats.nmadvise += nmadvise; /* Deallocate runs. */ for (mapelm = ql_first(&mapelms); mapelm != NULL; @@ -847,7 +707,7 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t)) + map_bias; arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)(pageind << PAGE_SHIFT)); + (uintptr_t)(pageind << LG_PAGE)); ql_remove(&mapelms, mapelm, u.ql_link); arena_run_dalloc(arena, run, false); @@ -859,23 +719,22 @@ arena_purge(arena_t *arena, bool all) { arena_chunk_t *chunk; size_t npurgatory; -#ifdef JEMALLOC_DEBUG - size_t ndirty = 0; + if (config_debug) { + size_t ndirty = 0; - ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { - assert(chunk->dirtied); - ndirty += chunk->ndirty; + ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { + assert(chunk->dirtied); + ndirty += chunk->ndirty; + } + assert(ndirty == arena->ndirty); } - assert(ndirty == arena->ndirty); -#endif assert(arena->ndirty > arena->npurgatory || all); assert(arena->ndirty - arena->npurgatory > chunk_npages || all); assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - arena->npurgatory) || all); -#ifdef JEMALLOC_STATS - arena->stats.npurge++; -#endif + if (config_stats) + arena->stats.npurge++; /* * Compute the minimum number of pages that this thread should try to @@ -957,44 +816,41 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) arena_chunk_t *chunk; size_t size, run_ind, run_pages, flag_dirty; arena_avail_tree_t *runs_avail; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) - >> PAGE_SHIFT); + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); assert(run_ind >= map_bias); assert(run_ind < chunk_npages); - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { - size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; - assert(size == PAGE_SIZE || - (chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - ~PAGE_MASK) == 0); - assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - CHUNK_MAP_LARGE) != 0); - assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) != 0); + if (arena_mapbits_large_get(chunk, run_ind) != 0) { + size = arena_mapbits_large_size_get(chunk, run_ind); + assert(size == PAGE || + arena_mapbits_large_size_get(chunk, + run_ind+(size>>LG_PAGE)-1) == 0); } else { size_t binind = arena_bin_index(arena, run->bin); arena_bin_info_t *bin_info = &arena_bin_info[binind]; size = bin_info->run_size; } - run_pages = (size >> PAGE_SHIFT); -#ifdef JEMALLOC_STATS - /* Update stats_cactive if nactive is crossing a chunk multiple. */ - cactive_diff = CHUNK_CEILING(arena->nactive << PAGE_SHIFT) - - CHUNK_CEILING((arena->nactive - run_pages) << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_sub(cactive_diff); -#endif + run_pages = (size >> LG_PAGE); + if (config_stats) { + /* + * Update stats_cactive if nactive is crossing a chunk + * multiple. + */ + size_t cactive_diff = CHUNK_CEILING(arena->nactive << LG_PAGE) - + CHUNK_CEILING((arena->nactive - run_pages) << LG_PAGE); + if (cactive_diff != 0) + stats_cactive_sub(cactive_diff); + } arena->nactive -= run_pages; /* * The run is dirty if the caller claims to have dirtied it, as well as * if it was already dirty before being allocated. */ - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + if (arena_mapbits_dirty_get(chunk, run_ind) != 0) dirty = true; flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; runs_avail = dirty ? &arena->runs_avail_dirty : @@ -1002,59 +858,53 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) /* Mark pages as unallocated in the chunk map. */ if (dirty) { - chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - CHUNK_MAP_DIRTY; + arena_mapbits_unallocated_set(chunk, run_ind, size, + CHUNK_MAP_DIRTY); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + CHUNK_MAP_DIRTY); chunk->ndirty += run_pages; arena->ndirty += run_pages; } else { - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED); + arena_mapbits_unallocated_set(chunk, run_ind, size, + arena_mapbits_unzeroed_get(chunk, run_ind)); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); } /* Try to coalesce forward. */ if (run_ind + run_pages < chunk_npages && - (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) - == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty) { - size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & - ~PAGE_MASK; - size_t nrun_pages = nrun_size >> PAGE_SHIFT; + arena_mapbits_allocated_get(chunk, run_ind+run_pages) == 0 && + arena_mapbits_dirty_get(chunk, run_ind+run_pages) == flag_dirty) { + size_t nrun_size = arena_mapbits_unallocated_size_get(chunk, + run_ind+run_pages); + size_t nrun_pages = nrun_size >> LG_PAGE; /* * Remove successor from runs_avail; the coalesced run is * inserted later. */ - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & ~PAGE_MASK) == nrun_size); - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & CHUNK_MAP_ALLOCATED) == 0); - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & CHUNK_MAP_DIRTY) == flag_dirty); + assert(arena_mapbits_unallocated_size_get(chunk, + run_ind+run_pages+nrun_pages-1) == nrun_size); + assert(arena_mapbits_dirty_get(chunk, + run_ind+run_pages+nrun_pages-1) == flag_dirty); arena_avail_tree_remove(runs_avail, - &chunk->map[run_ind+run_pages-map_bias]); + arena_mapp_get(chunk, run_ind+run_pages)); size += nrun_size; run_pages += nrun_pages; - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); + arena_mapbits_unallocated_size_set(chunk, run_ind, size); + arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, + size); } /* Try to coalesce backward. */ - if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty) { - size_t prun_size = chunk->map[run_ind-1-map_bias].bits & - ~PAGE_MASK; - size_t prun_pages = prun_size >> PAGE_SHIFT; + if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1) + == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) { + size_t prun_size = arena_mapbits_unallocated_size_get(chunk, + run_ind-1); + size_t prun_pages = prun_size >> LG_PAGE; run_ind -= prun_pages; @@ -1062,31 +912,26 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) * Remove predecessor from runs_avail; the coalesced run is * inserted later. */ - assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) - == prun_size); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) - == 0); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) - == flag_dirty); - arena_avail_tree_remove(runs_avail, - &chunk->map[run_ind-map_bias]); + assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == + prun_size); + assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); + arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, + run_ind)); size += prun_size; run_pages += prun_pages; - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); + arena_mapbits_unallocated_size_set(chunk, run_ind, size); + arena_mapbits_unallocated_size_set(chunk, run_ind+run_pages-1, + size); } /* Insert into runs_avail, now that coalescing is complete. */ - assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == - (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == - (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); - arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); + assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == + arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, run_ind)); if (dirty) { /* @@ -1100,14 +945,12 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) } } - /* - * Deallocate chunk if it is now completely unused. The bit - * manipulation checks whether the first run is unallocated and extends - * to the end of the chunk. - */ - if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == - arena_maxclass) + /* Deallocate chunk if it is now completely unused. */ + if (size == arena_maxclass) { + assert(run_ind == map_bias); + assert(run_pages == (arena_maxclass >> LG_PAGE)); arena_chunk_dealloc(arena, chunk); + } /* * It is okay to do dirty page processing here even if the chunk was @@ -1124,9 +967,9 @@ static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize) { - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; - size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + size_t head_npages = (oldsize - newsize) >> LG_PAGE; + size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); assert(oldsize > newsize); @@ -1135,31 +978,19 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, * leading run as separately allocated. Set the last element of each * run first, in case of single-page runs. */ - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | - (chunk->map[pageind+head_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind-map_bias].bits = (oldsize - newsize) - | flag_dirty | (chunk->map[pageind-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - -#ifdef JEMALLOC_DEBUG - { - size_t tail_npages = newsize >> PAGE_SHIFT; - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & ~PAGE_MASK) == 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_DIRTY) == flag_dirty); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_ALLOCATED) != 0); + assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); + arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); + arena_mapbits_large_set(chunk, pageind, oldsize-newsize, flag_dirty); + + if (config_debug) { + UNUSED size_t tail_npages = newsize >> LG_PAGE; + assert(arena_mapbits_large_size_get(chunk, + pageind+head_npages+tail_npages-1) == 0); + assert(arena_mapbits_dirty_get(chunk, + pageind+head_npages+tail_npages-1) == flag_dirty); } -#endif - chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | - (chunk->map[pageind+head_npages-map_bias].bits & - CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + arena_mapbits_large_set(chunk, pageind+head_npages, newsize, + flag_dirty); arena_run_dalloc(arena, run, false); } @@ -1168,11 +999,9 @@ static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize, bool dirty) { - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t head_npages = newsize >> PAGE_SHIFT; - size_t tail_npages = (oldsize - newsize) >> PAGE_SHIFT; - size_t flag_dirty = chunk->map[pageind-map_bias].bits & - CHUNK_MAP_DIRTY; + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + size_t head_npages = newsize >> LG_PAGE; + size_t flag_dirty = arena_mapbits_dirty_get(chunk, pageind); assert(oldsize > newsize); @@ -1181,61 +1010,92 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, * trailing run as separately allocated. Set the last element of each * run first, in case of single-page runs. */ - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | - (chunk->map[pageind+head_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind-map_bias].bits = newsize | flag_dirty | - (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - ~PAGE_MASK) == 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = - flag_dirty | - (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | - flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + assert(arena_mapbits_large_size_get(chunk, pageind) == oldsize); + arena_mapbits_large_set(chunk, pageind+head_npages-1, 0, flag_dirty); + arena_mapbits_large_set(chunk, pageind, newsize, flag_dirty); + + if (config_debug) { + UNUSED size_t tail_npages = (oldsize - newsize) >> LG_PAGE; + assert(arena_mapbits_large_size_get(chunk, + pageind+head_npages+tail_npages-1) == 0); + assert(arena_mapbits_dirty_get(chunk, + pageind+head_npages+tail_npages-1) == flag_dirty); + } + arena_mapbits_large_set(chunk, pageind+head_npages, oldsize-newsize, + flag_dirty); arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), dirty); } static arena_run_t * -arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) +arena_bin_runs_first(arena_bin_t *bin) { - arena_chunk_map_t *mapelm; - arena_run_t *run; - size_t binind; - arena_bin_info_t *bin_info; - - /* Look for a usable run. */ - mapelm = arena_run_tree_first(&bin->runs); + arena_chunk_map_t *mapelm = arena_run_tree_first(&bin->runs); if (mapelm != NULL) { arena_chunk_t *chunk; size_t pageind; - - /* run is guaranteed to have available space. */ - arena_run_tree_remove(&bin->runs, mapelm); + arena_run_t *run; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t))) + map_bias; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) - << PAGE_SHIFT)); -#ifdef JEMALLOC_STATS - bin->stats.reruns++; -#endif + arena_mapbits_small_runind_get(chunk, pageind)) << + LG_PAGE)); return (run); } + + return (NULL); +} + +static void +arena_bin_runs_insert(arena_bin_t *bin, arena_run_t *run) +{ + arena_chunk_t *chunk = CHUNK_ADDR2BASE(run); + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + + assert(arena_run_tree_search(&bin->runs, mapelm) == NULL); + + arena_run_tree_insert(&bin->runs, mapelm); +} + +static void +arena_bin_runs_remove(arena_bin_t *bin, arena_run_t *run) +{ + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE; + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); + + assert(arena_run_tree_search(&bin->runs, mapelm) != NULL); + + arena_run_tree_remove(&bin->runs, mapelm); +} + +static arena_run_t * +arena_bin_nonfull_run_tryget(arena_bin_t *bin) +{ + arena_run_t *run = arena_bin_runs_first(bin); + if (run != NULL) { + arena_bin_runs_remove(bin, run); + if (config_stats) + bin->stats.reruns++; + } + return (run); +} + +static arena_run_t * +arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) +{ + arena_run_t *run; + size_t binind; + arena_bin_info_t *bin_info; + + /* Look for a usable run. */ + run = arena_bin_nonfull_run_tryget(bin); + if (run != NULL) + return (run); /* No existing runs have any space available. */ binind = arena_bin_index(arena, bin); @@ -1245,30 +1105,27 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) malloc_mutex_unlock(&bin->lock); /******************************/ malloc_mutex_lock(&arena->lock); - run = arena_run_alloc(arena, bin_info->run_size, false, false); + run = arena_run_alloc(arena, bin_info->run_size, false, binind, false); if (run != NULL) { bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); /* Initialize run internals. */ + VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset - + bin_info->redzone_size); run->bin = bin; run->nextind = 0; run->nfree = bin_info->nregs; bitmap_init(bitmap, &bin_info->bitmap_info); -#ifdef JEMALLOC_DEBUG - run->magic = ARENA_RUN_MAGIC; -#endif } malloc_mutex_unlock(&arena->lock); /********************************/ malloc_mutex_lock(&bin->lock); if (run != NULL) { -#ifdef JEMALLOC_STATS - bin->stats.nruns++; - bin->stats.curruns++; - if (bin->stats.curruns > bin->stats.highruns) - bin->stats.highruns = bin->stats.curruns; -#endif + if (config_stats) { + bin->stats.nruns++; + bin->stats.curruns++; + } return (run); } @@ -1277,25 +1134,9 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) * sufficient memory available while this one dropped bin->lock above, * so search one more time. */ - mapelm = arena_run_tree_first(&bin->runs); - if (mapelm != NULL) { - arena_chunk_t *chunk; - size_t pageind; - - /* run is guaranteed to have available space. */ - arena_run_tree_remove(&bin->runs, mapelm); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t))) + map_bias; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) - << PAGE_SHIFT)); -#ifdef JEMALLOC_STATS - bin->stats.reruns++; -#endif + run = arena_bin_nonfull_run_tryget(bin); + if (run != NULL) return (run); - } return (NULL); } @@ -1318,7 +1159,6 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) * Another thread updated runcur while this one ran without the * bin lock in arena_bin_nonfull_run_get(). */ - dassert(bin->runcur->magic == ARENA_RUN_MAGIC); assert(bin->runcur->nfree > 0); ret = arena_run_reg_alloc(bin->runcur, bin_info); if (run != NULL) { @@ -1346,18 +1186,18 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) bin->runcur = run; - dassert(bin->runcur->magic == ARENA_RUN_MAGIC); assert(bin->runcur->nfree > 0); return (arena_run_reg_alloc(bin->runcur, bin_info)); } -#ifdef JEMALLOC_PROF void arena_prof_accum(arena_t *arena, uint64_t accumbytes) { - if (prof_interval != 0) { + cassert(config_prof); + + if (config_prof && prof_interval != 0) { arena->prof_accumbytes += accumbytes; if (arena->prof_accumbytes >= prof_interval) { prof_idump(); @@ -1365,15 +1205,10 @@ arena_prof_accum(arena_t *arena, uint64_t accumbytes) } } } -#endif -#ifdef JEMALLOC_TCACHE void -arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind -# ifdef JEMALLOC_PROF - , uint64_t prof_accumbytes -# endif - ) +arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, + uint64_t prof_accumbytes) { unsigned i, nfill; arena_bin_t *bin; @@ -1382,11 +1217,11 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind assert(tbin->ncached == 0); -#ifdef JEMALLOC_PROF - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, prof_accumbytes); - malloc_mutex_unlock(&arena->lock); -#endif + if (config_prof) { + malloc_mutex_lock(&arena->lock); + arena_prof_accum(arena, prof_accumbytes); + malloc_mutex_unlock(&arena->lock); + } bin = &arena->bins[binind]; malloc_mutex_lock(&bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> @@ -1397,20 +1232,72 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind ptr = arena_bin_malloc_hard(arena, bin); if (ptr == NULL) break; + if (config_fill && opt_junk) { + arena_alloc_junk_small(ptr, &arena_bin_info[binind], + true); + } /* Insert such that low regions get used first. */ tbin->avail[nfill - 1 - i] = ptr; } -#ifdef JEMALLOC_STATS - bin->stats.allocated += i * arena_bin_info[binind].reg_size; - bin->stats.nmalloc += i; - bin->stats.nrequests += tbin->tstats.nrequests; - bin->stats.nfills++; - tbin->tstats.nrequests = 0; -#endif + if (config_stats) { + bin->stats.allocated += i * arena_bin_info[binind].reg_size; + bin->stats.nmalloc += i; + bin->stats.nrequests += tbin->tstats.nrequests; + bin->stats.nfills++; + tbin->tstats.nrequests = 0; + } malloc_mutex_unlock(&bin->lock); tbin->ncached = i; } -#endif + +void +arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) +{ + + if (zero) { + size_t redzone_size = bin_info->redzone_size; + memset((void *)((uintptr_t)ptr - redzone_size), 0xa5, + redzone_size); + memset((void *)((uintptr_t)ptr + bin_info->reg_size), 0xa5, + redzone_size); + } else { + memset((void *)((uintptr_t)ptr - bin_info->redzone_size), 0xa5, + bin_info->reg_interval); + } +} + +void +arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) +{ + size_t size = bin_info->reg_size; + size_t redzone_size = bin_info->redzone_size; + size_t i; + bool error = false; + + for (i = 1; i <= redzone_size; i++) { + unsigned byte; + if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) { + error = true; + malloc_printf(": Corrupt redzone " + "%zu byte%s before %p (size %zu), byte=%#x\n", i, + (i == 1) ? "" : "s", ptr, size, byte); + } + } + for (i = 0; i < redzone_size; i++) { + unsigned byte; + if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) { + error = true; + malloc_printf(": Corrupt redzone " + "%zu byte%s after end of %p (size %zu), byte=%#x\n", + i, (i == 1) ? "" : "s", ptr, size, byte); + } + } + if (opt_abort && error) + abort(); + + memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, + bin_info->reg_interval); +} void * arena_malloc_small(arena_t *arena, size_t size, bool zero) @@ -1421,7 +1308,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) size_t binind; binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); + assert(binind < NBINS); bin = &arena->bins[binind]; size = arena_bin_info[binind].reg_size; @@ -1436,29 +1323,34 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) return (NULL); } -#ifdef JEMALLOC_STATS - bin->stats.allocated += size; - bin->stats.nmalloc++; - bin->stats.nrequests++; -#endif + if (config_stats) { + bin->stats.allocated += size; + bin->stats.nmalloc++; + bin->stats.nrequests++; + } malloc_mutex_unlock(&bin->lock); -#ifdef JEMALLOC_PROF - if (isthreaded == false) { + if (config_prof && isthreaded == false) { malloc_mutex_lock(&arena->lock); arena_prof_accum(arena, size); malloc_mutex_unlock(&arena->lock); } -#endif if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else + if (config_fill) { + if (opt_junk) { + arena_alloc_junk_small(ret, + &arena_bin_info[binind], false); + } else if (opt_zero) + memset(ret, 0, size); + } + } else { + if (config_fill && opt_junk) { + arena_alloc_junk_small(ret, &arena_bin_info[binind], + true); + } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); + } return (ret); } @@ -1471,243 +1363,112 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) /* Large allocation. */ size = PAGE_CEILING(size); malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, size, true, zero); + ret = (void *)arena_run_alloc(arena, size, true, BININD_INVALID, zero); if (ret == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } -#ifdef JEMALLOC_STATS - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + if (config_stats) { + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif -#ifdef JEMALLOC_PROF - arena_prof_accum(arena, size); -#endif + if (config_prof) + arena_prof_accum(arena, size); malloc_mutex_unlock(&arena->lock); if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif + if (config_fill) { + if (opt_junk) + memset(ret, 0xa5, size); + else if (opt_zero) + memset(ret, 0, size); + } } return (ret); } -void * -arena_malloc(size_t size, bool zero) -{ - - assert(size != 0); - assert(QUANTUM_CEILING(size) <= arena_maxclass); - - if (size <= small_maxclass) { -#ifdef JEMALLOC_TCACHE - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - return (tcache_alloc_small(tcache, size, zero)); - else - -#endif - return (arena_malloc_small(choose_arena(), size, zero)); - } else { -#ifdef JEMALLOC_TCACHE - if (size <= tcache_maxclass) { - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - return (tcache_alloc_large(tcache, size, zero)); - else { - return (arena_malloc_large(choose_arena(), - size, zero)); - } - } else -#endif - return (arena_malloc_large(choose_arena(), size, zero)); - } -} - /* Only handles large allocations that require more than page alignment. */ void * -arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, - bool zero) +arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) { void *ret; - size_t offset; + size_t alloc_size, leadsize, trailsize; + arena_run_t *run; arena_chunk_t *chunk; assert((size & PAGE_MASK) == 0); alignment = PAGE_CEILING(alignment); + alloc_size = size + alignment - PAGE; malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); - if (ret == NULL) { + run = arena_run_alloc(arena, alloc_size, true, BININD_INVALID, zero); + if (run == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); } + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); - - offset = (uintptr_t)ret & (alignment - 1); - assert((offset & PAGE_MASK) == 0); - assert(offset < alloc_size); - if (offset == 0) - arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); - else { - size_t leadsize, trailsize; - - leadsize = alignment - offset; - if (leadsize > 0) { - arena_run_trim_head(arena, chunk, ret, alloc_size, - alloc_size - leadsize); - ret = (void *)((uintptr_t)ret + leadsize); - } - - trailsize = alloc_size - leadsize - size; - if (trailsize != 0) { - /* Trim trailing space. */ - assert(trailsize < alloc_size); - arena_run_trim_tail(arena, chunk, ret, size + trailsize, - size, false); - } + leadsize = ALIGNMENT_CEILING((uintptr_t)run, alignment) - + (uintptr_t)run; + assert(alloc_size >= leadsize + size); + trailsize = alloc_size - leadsize - size; + ret = (void *)((uintptr_t)run + leadsize); + if (leadsize != 0) { + arena_run_trim_head(arena, chunk, run, alloc_size, alloc_size - + leadsize); + } + if (trailsize != 0) { + arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, + false); } -#ifdef JEMALLOC_STATS - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + if (config_stats) { + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif malloc_mutex_unlock(&arena->lock); -#ifdef JEMALLOC_FILL - if (zero == false) { + if (config_fill && zero == false) { if (opt_junk) memset(ret, 0xa5, size); else if (opt_zero) memset(ret, 0, size); } -#endif return (ret); } -/* Return the size of the allocation pointed to by ptr. */ -size_t -arena_salloc(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - assert(ret != 0); - } - - return (ret); -} - -#ifdef JEMALLOC_PROF void arena_prof_promoted(const void *ptr, size_t size) { arena_chunk_t *chunk; size_t pageind, binind; + cassert(config_prof); assert(ptr != NULL); assert(CHUNK_ADDR2BASE(ptr) != ptr); - assert(isalloc(ptr) == PAGE_SIZE); - assert(size <= small_maxclass); + assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, true) == PAGE); + assert(size <= SMALL_MAXCLASS); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); - chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & - ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); -} - -size_t -arena_salloc_demote(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - if (prof_promote && ret == PAGE_SIZE && (mapbits & - CHUNK_MAP_CLASS_MASK) != 0) { - size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> - CHUNK_MAP_CLASS_SHIFT) - 1; - assert(binind < nbins); - ret = arena_bin_info[binind].reg_size; - } - assert(ret != 0); - } + assert(binind < NBINS); + arena_mapbits_large_binind_set(chunk, pageind, binind); - return (ret); + assert(isalloc(ptr, false) == PAGE); + assert(isalloc(ptr, true) == size); } -#endif static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, @@ -1722,16 +1483,12 @@ arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, arena_bin_info_t *bin_info = &arena_bin_info[binind]; if (bin_info->nregs != 1) { - size_t run_pageind = (((uintptr_t)run - - (uintptr_t)chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *run_mapelm = - &chunk->map[run_pageind-map_bias]; /* * This block's conditional is necessary because if the * run only contains one region, then it never gets * inserted into the non-full runs tree. */ - arena_run_tree_remove(&bin->runs, run_mapelm); + arena_bin_runs_remove(bin, run); } } } @@ -1745,19 +1502,21 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t npages, run_ind, past; assert(run != bin->runcur); - assert(arena_run_tree_search(&bin->runs, &chunk->map[ - (((uintptr_t)run-(uintptr_t)chunk)>>PAGE_SHIFT)-map_bias]) == NULL); + assert(arena_run_tree_search(&bin->runs, + arena_mapp_get(chunk, ((uintptr_t)run-(uintptr_t)chunk)>>LG_PAGE)) + == NULL); binind = arena_bin_index(chunk->arena, run->bin); bin_info = &arena_bin_info[binind]; malloc_mutex_unlock(&bin->lock); /******************************/ - npages = bin_info->run_size >> PAGE_SHIFT; - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); + npages = bin_info->run_size >> LG_PAGE; + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); past = (size_t)(PAGE_CEILING((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * - bin_info->reg_size) - (uintptr_t)chunk) >> PAGE_SHIFT); + bin_info->reg_interval - bin_info->redzone_size) - + (uintptr_t)chunk) >> LG_PAGE); malloc_mutex_lock(&arena->lock); /* @@ -1765,32 +1524,24 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, * trim the clean pages before deallocating the dirty portion of the * run. */ - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past - - run_ind < npages) { - /* - * Trim clean pages. Convert to large run beforehand. Set the - * last map element first, in case this is a one-page run. - */ - chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | - (chunk->map[run_ind+npages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind-map_bias].bits = bin_info->run_size | - CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - arena_run_trim_tail(arena, chunk, run, (npages << PAGE_SHIFT), - ((past - run_ind) << PAGE_SHIFT), false); + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+npages-1)); + if (arena_mapbits_dirty_get(chunk, run_ind) == 0 && past - run_ind < + npages) { + /* Trim clean pages. Convert to large run beforehand. */ + assert(npages > 0); + arena_mapbits_large_set(chunk, run_ind, bin_info->run_size, 0); + arena_mapbits_large_set(chunk, run_ind+npages-1, 0, 0); + arena_run_trim_tail(arena, chunk, run, (npages << LG_PAGE), + ((past - run_ind) << LG_PAGE), false); /* npages = past - run_ind; */ } -#ifdef JEMALLOC_DEBUG - run->magic = 0; -#endif arena_run_dalloc(arena, run, true); malloc_mutex_unlock(&arena->lock); /****************************/ malloc_mutex_lock(&bin->lock); -#ifdef JEMALLOC_STATS - bin->stats.curruns--; -#endif + if (config_stats) + bin->stats.curruns--; } static void @@ -1799,62 +1550,42 @@ arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, { /* - * Make sure that bin->runcur always refers to the lowest non-full run, - * if one exists. + * Make sure that if bin->runcur is non-NULL, it refers to the lowest + * non-full run. It is okay to NULL runcur out rather than proactively + * keeping it pointing at the lowest non-full run. */ - if (bin->runcur == NULL) - bin->runcur = run; - else if ((uintptr_t)run < (uintptr_t)bin->runcur) { + if ((uintptr_t)run < (uintptr_t)bin->runcur) { /* Switch runcur. */ - if (bin->runcur->nfree > 0) { - arena_chunk_t *runcur_chunk = - CHUNK_ADDR2BASE(bin->runcur); - size_t runcur_pageind = (((uintptr_t)bin->runcur - - (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *runcur_mapelm = - &runcur_chunk->map[runcur_pageind-map_bias]; - - /* Insert runcur. */ - arena_run_tree_insert(&bin->runs, runcur_mapelm); - } + if (bin->runcur->nfree > 0) + arena_bin_runs_insert(bin, bin->runcur); bin->runcur = run; - } else { - size_t run_pageind = (((uintptr_t)run - - (uintptr_t)chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *run_mapelm = - &chunk->map[run_pageind-map_bias]; - - assert(arena_run_tree_search(&bin->runs, run_mapelm) == NULL); - arena_run_tree_insert(&bin->runs, run_mapelm); - } + if (config_stats) + bin->stats.reruns++; + } else + arena_bin_runs_insert(bin, run); } void -arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, +arena_dalloc_bin_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_chunk_map_t *mapelm) { size_t pageind; arena_run_t *run; arena_bin_t *bin; -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size_t size; -#endif + arena_bin_info_t *bin_info; + size_t size, binind; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; + pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); + arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); bin = run->bin; - size_t binind = arena_bin_index(arena, bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size = bin_info->reg_size; -#endif + binind = arena_ptr_small_binind_get(ptr, mapelm->bits); + bin_info = &arena_bin_info[binind]; + if (config_fill || config_stats) + size = bin_info->reg_size; -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, size); -#endif + if (config_fill && opt_junk) + arena_dalloc_junk_small(ptr, bin_info); arena_run_reg_dalloc(run, ptr); if (run->nfree == bin_info->nregs) { @@ -1863,13 +1594,41 @@ arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, } else if (run->nfree == 1 && run != bin->runcur) arena_bin_lower_run(arena, chunk, run, bin); -#ifdef JEMALLOC_STATS - bin->stats.allocated -= size; - bin->stats.ndalloc++; -#endif + if (config_stats) { + bin->stats.allocated -= size; + bin->stats.ndalloc++; + } } -#ifdef JEMALLOC_STATS +void +arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind, arena_chunk_map_t *mapelm) +{ + arena_run_t *run; + arena_bin_t *bin; + + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - + arena_mapbits_small_runind_get(chunk, pageind)) << LG_PAGE)); + bin = run->bin; + malloc_mutex_lock(&bin->lock); + arena_dalloc_bin_locked(arena, chunk, ptr, mapelm); + malloc_mutex_unlock(&bin->lock); +} + +void +arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, + size_t pageind) +{ + arena_chunk_map_t *mapelm; + + if (config_debug) { + /* arena_ptr_small_binind_get() does extra sanity checking. */ + assert(arena_ptr_small_binind_get(ptr, arena_mapbits_get(chunk, + pageind)) != BININD_INVALID); + } + mapelm = arena_mapp_get(chunk, pageind); + arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); +} void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, @@ -1894,12 +1653,11 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; lstats[i].nrequests += arena->stats.lstats[i].nrequests; - lstats[i].highruns += arena->stats.lstats[i].highruns; lstats[i].curruns += arena->stats.lstats[i].curruns; } malloc_mutex_unlock(&arena->lock); - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(&bin->lock); @@ -1907,53 +1665,47 @@ arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, bstats[i].nmalloc += bin->stats.nmalloc; bstats[i].ndalloc += bin->stats.ndalloc; bstats[i].nrequests += bin->stats.nrequests; -#ifdef JEMALLOC_TCACHE - bstats[i].nfills += bin->stats.nfills; - bstats[i].nflushes += bin->stats.nflushes; -#endif + if (config_tcache) { + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; + } bstats[i].nruns += bin->stats.nruns; bstats[i].reruns += bin->stats.reruns; - bstats[i].highruns += bin->stats.highruns; bstats[i].curruns += bin->stats.curruns; malloc_mutex_unlock(&bin->lock); } } -#endif void -arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) +arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) { - /* Large allocation. */ -#ifdef JEMALLOC_FILL -# ifndef JEMALLOC_STATS - if (opt_junk) -# endif -#endif - { -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> - PAGE_SHIFT; - size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; -#endif + if (config_fill || config_stats) { + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t size = arena_mapbits_large_size_get(chunk, pageind); -#ifdef JEMALLOC_FILL -# ifdef JEMALLOC_STATS - if (opt_junk) -# endif + if (config_fill && config_stats && opt_junk) memset(ptr, 0x5a, size); -#endif -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; -#endif + if (config_stats) { + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= size; + arena->stats.lstats[(size >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns--; + } } arena_run_dalloc(arena, (arena_run_t *)ptr, true); } +void +arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) +{ + + malloc_mutex_lock(&arena->lock); + arena_dalloc_large_locked(arena, chunk, ptr); + malloc_mutex_unlock(&arena->lock); +} + static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size) @@ -1968,24 +1720,19 @@ arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, malloc_mutex_lock(&arena->lock); arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, true); -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; - - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; + if (config_stats) { + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif malloc_mutex_unlock(&arena->lock); } @@ -1993,20 +1740,19 @@ static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t npages = oldsize >> PAGE_SHIFT; + size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; + size_t npages = oldsize >> LG_PAGE; size_t followsize; - assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); + assert(oldsize == arena_mapbits_large_size_get(chunk, pageind)); /* Try to extend the run. */ assert(size + extra > oldsize); malloc_mutex_lock(&arena->lock); if (pageind + npages < chunk_npages && - (chunk->map[pageind+npages-map_bias].bits - & CHUNK_MAP_ALLOCATED) == 0 && (followsize = - chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - - oldsize) { + arena_mapbits_allocated_get(chunk, pageind+npages) == 0 && + (followsize = arena_mapbits_unallocated_size_get(chunk, + pageind+npages)) >= size - oldsize) { /* * The next run is available and sufficiently large. Split the * following run, then merge the first part with the existing @@ -2016,10 +1762,11 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t splitsize = (oldsize + followsize <= size + extra) ? followsize : size + extra - oldsize; arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + - ((pageind+npages) << PAGE_SHIFT)), splitsize, true, zero); + ((pageind+npages) << LG_PAGE)), splitsize, true, + BININD_INVALID, zero); size = oldsize + splitsize; - npages = size >> PAGE_SHIFT; + npages = size >> LG_PAGE; /* * Mark the extended run as dirty if either portion of the run @@ -2029,34 +1776,24 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, * arena_run_dalloc() with the dirty argument set to false * (which is when dirty flag consistency would really matter). */ - flag_dirty = (chunk->map[pageind-map_bias].bits & - CHUNK_MAP_DIRTY) | - (chunk->map[pageind+npages-1-map_bias].bits & - CHUNK_MAP_DIRTY); - chunk->map[pageind-map_bias].bits = size | flag_dirty - | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; - - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - - 1].curruns; + flag_dirty = arena_mapbits_dirty_get(chunk, pageind) | + arena_mapbits_dirty_get(chunk, pageind+npages-1); + arena_mapbits_large_set(chunk, pageind, size, flag_dirty); + arena_mapbits_large_set(chunk, pageind+npages-1, 0, flag_dirty); + + if (config_stats) { + arena->stats.ndalloc_large++; + arena->stats.allocated_large -= oldsize; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(oldsize >> LG_PAGE) - 1].curruns--; + + arena->stats.nmalloc_large++; + arena->stats.nrequests_large++; + arena->stats.allocated_large += size; + arena->stats.lstats[(size >> LG_PAGE) - 1].nmalloc++; + arena->stats.lstats[(size >> LG_PAGE) - 1].nrequests++; + arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } -#endif malloc_mutex_unlock(&arena->lock); return (false); } @@ -2078,12 +1815,10 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, psize = PAGE_CEILING(size + extra); if (psize == oldsize) { /* Same size class. */ -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { + if (config_fill && opt_junk && size < oldsize) { memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); } -#endif return (false); } else { arena_chunk_t *chunk; @@ -2091,16 +1826,13 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); arena = chunk->arena; - dassert(arena->magic == ARENA_MAGIC); if (psize < oldsize) { -#ifdef JEMALLOC_FILL /* Fill before shrinking in order avoid a race. */ - if (opt_junk) { + if (config_fill && opt_junk) { memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); } -#endif arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, psize); return (false); @@ -2108,12 +1840,11 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, PAGE_CEILING(size), psize - PAGE_CEILING(size), zero); -#ifdef JEMALLOC_FILL - if (ret == false && zero == false && opt_zero) { + if (config_fill && ret == false && zero == false && + opt_zero) { memset((void *)((uintptr_t)ptr + oldsize), 0, size - oldsize); } -#endif return (ret); } } @@ -2128,24 +1859,22 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, * Avoid moving the allocation if the size class can be left the same. */ if (oldsize <= arena_maxclass) { - if (oldsize <= small_maxclass) { + if (oldsize <= SMALL_MAXCLASS) { assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size == oldsize); - if ((size + extra <= small_maxclass && + if ((size + extra <= SMALL_MAXCLASS && SMALL_SIZE2BIN(size + extra) == SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && size + extra >= oldsize)) { -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { + if (config_fill && opt_junk && size < oldsize) { memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); } -#endif return (ptr); } } else { assert(size <= arena_maxclass); - if (size + extra > small_maxclass) { + if (size + extra > SMALL_MAXCLASS) { if (arena_ralloc_large(ptr, oldsize, size, extra, zero) == false) return (ptr); @@ -2159,7 +1888,7 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, void * arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) + size_t alignment, bool zero, bool try_tcache) { void *ret; size_t copysize; @@ -2175,24 +1904,24 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * copying. */ if (alignment != 0) { - size_t usize = sa2u(size + extra, alignment, NULL); + size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); ret = ipalloc(usize, alignment, zero); } else - ret = arena_malloc(size + extra, zero); + ret = arena_malloc(NULL, size + extra, zero, try_tcache); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment != 0) { - size_t usize = sa2u(size, alignment, NULL); + size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); ret = ipalloc(usize, alignment, zero); } else - ret = arena_malloc(size, zero); + ret = arena_malloc(NULL, size, zero, try_tcache); if (ret == NULL) return (NULL); @@ -2205,8 +1934,9 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * expectation that the extra bytes will be reliably preserved. */ copysize = (size < oldsize) ? size : oldsize; + VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloc(ptr); return (ret); } @@ -2222,22 +1952,21 @@ arena_new(arena_t *arena, unsigned ind) if (malloc_mutex_init(&arena->lock)) return (true); -#ifdef JEMALLOC_STATS - memset(&arena->stats, 0, sizeof(arena_stats_t)); - arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses * - sizeof(malloc_large_stats_t)); - if (arena->stats.lstats == NULL) - return (true); - memset(arena->stats.lstats, 0, nlclasses * - sizeof(malloc_large_stats_t)); -# ifdef JEMALLOC_TCACHE - ql_new(&arena->tcache_ql); -# endif -#endif + if (config_stats) { + memset(&arena->stats, 0, sizeof(arena_stats_t)); + arena->stats.lstats = + (malloc_large_stats_t *)base_alloc(nlclasses * + sizeof(malloc_large_stats_t)); + if (arena->stats.lstats == NULL) + return (true); + memset(arena->stats.lstats, 0, nlclasses * + sizeof(malloc_large_stats_t)); + if (config_tcache) + ql_new(&arena->tcache_ql); + } -#ifdef JEMALLOC_PROF - arena->prof_accumbytes = 0; -#endif + if (config_prof) + arena->prof_accumbytes = 0; /* Initialize chunks. */ ql_new(&arena->chunks_dirty); @@ -2251,183 +1980,17 @@ arena_new(arena_t *arena, unsigned ind) arena_avail_tree_new(&arena->runs_avail_dirty); /* Initialize bins. */ - i = 0; -#ifdef JEMALLOC_TINY - /* (2^n)-spaced tiny bins. */ - for (; i < ntbins; i++) { + for (i = 0; i < NBINS; i++) { bin = &arena->bins[i]; if (malloc_mutex_init(&bin->lock)) return (true); bin->runcur = NULL; arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif + if (config_stats) + memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); } -#endif - /* Quantum-spaced bins. */ - for (; i < ntbins + nqbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - - /* Cacheline-spaced bins. */ - for (; i < ntbins + nqbins + ncbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - - /* Subpage-spaced bins. */ - for (; i < nbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - -#ifdef JEMALLOC_DEBUG - arena->magic = ARENA_MAGIC; -#endif - - return (false); -} - -#ifdef JEMALLOC_DEBUG -static void -small_size2bin_validate(void) -{ - size_t i, size, binind; - - i = 1; -# ifdef JEMALLOC_TINY - /* Tiny. */ - for (; i < (1U << LG_TINY_MIN); i++) { - size = pow2_ceil(1U << LG_TINY_MIN); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - assert(SMALL_SIZE2BIN(i) == binind); - } - for (; i < qspace_min; i++) { - size = pow2_ceil(i); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - assert(SMALL_SIZE2BIN(i) == binind); - } -# endif - /* Quantum-spaced. */ - for (; i <= qspace_max; i++) { - size = QUANTUM_CEILING(i); - binind = ntbins + (size >> LG_QUANTUM) - 1; - assert(SMALL_SIZE2BIN(i) == binind); - } - /* Cacheline-spaced. */ - for (; i <= cspace_max; i++) { - size = CACHELINE_CEILING(i); - binind = ntbins + nqbins + ((size - cspace_min) >> - LG_CACHELINE); - assert(SMALL_SIZE2BIN(i) == binind); - } - /* Sub-page. */ - for (; i <= sspace_max; i++) { - size = SUBPAGE_CEILING(i); - binind = ntbins + nqbins + ncbins + ((size - sspace_min) - >> LG_SUBPAGE); - assert(SMALL_SIZE2BIN(i) == binind); - } -} -#endif - -static bool -small_size2bin_init(void) -{ - - if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT - || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT - || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> - LG_TINY_MIN) + 1)) - return (small_size2bin_init_hard()); - - small_size2bin = const_small_size2bin; -#ifdef JEMALLOC_DEBUG - small_size2bin_validate(); -#endif - return (false); -} - -static bool -small_size2bin_init_hard(void) -{ - size_t i, size, binind; - uint8_t *custom_small_size2bin; -#define CUSTOM_SMALL_SIZE2BIN(s) \ - custom_small_size2bin[(s-1) >> LG_TINY_MIN] - - assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT - || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT - || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> - LG_TINY_MIN) + 1)); - - custom_small_size2bin = (uint8_t *) - base_alloc(small_maxclass >> LG_TINY_MIN); - if (custom_small_size2bin == NULL) - return (true); - - i = 1; -#ifdef JEMALLOC_TINY - /* Tiny. */ - for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) { - size = pow2_ceil(1U << LG_TINY_MIN); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - for (; i < qspace_min; i += TINY_MIN) { - size = pow2_ceil(i); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } -#endif - /* Quantum-spaced. */ - for (; i <= qspace_max; i += TINY_MIN) { - size = QUANTUM_CEILING(i); - binind = ntbins + (size >> LG_QUANTUM) - 1; - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - /* Cacheline-spaced. */ - for (; i <= cspace_max; i += TINY_MIN) { - size = CACHELINE_CEILING(i); - binind = ntbins + nqbins + ((size - cspace_min) >> - LG_CACHELINE); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - /* Sub-page. */ - for (; i <= sspace_max; i += TINY_MIN) { - size = SUBPAGE_CEILING(i); - binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> - LG_SUBPAGE); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - - small_size2bin = custom_small_size2bin; -#ifdef JEMALLOC_DEBUG - small_size2bin_validate(); -#endif return (false); -#undef CUSTOM_SMALL_SIZE2BIN } /* @@ -2444,18 +2007,40 @@ small_size2bin_init_hard(void) static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) { + size_t pad_size; size_t try_run_size, good_run_size; uint32_t try_nregs, good_nregs; uint32_t try_hdr_size, good_hdr_size; uint32_t try_bitmap_offset, good_bitmap_offset; -#ifdef JEMALLOC_PROF uint32_t try_ctx0_offset, good_ctx0_offset; -#endif - uint32_t try_reg0_offset, good_reg0_offset; + uint32_t try_redzone0_offset, good_redzone0_offset; - assert(min_run_size >= PAGE_SIZE); + assert(min_run_size >= PAGE); assert(min_run_size <= arena_maxclass); + /* + * Determine redzone size based on minimum alignment and minimum + * redzone size. Add padding to the end of the run if it is needed to + * align the regions. The padding allows each redzone to be half the + * minimum alignment; without the padding, each redzone would have to + * be twice as large in order to maintain alignment. + */ + if (config_fill && opt_redzone) { + size_t align_min = ZU(1) << (ffs(bin_info->reg_size) - 1); + if (align_min <= REDZONE_MINSIZE) { + bin_info->redzone_size = REDZONE_MINSIZE; + pad_size = 0; + } else { + bin_info->redzone_size = align_min >> 1; + pad_size = bin_info->redzone_size; + } + } else { + bin_info->redzone_size = 0; + pad_size = 0; + } + bin_info->reg_interval = bin_info->reg_size + + (bin_info->redzone_size << 1); + /* * Calculate known-valid settings before entering the run_size * expansion loop, so that the first part of the loop always copies @@ -2467,7 +2052,8 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) * header's mask length and the number of regions. */ try_run_size = min_run_size; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) + try_nregs = ((try_run_size - sizeof(arena_run_t)) / + bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS @@ -2481,8 +2067,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote == false) { + if (config_prof && opt_prof && prof_promote == false) { /* Pad to a quantum boundary. */ try_hdr_size = QUANTUM_CEILING(try_hdr_size); try_ctx0_offset = try_hdr_size; @@ -2490,10 +2075,9 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_hdr_size += try_nregs * sizeof(prof_ctx_t *); } else try_ctx0_offset = 0; -#endif - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); + try_redzone0_offset = try_run_size - (try_nregs * + bin_info->reg_interval) - pad_size; + } while (try_hdr_size > try_redzone0_offset); /* run_size expansion loop. */ do { @@ -2504,15 +2088,13 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) good_nregs = try_nregs; good_hdr_size = try_hdr_size; good_bitmap_offset = try_bitmap_offset; -#ifdef JEMALLOC_PROF good_ctx0_offset = try_ctx0_offset; -#endif - good_reg0_offset = try_reg0_offset; + good_redzone0_offset = try_redzone0_offset; /* Try more aggressive settings. */ - try_run_size += PAGE_SIZE; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin_info->reg_size) + try_run_size += PAGE; + try_nregs = ((try_run_size - sizeof(arena_run_t) - pad_size) / + bin_info->reg_interval) + 1; /* Counter-act try_nregs-- in loop. */ if (try_nregs > RUN_MAXREGS) { try_nregs = RUN_MAXREGS @@ -2526,8 +2108,7 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_bitmap_offset = try_hdr_size; /* Add space for bitmap. */ try_hdr_size += bitmap_size(try_nregs); -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote == false) { + if (config_prof && opt_prof && prof_promote == false) { /* Pad to a quantum boundary. */ try_hdr_size = QUANTUM_CEILING(try_hdr_size); try_ctx0_offset = try_hdr_size; @@ -2537,140 +2118,52 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) try_hdr_size += try_nregs * sizeof(prof_ctx_t *); } -#endif - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); + try_redzone0_offset = try_run_size - (try_nregs * + bin_info->reg_interval) - pad_size; + } while (try_hdr_size > try_redzone0_offset); } while (try_run_size <= arena_maxclass && try_run_size <= arena_maxclass - && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX - && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size + && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > + RUN_MAX_OVRHD_RELAX + && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size && try_nregs < RUN_MAXREGS); - assert(good_hdr_size <= good_reg0_offset); + assert(good_hdr_size <= good_redzone0_offset); /* Copy final settings. */ bin_info->run_size = good_run_size; bin_info->nregs = good_nregs; bin_info->bitmap_offset = good_bitmap_offset; -#ifdef JEMALLOC_PROF bin_info->ctx0_offset = good_ctx0_offset; -#endif - bin_info->reg0_offset = good_reg0_offset; + bin_info->reg0_offset = good_redzone0_offset + bin_info->redzone_size; + + assert(bin_info->reg0_offset - bin_info->redzone_size + (bin_info->nregs + * bin_info->reg_interval) + pad_size == bin_info->run_size); return (good_run_size); } -static bool +static void bin_info_init(void) { arena_bin_info_t *bin_info; - unsigned i; - size_t prev_run_size; - - arena_bin_info = base_alloc(sizeof(arena_bin_info_t) * nbins); - if (arena_bin_info == NULL) - return (true); - - prev_run_size = PAGE_SIZE; - i = 0; -#ifdef JEMALLOC_TINY - /* (2^n)-spaced tiny bins. */ - for (; i < ntbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = (1U << (LG_TINY_MIN + i)); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } -#endif - - /* Quantum-spaced bins. */ - for (; i < ntbins + nqbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = (i - ntbins + 1) << LG_QUANTUM; - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - /* Cacheline-spaced bins. */ - for (; i < ntbins + nqbins + ncbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = cspace_min + ((i - (ntbins + nqbins)) << - LG_CACHELINE); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - /* Subpage-spaced bins. */ - for (; i < nbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = sspace_min + ((i - (ntbins + nqbins + - ncbins)) << LG_SUBPAGE); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - return (false); + size_t prev_run_size = PAGE; + +#define SIZE_CLASS(bin, delta, size) \ + bin_info = &arena_bin_info[bin]; \ + bin_info->reg_size = size; \ + prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size);\ + bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); + SIZE_CLASSES +#undef SIZE_CLASS } -bool +void arena_boot(void) { size_t header_size; unsigned i; - /* Set variables according to the value of opt_lg_[qc]space_max. */ - qspace_max = (1U << opt_lg_qspace_max); - cspace_min = CACHELINE_CEILING(qspace_max); - if (cspace_min == qspace_max) - cspace_min += CACHELINE; - cspace_max = (1U << opt_lg_cspace_max); - sspace_min = SUBPAGE_CEILING(cspace_max); - if (sspace_min == cspace_max) - sspace_min += SUBPAGE; - assert(sspace_min < PAGE_SIZE); - sspace_max = PAGE_SIZE - SUBPAGE; - -#ifdef JEMALLOC_TINY - assert(LG_QUANTUM >= LG_TINY_MIN); -#endif - assert(ntbins <= LG_QUANTUM); - nqbins = qspace_max >> LG_QUANTUM; - ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; - nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; - nbins = ntbins + nqbins + ncbins + nsbins; - - /* - * The small_size2bin lookup table uses uint8_t to encode each bin - * index, so we cannot support more than 256 small size classes. This - * limit is difficult to exceed (not even possible with 16B quantum and - * 4KiB pages), and such configurations are impractical, but - * nonetheless we need to protect against this case in order to avoid - * undefined behavior. - * - * Further constrain nbins to 255 if prof_promote is true, since all - * small size classes, plus a "not small" size class must be stored in - * 8 bits of arena_chunk_map_t's bits field. - */ -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote) { - if (nbins > 255) { - char line_buf[UMAX2S_BUFSIZE]; - malloc_write(": Too many small size classes ("); - malloc_write(u2s(nbins, 10, line_buf)); - malloc_write(" > max 255)\n"); - abort(); - } - } else -#endif - if (nbins > 256) { - char line_buf[UMAX2S_BUFSIZE]; - malloc_write(": Too many small size classes ("); - malloc_write(u2s(nbins, 10, line_buf)); - malloc_write(" > max 256)\n"); - abort(); - } - /* * Compute the header size such that it is large enough to contain the * page map. The page map is biased to omit entries for the header @@ -2685,20 +2178,44 @@ arena_boot(void) */ map_bias = 0; for (i = 0; i < 3; i++) { - header_size = offsetof(arena_chunk_t, map) - + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); - map_bias = (header_size >> PAGE_SHIFT) + ((header_size & - PAGE_MASK) != 0); + header_size = offsetof(arena_chunk_t, map) + + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); + map_bias = (header_size >> LG_PAGE) + ((header_size & PAGE_MASK) + != 0); } assert(map_bias > 0); - arena_maxclass = chunksize - (map_bias << PAGE_SHIFT); + arena_maxclass = chunksize - (map_bias << LG_PAGE); - if (small_size2bin_init()) - return (true); + bin_info_init(); +} - if (bin_info_init()) - return (true); +void +arena_prefork(arena_t *arena) +{ + unsigned i; - return (false); + malloc_mutex_prefork(&arena->lock); + for (i = 0; i < NBINS; i++) + malloc_mutex_prefork(&arena->bins[i].lock); +} + +void +arena_postfork_parent(arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_parent(&arena->bins[i].lock); + malloc_mutex_postfork_parent(&arena->lock); +} + +void +arena_postfork_child(arena_t *arena) +{ + unsigned i; + + for (i = 0; i < NBINS; i++) + malloc_mutex_postfork_child(&arena->bins[i].lock); + malloc_mutex_postfork_child(&arena->lock); } diff --git a/deps/jemalloc/src/base.c b/deps/jemalloc/src/base.c index cc85e8494ec..bafaa743801 100644 --- a/deps/jemalloc/src/base.c +++ b/deps/jemalloc/src/base.c @@ -4,7 +4,7 @@ /******************************************************************************/ /* Data. */ -malloc_mutex_t base_mtx; +static malloc_mutex_t base_mtx; /* * Current pages that are being used for internal memory allocations. These @@ -32,7 +32,7 @@ base_pages_alloc(size_t minsize) assert(minsize != 0); csize = CHUNK_CEILING(minsize); zero = false; - base_pages = chunk_alloc(csize, true, &zero); + base_pages = chunk_alloc(csize, chunksize, true, &zero); if (base_pages == NULL) return (true); base_next_addr = base_pages; @@ -66,6 +66,17 @@ base_alloc(size_t size) return (ret); } +void * +base_calloc(size_t number, size_t size) +{ + void *ret = base_alloc(number * size); + + if (ret != NULL) + memset(ret, 0, number * size); + + return (ret); +} + extent_node_t * base_node_alloc(void) { @@ -104,3 +115,24 @@ base_boot(void) return (false); } + +void +base_prefork(void) +{ + + malloc_mutex_prefork(&base_mtx); +} + +void +base_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&base_mtx); +} + +void +base_postfork_child(void) +{ + + malloc_mutex_postfork_child(&base_mtx); +} diff --git a/deps/jemalloc/src/chunk.c b/deps/jemalloc/src/chunk.c index d190c6f49b3..6bc24544797 100644 --- a/deps/jemalloc/src/chunk.c +++ b/deps/jemalloc/src/chunk.c @@ -5,18 +5,20 @@ /* Data. */ size_t opt_lg_chunk = LG_CHUNK_DEFAULT; -#ifdef JEMALLOC_SWAP -bool opt_overcommit = true; -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) malloc_mutex_t chunks_mtx; chunk_stats_t stats_chunks; -#endif -#ifdef JEMALLOC_IVSALLOC +/* + * Trees of chunks that were previously allocated (trees differ only in node + * ordering). These are used when allocating chunks, in an attempt to re-use + * address space. Depending on function, different tree orderings are needed, + * which is why there are two trees with the same contents. + */ +static extent_tree_t chunks_szad; +static extent_tree_t chunks_ad; + rtree_t *chunks_rtree; -#endif /* Various chunk-related settings. */ size_t chunksize; @@ -26,6 +28,98 @@ size_t map_bias; size_t arena_maxclass; /* Max size class for arenas. */ /******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void *chunk_recycle(size_t size, size_t alignment, bool base, + bool *zero); +static void chunk_record(void *chunk, size_t size); + +/******************************************************************************/ + +static void * +chunk_recycle(size_t size, size_t alignment, bool base, bool *zero) +{ + void *ret; + extent_node_t *node; + extent_node_t key; + size_t alloc_size, leadsize, trailsize; + + if (base) { + /* + * This function may need to call base_node_{,de}alloc(), but + * the current chunk allocation request is on behalf of the + * base allocator. Avoid deadlock (and if that weren't an + * issue, potential for infinite recursion) by returning NULL. + */ + return (NULL); + } + + alloc_size = size + alignment - chunksize; + /* Beware size_t wrap-around. */ + if (alloc_size < size) + return (NULL); + key.addr = NULL; + key.size = alloc_size; + malloc_mutex_lock(&chunks_mtx); + node = extent_tree_szad_nsearch(&chunks_szad, &key); + if (node == NULL) { + malloc_mutex_unlock(&chunks_mtx); + return (NULL); + } + leadsize = ALIGNMENT_CEILING((uintptr_t)node->addr, alignment) - + (uintptr_t)node->addr; + assert(node->size >= leadsize + size); + trailsize = node->size - leadsize - size; + ret = (void *)((uintptr_t)node->addr + leadsize); + /* Remove node from the tree. */ + extent_tree_szad_remove(&chunks_szad, node); + extent_tree_ad_remove(&chunks_ad, node); + if (leadsize != 0) { + /* Insert the leading space as a smaller chunk. */ + node->size = leadsize; + extent_tree_szad_insert(&chunks_szad, node); + extent_tree_ad_insert(&chunks_ad, node); + node = NULL; + } + if (trailsize != 0) { + /* Insert the trailing space as a smaller chunk. */ + if (node == NULL) { + /* + * An additional node is required, but + * base_node_alloc() can cause a new base chunk to be + * allocated. Drop chunks_mtx in order to avoid + * deadlock, and if node allocation fails, deallocate + * the result before returning an error. + */ + malloc_mutex_unlock(&chunks_mtx); + node = base_node_alloc(); + if (node == NULL) { + chunk_dealloc(ret, size, true); + return (NULL); + } + malloc_mutex_lock(&chunks_mtx); + } + node->addr = (void *)((uintptr_t)(ret) + size); + node->size = trailsize; + extent_tree_szad_insert(&chunks_szad, node); + extent_tree_ad_insert(&chunks_ad, node); + node = NULL; + } + malloc_mutex_unlock(&chunks_mtx); + + if (node != NULL) + base_node_dealloc(node); +#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED + /* Pages are zeroed as a side effect of pages_purge(). */ + *zero = true; +#else + if (*zero) { + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + memset(ret, 0, size); + } +#endif + return (ret); +} /* * If the caller specifies (*zero == false), it is still possible to receive @@ -34,79 +128,138 @@ size_t arena_maxclass; /* Max size class for arenas. */ * advantage of them if they are returned. */ void * -chunk_alloc(size_t size, bool base, bool *zero) +chunk_alloc(size_t size, size_t alignment, bool base, bool *zero) { void *ret; assert(size != 0); assert((size & chunksize_mask) == 0); + assert(alignment != 0); + assert((alignment & chunksize_mask) == 0); -#ifdef JEMALLOC_SWAP - if (swap_enabled) { - ret = chunk_alloc_swap(size, zero); - if (ret != NULL) - goto RETURN; - } + ret = chunk_recycle(size, alignment, base, zero); + if (ret != NULL) + goto label_return; - if (swap_enabled == false || opt_overcommit) { -#endif -#ifdef JEMALLOC_DSS - ret = chunk_alloc_dss(size, zero); + ret = chunk_alloc_mmap(size, alignment, zero); + if (ret != NULL) + goto label_return; + + if (config_dss) { + ret = chunk_alloc_dss(size, alignment, zero); if (ret != NULL) - goto RETURN; -#endif - ret = chunk_alloc_mmap(size); - if (ret != NULL) { - *zero = true; - goto RETURN; - } -#ifdef JEMALLOC_SWAP + goto label_return; } -#endif /* All strategies for allocation failed. */ ret = NULL; -RETURN: -#ifdef JEMALLOC_IVSALLOC - if (base == false && ret != NULL) { +label_return: + if (config_ivsalloc && base == false && ret != NULL) { if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) { chunk_dealloc(ret, size, true); return (NULL); } } -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - if (ret != NULL) { -# ifdef JEMALLOC_PROF + if ((config_stats || config_prof) && ret != NULL) { bool gdump; -# endif malloc_mutex_lock(&chunks_mtx); -# ifdef JEMALLOC_STATS - stats_chunks.nchunks += (size / chunksize); -# endif + if (config_stats) + stats_chunks.nchunks += (size / chunksize); stats_chunks.curchunks += (size / chunksize); if (stats_chunks.curchunks > stats_chunks.highchunks) { stats_chunks.highchunks = stats_chunks.curchunks; -# ifdef JEMALLOC_PROF - gdump = true; -# endif - } -# ifdef JEMALLOC_PROF - else + if (config_prof) + gdump = true; + } else if (config_prof) gdump = false; -# endif malloc_mutex_unlock(&chunks_mtx); -# ifdef JEMALLOC_PROF - if (opt_prof && opt_prof_gdump && gdump) + if (config_prof && opt_prof && opt_prof_gdump && gdump) prof_gdump(); -# endif } -#endif + if (config_debug && *zero && ret != NULL) { + size_t i; + size_t *p = (size_t *)(uintptr_t)ret; + VALGRIND_MAKE_MEM_DEFINED(ret, size); + for (i = 0; i < size / sizeof(size_t); i++) + assert(p[i] == 0); + } assert(CHUNK_ADDR2BASE(ret) == ret); return (ret); } +static void +chunk_record(void *chunk, size_t size) +{ + extent_node_t *xnode, *node, *prev, key; + + pages_purge(chunk, size); + + /* + * Allocate a node before acquiring chunks_mtx even though it might not + * be needed, because base_node_alloc() may cause a new base chunk to + * be allocated, which could cause deadlock if chunks_mtx were already + * held. + */ + xnode = base_node_alloc(); + + malloc_mutex_lock(&chunks_mtx); + key.addr = (void *)((uintptr_t)chunk + size); + node = extent_tree_ad_nsearch(&chunks_ad, &key); + /* Try to coalesce forward. */ + if (node != NULL && node->addr == key.addr) { + /* + * Coalesce chunk with the following address range. This does + * not change the position within chunks_ad, so only + * remove/insert from/into chunks_szad. + */ + extent_tree_szad_remove(&chunks_szad, node); + node->addr = chunk; + node->size += size; + extent_tree_szad_insert(&chunks_szad, node); + if (xnode != NULL) + base_node_dealloc(xnode); + } else { + /* Coalescing forward failed, so insert a new node. */ + if (xnode == NULL) { + /* + * base_node_alloc() failed, which is an exceedingly + * unlikely failure. Leak chunk; its pages have + * already been purged, so this is only a virtual + * memory leak. + */ + malloc_mutex_unlock(&chunks_mtx); + return; + } + node = xnode; + node->addr = chunk; + node->size = size; + extent_tree_ad_insert(&chunks_ad, node); + extent_tree_szad_insert(&chunks_szad, node); + } + + /* Try to coalesce backward. */ + prev = extent_tree_ad_prev(&chunks_ad, node); + if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == + chunk) { + /* + * Coalesce chunk with the previous address range. This does + * not change the position within chunks_ad, so only + * remove/insert node from/into chunks_szad. + */ + extent_tree_szad_remove(&chunks_szad, prev); + extent_tree_ad_remove(&chunks_ad, prev); + + extent_tree_szad_remove(&chunks_szad, node); + node->addr = prev->addr; + node->size += prev->size; + extent_tree_szad_insert(&chunks_szad, node); + + base_node_dealloc(prev); + } + malloc_mutex_unlock(&chunks_mtx); +} + void chunk_dealloc(void *chunk, size_t size, bool unmap) { @@ -116,25 +269,18 @@ chunk_dealloc(void *chunk, size_t size, bool unmap) assert(size != 0); assert((size & chunksize_mask) == 0); -#ifdef JEMALLOC_IVSALLOC - rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - malloc_mutex_lock(&chunks_mtx); - stats_chunks.curchunks -= (size / chunksize); - malloc_mutex_unlock(&chunks_mtx); -#endif + if (config_ivsalloc) + rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); + if (config_stats || config_prof) { + malloc_mutex_lock(&chunks_mtx); + stats_chunks.curchunks -= (size / chunksize); + malloc_mutex_unlock(&chunks_mtx); + } if (unmap) { -#ifdef JEMALLOC_SWAP - if (swap_enabled && chunk_dealloc_swap(chunk, size) == false) - return; -#endif -#ifdef JEMALLOC_DSS - if (chunk_dealloc_dss(chunk, size) == false) - return; -#endif - chunk_dealloc_mmap(chunk, size); + if ((config_dss && chunk_in_dss(chunk)) || + chunk_dealloc_mmap(chunk, size)) + chunk_record(chunk, size); } } @@ -144,30 +290,25 @@ chunk_boot(void) /* Set variables according to the value of opt_lg_chunk. */ chunksize = (ZU(1) << opt_lg_chunk); - assert(chunksize >= PAGE_SIZE); + assert(chunksize >= PAGE); chunksize_mask = chunksize - 1; - chunk_npages = (chunksize >> PAGE_SHIFT); + chunk_npages = (chunksize >> LG_PAGE); -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - if (malloc_mutex_init(&chunks_mtx)) - return (true); - memset(&stats_chunks, 0, sizeof(chunk_stats_t)); -#endif -#ifdef JEMALLOC_SWAP - if (chunk_swap_boot()) - return (true); -#endif - if (chunk_mmap_boot()) - return (true); -#ifdef JEMALLOC_DSS - if (chunk_dss_boot()) - return (true); -#endif -#ifdef JEMALLOC_IVSALLOC - chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk); - if (chunks_rtree == NULL) + if (config_stats || config_prof) { + if (malloc_mutex_init(&chunks_mtx)) + return (true); + memset(&stats_chunks, 0, sizeof(chunk_stats_t)); + } + if (config_dss && chunk_dss_boot()) return (true); -#endif + extent_tree_szad_new(&chunks_szad); + extent_tree_ad_new(&chunks_ad); + if (config_ivsalloc) { + chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - + opt_lg_chunk); + if (chunks_rtree == NULL) + return (true); + } return (false); } diff --git a/deps/jemalloc/src/chunk_dss.c b/deps/jemalloc/src/chunk_dss.c index 5c0e290e441..2d68e4804d7 100644 --- a/deps/jemalloc/src/chunk_dss.c +++ b/deps/jemalloc/src/chunk_dss.c @@ -1,82 +1,42 @@ #define JEMALLOC_CHUNK_DSS_C_ #include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_DSS /******************************************************************************/ /* Data. */ -malloc_mutex_t dss_mtx; +/* + * Protects sbrk() calls. This avoids malloc races among threads, though it + * does not protect against races with threads that call sbrk() directly. + */ +static malloc_mutex_t dss_mtx; /* Base address of the DSS. */ -static void *dss_base; +static void *dss_base; /* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ -static void *dss_prev; +static void *dss_prev; /* Current upper limit on DSS addresses. */ -static void *dss_max; - -/* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. - */ -static extent_tree_t dss_chunks_szad; -static extent_tree_t dss_chunks_ad; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void *chunk_recycle_dss(size_t size, bool *zero); -static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size); +static void *dss_max; /******************************************************************************/ +#ifndef JEMALLOC_HAVE_SBRK static void * -chunk_recycle_dss(size_t size, bool *zero) +sbrk(intptr_t increment) { - extent_node_t *node, key; - - key.addr = NULL; - key.size = size; - malloc_mutex_lock(&dss_mtx); - node = extent_tree_szad_nsearch(&dss_chunks_szad, &key); - if (node != NULL) { - void *ret = node->addr; - /* Remove node from the tree. */ - extent_tree_szad_remove(&dss_chunks_szad, node); - if (node->size == size) { - extent_tree_ad_remove(&dss_chunks_ad, node); - base_node_dealloc(node); - } else { - /* - * Insert the remainder of node's address range as a - * smaller chunk. Its position within dss_chunks_ad - * does not change. - */ - assert(node->size > size); - node->addr = (void *)((uintptr_t)node->addr + size); - node->size -= size; - extent_tree_szad_insert(&dss_chunks_szad, node); - } - malloc_mutex_unlock(&dss_mtx); - - if (*zero) - memset(ret, 0, size); - return (ret); - } - malloc_mutex_unlock(&dss_mtx); + not_implemented(); return (NULL); } +#endif void * -chunk_alloc_dss(size_t size, bool *zero) +chunk_alloc_dss(size_t size, size_t alignment, bool *zero) { void *ret; - ret = chunk_recycle_dss(size, zero); - if (ret != NULL) - return (ret); + cassert(config_dss); + assert(size > 0 && (size & chunksize_mask) == 0); + assert(alignment > 0 && (alignment & chunksize_mask) == 0); /* * sbrk() uses a signed increment argument, so take care not to @@ -87,6 +47,8 @@ chunk_alloc_dss(size_t size, bool *zero) malloc_mutex_lock(&dss_mtx); if (dss_prev != (void *)-1) { + size_t gap_size, cpad_size; + void *cpad, *dss_next; intptr_t incr; /* @@ -97,26 +59,40 @@ chunk_alloc_dss(size_t size, bool *zero) do { /* Get the current end of the DSS. */ dss_max = sbrk(0); - /* * Calculate how much padding is necessary to * chunk-align the end of the DSS. */ - incr = (intptr_t)size - - (intptr_t)CHUNK_ADDR2OFFSET(dss_max); - if (incr == (intptr_t)size) - ret = dss_max; - else { - ret = (void *)((intptr_t)dss_max + incr); - incr += size; + gap_size = (chunksize - CHUNK_ADDR2OFFSET(dss_max)) & + chunksize_mask; + /* + * Compute how much chunk-aligned pad space (if any) is + * necessary to satisfy alignment. This space can be + * recycled for later use. + */ + cpad = (void *)((uintptr_t)dss_max + gap_size); + ret = (void *)ALIGNMENT_CEILING((uintptr_t)dss_max, + alignment); + cpad_size = (uintptr_t)ret - (uintptr_t)cpad; + dss_next = (void *)((uintptr_t)ret + size); + if ((uintptr_t)ret < (uintptr_t)dss_max || + (uintptr_t)dss_next < (uintptr_t)dss_max) { + /* Wrap-around. */ + malloc_mutex_unlock(&dss_mtx); + return (NULL); } - + incr = gap_size + cpad_size + size; dss_prev = sbrk(incr); if (dss_prev == dss_max) { /* Success. */ - dss_max = (void *)((intptr_t)dss_prev + incr); + dss_max = dss_next; malloc_mutex_unlock(&dss_mtx); - *zero = true; + if (cpad_size != 0) + chunk_dealloc(cpad, cpad_size, true); + if (*zero) { + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); + memset(ret, 0, size); + } return (ret); } } while (dss_prev != (void *)-1); @@ -126,84 +102,13 @@ chunk_alloc_dss(size_t size, bool *zero) return (NULL); } -static extent_node_t * -chunk_dealloc_dss_record(void *chunk, size_t size) -{ - extent_node_t *xnode, *node, *prev, key; - - xnode = NULL; - while (true) { - key.addr = (void *)((uintptr_t)chunk + size); - node = extent_tree_ad_nsearch(&dss_chunks_ad, &key); - /* Try to coalesce forward. */ - if (node != NULL && node->addr == key.addr) { - /* - * Coalesce chunk with the following address range. - * This does not change the position within - * dss_chunks_ad, so only remove/insert from/into - * dss_chunks_szad. - */ - extent_tree_szad_remove(&dss_chunks_szad, node); - node->addr = chunk; - node->size += size; - extent_tree_szad_insert(&dss_chunks_szad, node); - break; - } else if (xnode == NULL) { - /* - * It is possible that base_node_alloc() will cause a - * new base chunk to be allocated, so take care not to - * deadlock on dss_mtx, and recover if another thread - * deallocates an adjacent chunk while this one is busy - * allocating xnode. - */ - malloc_mutex_unlock(&dss_mtx); - xnode = base_node_alloc(); - malloc_mutex_lock(&dss_mtx); - if (xnode == NULL) - return (NULL); - } else { - /* Coalescing forward failed, so insert a new node. */ - node = xnode; - xnode = NULL; - node->addr = chunk; - node->size = size; - extent_tree_ad_insert(&dss_chunks_ad, node); - extent_tree_szad_insert(&dss_chunks_szad, node); - break; - } - } - /* Discard xnode if it ended up unused do to a race. */ - if (xnode != NULL) - base_node_dealloc(xnode); - - /* Try to coalesce backward. */ - prev = extent_tree_ad_prev(&dss_chunks_ad, node); - if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == - chunk) { - /* - * Coalesce chunk with the previous address range. This does - * not change the position within dss_chunks_ad, so only - * remove/insert node from/into dss_chunks_szad. - */ - extent_tree_szad_remove(&dss_chunks_szad, prev); - extent_tree_ad_remove(&dss_chunks_ad, prev); - - extent_tree_szad_remove(&dss_chunks_szad, node); - node->addr = prev->addr; - node->size += prev->size; - extent_tree_szad_insert(&dss_chunks_szad, node); - - base_node_dealloc(prev); - } - - return (node); -} - bool chunk_in_dss(void *chunk) { bool ret; + cassert(config_dss); + malloc_mutex_lock(&dss_mtx); if ((uintptr_t)chunk >= (uintptr_t)dss_base && (uintptr_t)chunk < (uintptr_t)dss_max) @@ -216,69 +121,42 @@ chunk_in_dss(void *chunk) } bool -chunk_dealloc_dss(void *chunk, size_t size) +chunk_dss_boot(void) { - bool ret; - malloc_mutex_lock(&dss_mtx); - if ((uintptr_t)chunk >= (uintptr_t)dss_base - && (uintptr_t)chunk < (uintptr_t)dss_max) { - extent_node_t *node; + cassert(config_dss); - /* Try to coalesce with other unused chunks. */ - node = chunk_dealloc_dss_record(chunk, size); - if (node != NULL) { - chunk = node->addr; - size = node->size; - } + if (malloc_mutex_init(&dss_mtx)) + return (true); + dss_base = sbrk(0); + dss_prev = dss_base; + dss_max = dss_base; - /* Get the current end of the DSS. */ - dss_max = sbrk(0); + return (false); +} - /* - * Try to shrink the DSS if this chunk is at the end of the - * DSS. The sbrk() call here is subject to a race condition - * with threads that use brk(2) or sbrk(2) directly, but the - * alternative would be to leak memory for the sake of poorly - * designed multi-threaded programs. - */ - if ((void *)((uintptr_t)chunk + size) == dss_max - && (dss_prev = sbrk(-(intptr_t)size)) == dss_max) { - /* Success. */ - dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size); +void +chunk_dss_prefork(void) +{ - if (node != NULL) { - extent_tree_szad_remove(&dss_chunks_szad, node); - extent_tree_ad_remove(&dss_chunks_ad, node); - base_node_dealloc(node); - } - } else - madvise(chunk, size, MADV_DONTNEED); + if (config_dss) + malloc_mutex_prefork(&dss_mtx); +} - ret = false; - goto RETURN; - } +void +chunk_dss_postfork_parent(void) +{ - ret = true; -RETURN: - malloc_mutex_unlock(&dss_mtx); - return (ret); + if (config_dss) + malloc_mutex_postfork_parent(&dss_mtx); } -bool -chunk_dss_boot(void) +void +chunk_dss_postfork_child(void) { - if (malloc_mutex_init(&dss_mtx)) - return (true); - dss_base = sbrk(0); - dss_prev = dss_base; - dss_max = dss_base; - extent_tree_szad_new(&dss_chunks_szad); - extent_tree_ad_new(&dss_chunks_ad); - - return (false); + if (config_dss) + malloc_mutex_postfork_child(&dss_mtx); } /******************************************************************************/ -#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc/src/chunk_mmap.c b/deps/jemalloc/src/chunk_mmap.c index 164e86e7b38..c8da6556b09 100644 --- a/deps/jemalloc/src/chunk_mmap.c +++ b/deps/jemalloc/src/chunk_mmap.c @@ -1,54 +1,37 @@ #define JEMALLOC_CHUNK_MMAP_C_ #include "jemalloc/internal/jemalloc_internal.h" -/******************************************************************************/ -/* Data. */ - -/* - * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and - * potentially avoid some system calls. - */ -#ifndef NO_TLS -static __thread bool mmap_unaligned_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -#define MMAP_UNALIGNED_GET() mmap_unaligned_tls -#define MMAP_UNALIGNED_SET(v) do { \ - mmap_unaligned_tls = (v); \ -} while (0) -#else -static pthread_key_t mmap_unaligned_tsd; -#define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd)) -#define MMAP_UNALIGNED_SET(v) do { \ - pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \ -} while (0) -#endif - /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static void *pages_map(void *addr, size_t size, bool noreserve); +static void *pages_map(void *addr, size_t size); static void pages_unmap(void *addr, size_t size); -static void *chunk_alloc_mmap_slow(size_t size, bool unaligned, - bool noreserve); -static void *chunk_alloc_mmap_internal(size_t size, bool noreserve); +static void *chunk_alloc_mmap_slow(size_t size, size_t alignment, + bool *zero); /******************************************************************************/ static void * -pages_map(void *addr, size_t size, bool noreserve) +pages_map(void *addr, size_t size) { void *ret; + assert(size != 0); + +#ifdef _WIN32 + /* + * If VirtualAlloc can't allocate at the given address when one is + * given, it fails and returns NULL. + */ + ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE, + PAGE_READWRITE); +#else /* * We don't use MAP_FIXED here, because it can cause the *replacement* * of existing mappings, and we only want to create new mappings. */ - int flags = MAP_PRIVATE | MAP_ANON; -#ifdef MAP_NORESERVE - if (noreserve) - flags |= MAP_NORESERVE; -#endif - ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); + ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, + -1, 0); assert(ret != NULL); if (ret == MAP_FAILED) @@ -60,16 +43,15 @@ pages_map(void *addr, size_t size, bool noreserve) if (munmap(ret, size) == -1) { char buf[BUFERROR_BUF]; - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); + buferror(buf, sizeof(buf)); + malloc_printf(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); + buferror(buf, sizeof(buf)); + malloc_printf(": Error in " +#ifdef _WIN32 + "VirtualFree" +#else + "munmap" +#endif + "(): %s\n", buf); if (opt_abort) abort(); } } static void * -chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve) +pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size) { - void *ret; - size_t offset; - - /* Beware size_t wrap-around. */ - if (size + chunksize <= size) + void *ret = (void *)((uintptr_t)addr + leadsize); + + assert(alloc_size >= leadsize + size); +#ifdef _WIN32 + { + void *new_addr; + + pages_unmap(addr, alloc_size); + new_addr = pages_map(ret, size); + if (new_addr == ret) + return (ret); + if (new_addr) + pages_unmap(new_addr, size); return (NULL); + } +#else + { + size_t trailsize = alloc_size - leadsize - size; + + if (leadsize != 0) + pages_unmap(addr, leadsize); + if (trailsize != 0) + pages_unmap((void *)((uintptr_t)ret + size), trailsize); + return (ret); + } +#endif +} - ret = pages_map(NULL, size + chunksize, noreserve); - if (ret == NULL) - return (NULL); +void +pages_purge(void *addr, size_t length) +{ - /* Clean up unneeded leading/trailing space. */ - offset = CHUNK_ADDR2OFFSET(ret); - if (offset != 0) { - /* Note that mmap() returned an unaligned mapping. */ - unaligned = true; - - /* Leading space. */ - pages_unmap(ret, chunksize - offset); - - ret = (void *)((uintptr_t)ret + - (chunksize - offset)); - - /* Trailing space. */ - pages_unmap((void *)((uintptr_t)ret + size), - offset); - } else { - /* Trailing space only. */ - pages_unmap((void *)((uintptr_t)ret + size), - chunksize); - } +#ifdef _WIN32 + VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE); +#else +# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED +# define JEMALLOC_MADV_PURGE MADV_DONTNEED +# elif defined(JEMALLOC_PURGE_MADVISE_FREE) +# define JEMALLOC_MADV_PURGE MADV_FREE +# else +# error "No method defined for purging unused dirty pages." +# endif + madvise(addr, length, JEMALLOC_MADV_PURGE); +#endif +} - /* - * If mmap() returned an aligned mapping, reset mmap_unaligned so that - * the next chunk_alloc_mmap() execution tries the fast allocation - * method. - */ - if (unaligned == false) - MMAP_UNALIGNED_SET(false); +static void * +chunk_alloc_mmap_slow(size_t size, size_t alignment, bool *zero) +{ + void *ret, *pages; + size_t alloc_size, leadsize; + alloc_size = size + alignment - PAGE; + /* Beware size_t wrap-around. */ + if (alloc_size < size) + return (NULL); + do { + pages = pages_map(NULL, alloc_size); + if (pages == NULL) + return (NULL); + leadsize = ALIGNMENT_CEILING((uintptr_t)pages, alignment) - + (uintptr_t)pages; + ret = pages_trim(pages, alloc_size, leadsize, size); + } while (ret == NULL); + + assert(ret != NULL); + *zero = true; return (ret); } -static void * -chunk_alloc_mmap_internal(size_t size, bool noreserve) +void * +chunk_alloc_mmap(size_t size, size_t alignment, bool *zero) { void *ret; + size_t offset; /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work * hard to efficiently create aligned mappings. The reliable, but * slow method is to create a mapping that is over-sized, then trim the - * excess. However, that always results in at least one call to + * excess. However, that always results in one or two calls to * pages_unmap(). * - * A more optimistic approach is to try mapping precisely the right - * amount, then try to append another mapping if alignment is off. In - * practice, this works out well as long as the application is not - * interleaving mappings via direct mmap() calls. If we do run into a - * situation where there is an interleaved mapping and we are unable to - * extend an unaligned mapping, our best option is to switch to the - * slow method until mmap() returns another aligned mapping. This will - * tend to leave a gap in the memory map that is too small to cause - * later problems for the optimistic method. - * - * Another possible confounding factor is address space layout - * randomization (ASLR), which causes mmap(2) to disregard the - * requested address. mmap_unaligned tracks whether the previous - * chunk_alloc_mmap() execution received any unaligned or relocated - * mappings, and if so, the current execution will immediately fall - * back to the slow method. However, we keep track of whether the fast - * method would have succeeded, and if so, we make a note to try the - * fast method next time. + * Optimistically try mapping precisely the right amount before falling + * back to the slow method, with the expectation that the optimistic + * approach works most of the time. */ - if (MMAP_UNALIGNED_GET() == false) { - size_t offset; + assert(alignment != 0); + assert((alignment & chunksize_mask) == 0); - ret = pages_map(NULL, size, noreserve); - if (ret == NULL) - return (NULL); - - offset = CHUNK_ADDR2OFFSET(ret); - if (offset != 0) { - MMAP_UNALIGNED_SET(true); - /* Try to extend chunk boundary. */ - if (pages_map((void *)((uintptr_t)ret + size), - chunksize - offset, noreserve) == NULL) { - /* - * Extension failed. Clean up, then revert to - * the reliable-but-expensive method. - */ - pages_unmap(ret, size); - ret = chunk_alloc_mmap_slow(size, true, - noreserve); - } else { - /* Clean up unneeded leading space. */ - pages_unmap(ret, chunksize - offset); - ret = (void *)((uintptr_t)ret + (chunksize - - offset)); - } - } - } else - ret = chunk_alloc_mmap_slow(size, false, noreserve); + ret = pages_map(NULL, size); + if (ret == NULL) + return (NULL); + offset = ALIGNMENT_ADDR2OFFSET(ret, alignment); + if (offset != 0) { + pages_unmap(ret, size); + return (chunk_alloc_mmap_slow(size, alignment, zero)); + } + assert(ret != NULL); + *zero = true; return (ret); } -void * -chunk_alloc_mmap(size_t size) -{ - - return (chunk_alloc_mmap_internal(size, false)); -} - -void * -chunk_alloc_mmap_noreserve(size_t size) -{ - - return (chunk_alloc_mmap_internal(size, true)); -} - -void -chunk_dealloc_mmap(void *chunk, size_t size) -{ - - pages_unmap(chunk, size); -} - bool -chunk_mmap_boot(void) +chunk_dealloc_mmap(void *chunk, size_t size) { -#ifdef NO_TLS - if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) { - malloc_write(": Error in pthread_key_create()\n"); - return (true); - } -#endif + if (config_munmap) + pages_unmap(chunk, size); - return (false); + return (config_munmap == false); } diff --git a/deps/jemalloc/src/ckh.c b/deps/jemalloc/src/ckh.c index 43fcc25239d..742a950bea2 100644 --- a/deps/jemalloc/src/ckh.c +++ b/deps/jemalloc/src/ckh.c @@ -73,7 +73,6 @@ ckh_isearch(ckh_t *ckh, const void *key) size_t hash1, hash2, bucket, cell; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); @@ -100,7 +99,7 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * Cycle through the cells in the bucket, starting at a random position. * The randomness avoids worst-case search overhead as buckets fill up. */ - prn32(offset, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); + prng32(offset, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C); for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; @@ -142,7 +141,7 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, * were an item for which both hashes indicated the same * bucket. */ - prn32(i, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); + prng32(i, LG_CKH_BUCKET_CELLS, ckh->prng_state, CKH_A, CKH_C); cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; assert(cell->key != NULL); @@ -265,15 +264,15 @@ ckh_grow(ckh_t *ckh) size_t usize; lg_curcells++; - usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); + usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) { ret = true; - goto RETURN; + goto label_return; } tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); if (tab == NULL) { ret = true; - goto RETURN; + goto label_return; } /* Swap in new table. */ ttab = ckh->tab; @@ -293,7 +292,7 @@ ckh_grow(ckh_t *ckh) } ret = false; -RETURN: +label_return: return (ret); } @@ -310,7 +309,7 @@ ckh_shrink(ckh_t *ckh) */ lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; - usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); + usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); if (usize == 0) return; tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); @@ -362,7 +361,7 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) ckh->ninserts = 0; ckh->nrelocs = 0; #endif - ckh->prn_state = 42; /* Value doesn't really matter. */ + ckh->prng_state = 42; /* Value doesn't really matter. */ ckh->count = 0; /* @@ -383,23 +382,19 @@ ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) ckh->hash = hash; ckh->keycomp = keycomp; - usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE, NULL); + usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); if (usize == 0) { ret = true; - goto RETURN; + goto label_return; } ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); if (ckh->tab == NULL) { ret = true; - goto RETURN; + goto label_return; } -#ifdef JEMALLOC_DEBUG - ckh->magic = CKH_MAGIC; -#endif - ret = false; -RETURN: +label_return: return (ret); } @@ -408,7 +403,6 @@ ckh_delete(ckh_t *ckh) { assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); #ifdef CKH_VERBOSE malloc_printf( @@ -433,7 +427,6 @@ ckh_count(ckh_t *ckh) { assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); return (ckh->count); } @@ -464,7 +457,6 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data) bool ret; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); assert(ckh_search(ckh, key, NULL, NULL)); #ifdef CKH_COUNT @@ -474,12 +466,12 @@ ckh_insert(ckh_t *ckh, const void *key, const void *data) while (ckh_try_insert(ckh, &key, &data)) { if (ckh_grow(ckh)) { ret = true; - goto RETURN; + goto label_return; } } ret = false; -RETURN: +label_return: return (ret); } @@ -489,7 +481,6 @@ ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data) size_t cell; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); cell = ckh_isearch(ckh, searchkey); if (cell != SIZE_T_MAX) { @@ -521,7 +512,6 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) size_t cell; assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); cell = ckh_isearch(ckh, searchkey); if (cell != SIZE_T_MAX) { @@ -545,7 +535,7 @@ ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) assert(hash1 != NULL); assert(hash2 != NULL); - h = hash(key, strlen((const char *)key), 0x94122f335b332aeaLLU); + h = hash(key, strlen((const char *)key), UINT64_C(0x94122f335b332aea)); if (minbits <= 32) { /* * Avoid doing multiple hashes, since a single hash provides @@ -556,7 +546,7 @@ ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) } else { ret1 = h; ret2 = hash(key, strlen((const char *)key), - 0x8432a476666bbc13LLU); + UINT64_C(0x8432a476666bbc13)); } *hash1 = ret1; @@ -593,7 +583,7 @@ ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, u.i = 0; #endif u.v = key; - h = hash(&u.i, sizeof(u.i), 0xd983396e68886082LLU); + h = hash(&u.i, sizeof(u.i), UINT64_C(0xd983396e68886082)); if (minbits <= 32) { /* * Avoid doing multiple hashes, since a single hash provides @@ -604,7 +594,7 @@ ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, } else { assert(SIZEOF_PTR == 8); ret1 = h; - ret2 = hash(&u.i, sizeof(u.i), 0x5e2be9aff8709a5dLLU); + ret2 = hash(&u.i, sizeof(u.i), UINT64_C(0x5e2be9aff8709a5d)); } *hash1 = ret1; diff --git a/deps/jemalloc/src/ctl.c b/deps/jemalloc/src/ctl.c index e5336d36949..55e766777fd 100644 --- a/deps/jemalloc/src/ctl.c +++ b/deps/jemalloc/src/ctl.c @@ -8,14 +8,38 @@ * ctl_mtx protects the following: * - ctl_stats.* * - opt_prof_active - * - swap_enabled - * - swap_prezeroed */ static malloc_mutex_t ctl_mtx; static bool ctl_initialized; static uint64_t ctl_epoch; static ctl_stats_t ctl_stats; +/******************************************************************************/ +/* Helpers for named and indexed nodes. */ + +static inline const ctl_named_node_t * +ctl_named_node(const ctl_node_t *node) +{ + + return ((node->named) ? (const ctl_named_node_t *)node : NULL); +} + +static inline const ctl_named_node_t * +ctl_named_children(const ctl_named_node_t *node, int index) +{ + const ctl_named_node_t *children = ctl_named_node(node->children); + + return (children ? &children[index] : NULL); +} + +static inline const ctl_indexed_node_t * +ctl_indexed_node(const ctl_node_t *node) +{ + + return ((node->named == false) ? (const ctl_indexed_node_t *)node : + NULL); +} + /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -24,19 +48,15 @@ static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ size_t *oldlenp, void *newp, size_t newlen); #define INDEX_PROTO(n) \ -const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \ +const ctl_named_node_t *n##_index(const size_t *mib, size_t miblen, \ size_t i); -#ifdef JEMALLOC_STATS static bool ctl_arena_init(ctl_arena_stats_t *astats); -#endif static void ctl_arena_clear(ctl_arena_stats_t *astats); -#ifdef JEMALLOC_STATS static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena); static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats); -#endif static void ctl_arena_refresh(arena_t *arena, unsigned i); static void ctl_refresh(void); static bool ctl_init(void); @@ -45,67 +65,51 @@ static int ctl_lookup(const char *name, ctl_node_t const **nodesp, CTL_PROTO(version) CTL_PROTO(epoch) -#ifdef JEMALLOC_TCACHE -CTL_PROTO(tcache_flush) -#endif +CTL_PROTO(thread_tcache_enabled) +CTL_PROTO(thread_tcache_flush) CTL_PROTO(thread_arena) -#ifdef JEMALLOC_STATS CTL_PROTO(thread_allocated) CTL_PROTO(thread_allocatedp) CTL_PROTO(thread_deallocated) CTL_PROTO(thread_deallocatedp) -#endif CTL_PROTO(config_debug) CTL_PROTO(config_dss) -CTL_PROTO(config_dynamic_page_shift) CTL_PROTO(config_fill) CTL_PROTO(config_lazy_lock) +CTL_PROTO(config_mremap) +CTL_PROTO(config_munmap) CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) CTL_PROTO(config_stats) -CTL_PROTO(config_swap) -CTL_PROTO(config_sysv) CTL_PROTO(config_tcache) -CTL_PROTO(config_tiny) CTL_PROTO(config_tls) +CTL_PROTO(config_utrace) +CTL_PROTO(config_valgrind) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) -CTL_PROTO(opt_lg_qspace_max) -CTL_PROTO(opt_lg_cspace_max) CTL_PROTO(opt_lg_chunk) CTL_PROTO(opt_narenas) CTL_PROTO(opt_lg_dirty_mult) CTL_PROTO(opt_stats_print) -#ifdef JEMALLOC_FILL CTL_PROTO(opt_junk) CTL_PROTO(opt_zero) -#endif -#ifdef JEMALLOC_SYSV -CTL_PROTO(opt_sysv) -#endif -#ifdef JEMALLOC_XMALLOC +CTL_PROTO(opt_quarantine) +CTL_PROTO(opt_redzone) +CTL_PROTO(opt_utrace) +CTL_PROTO(opt_valgrind) CTL_PROTO(opt_xmalloc) -#endif -#ifdef JEMALLOC_TCACHE CTL_PROTO(opt_tcache) -CTL_PROTO(opt_lg_tcache_gc_sweep) -#endif -#ifdef JEMALLOC_PROF +CTL_PROTO(opt_lg_tcache_max) CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) CTL_PROTO(opt_prof_active) -CTL_PROTO(opt_lg_prof_bt_max) CTL_PROTO(opt_lg_prof_sample) CTL_PROTO(opt_lg_prof_interval) CTL_PROTO(opt_prof_gdump) +CTL_PROTO(opt_prof_final) CTL_PROTO(opt_prof_leak) CTL_PROTO(opt_prof_accum) -CTL_PROTO(opt_lg_prof_tcmax) -#endif -#ifdef JEMALLOC_SWAP -CTL_PROTO(opt_overcommit) -#endif CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_run_size) @@ -115,39 +119,15 @@ INDEX_PROTO(arenas_lrun_i) CTL_PROTO(arenas_narenas) CTL_PROTO(arenas_initialized) CTL_PROTO(arenas_quantum) -CTL_PROTO(arenas_cacheline) -CTL_PROTO(arenas_subpage) -CTL_PROTO(arenas_pagesize) -CTL_PROTO(arenas_chunksize) -#ifdef JEMALLOC_TINY -CTL_PROTO(arenas_tspace_min) -CTL_PROTO(arenas_tspace_max) -#endif -CTL_PROTO(arenas_qspace_min) -CTL_PROTO(arenas_qspace_max) -CTL_PROTO(arenas_cspace_min) -CTL_PROTO(arenas_cspace_max) -CTL_PROTO(arenas_sspace_min) -CTL_PROTO(arenas_sspace_max) -#ifdef JEMALLOC_TCACHE +CTL_PROTO(arenas_page) CTL_PROTO(arenas_tcache_max) -#endif -CTL_PROTO(arenas_ntbins) -CTL_PROTO(arenas_nqbins) -CTL_PROTO(arenas_ncbins) -CTL_PROTO(arenas_nsbins) CTL_PROTO(arenas_nbins) -#ifdef JEMALLOC_TCACHE CTL_PROTO(arenas_nhbins) -#endif CTL_PROTO(arenas_nlruns) CTL_PROTO(arenas_purge) -#ifdef JEMALLOC_PROF CTL_PROTO(prof_active) CTL_PROTO(prof_dump) CTL_PROTO(prof_interval) -#endif -#ifdef JEMALLOC_STATS CTL_PROTO(stats_chunks_current) CTL_PROTO(stats_chunks_total) CTL_PROTO(stats_chunks_high) @@ -166,46 +146,29 @@ CTL_PROTO(stats_arenas_i_bins_j_allocated) CTL_PROTO(stats_arenas_i_bins_j_nmalloc) CTL_PROTO(stats_arenas_i_bins_j_ndalloc) CTL_PROTO(stats_arenas_i_bins_j_nrequests) -#ifdef JEMALLOC_TCACHE CTL_PROTO(stats_arenas_i_bins_j_nfills) CTL_PROTO(stats_arenas_i_bins_j_nflushes) -#endif CTL_PROTO(stats_arenas_i_bins_j_nruns) CTL_PROTO(stats_arenas_i_bins_j_nreruns) -CTL_PROTO(stats_arenas_i_bins_j_highruns) CTL_PROTO(stats_arenas_i_bins_j_curruns) INDEX_PROTO(stats_arenas_i_bins_j) CTL_PROTO(stats_arenas_i_lruns_j_nmalloc) CTL_PROTO(stats_arenas_i_lruns_j_ndalloc) CTL_PROTO(stats_arenas_i_lruns_j_nrequests) -CTL_PROTO(stats_arenas_i_lruns_j_highruns) CTL_PROTO(stats_arenas_i_lruns_j_curruns) INDEX_PROTO(stats_arenas_i_lruns_j) -#endif CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_pactive) CTL_PROTO(stats_arenas_i_pdirty) -#ifdef JEMALLOC_STATS CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_npurge) CTL_PROTO(stats_arenas_i_nmadvise) CTL_PROTO(stats_arenas_i_purged) -#endif INDEX_PROTO(stats_arenas_i) -#ifdef JEMALLOC_STATS CTL_PROTO(stats_cactive) CTL_PROTO(stats_allocated) CTL_PROTO(stats_active) CTL_PROTO(stats_mapped) -#endif -#ifdef JEMALLOC_SWAP -# ifdef JEMALLOC_STATS -CTL_PROTO(swap_avail) -# endif -CTL_PROTO(swap_prezeroed) -CTL_PROTO(swap_nfds) -CTL_PROTO(swap_fds) -#endif /******************************************************************************/ /* mallctl tree. */ @@ -213,296 +176,223 @@ CTL_PROTO(swap_fds) /* Maximum tree depth. */ #define CTL_MAX_DEPTH 6 -#define NAME(n) true, {.named = {n -#define CHILD(c) sizeof(c##_node) / sizeof(ctl_node_t), c##_node}}, NULL -#define CTL(c) 0, NULL}}, c##_ctl +#define NAME(n) {true}, n +#define CHILD(t, c) \ + sizeof(c##_node) / sizeof(ctl_##t##_node_t), \ + (ctl_node_t *)c##_node, \ + NULL +#define CTL(c) 0, NULL, c##_ctl /* * Only handles internal indexed nodes, since there are currently no external * ones. */ -#define INDEX(i) false, {.indexed = {i##_index}}, NULL +#define INDEX(i) {false}, i##_index -#ifdef JEMALLOC_TCACHE -static const ctl_node_t tcache_node[] = { - {NAME("flush"), CTL(tcache_flush)} +static const ctl_named_node_t tcache_node[] = { + {NAME("enabled"), CTL(thread_tcache_enabled)}, + {NAME("flush"), CTL(thread_tcache_flush)} }; -#endif -static const ctl_node_t thread_node[] = { - {NAME("arena"), CTL(thread_arena)} -#ifdef JEMALLOC_STATS - , +static const ctl_named_node_t thread_node[] = { + {NAME("arena"), CTL(thread_arena)}, {NAME("allocated"), CTL(thread_allocated)}, {NAME("allocatedp"), CTL(thread_allocatedp)}, {NAME("deallocated"), CTL(thread_deallocated)}, - {NAME("deallocatedp"), CTL(thread_deallocatedp)} -#endif + {NAME("deallocatedp"), CTL(thread_deallocatedp)}, + {NAME("tcache"), CHILD(named, tcache)} }; -static const ctl_node_t config_node[] = { +static const ctl_named_node_t config_node[] = { {NAME("debug"), CTL(config_debug)}, {NAME("dss"), CTL(config_dss)}, - {NAME("dynamic_page_shift"), CTL(config_dynamic_page_shift)}, {NAME("fill"), CTL(config_fill)}, {NAME("lazy_lock"), CTL(config_lazy_lock)}, + {NAME("mremap"), CTL(config_mremap)}, + {NAME("munmap"), CTL(config_munmap)}, {NAME("prof"), CTL(config_prof)}, {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, {NAME("stats"), CTL(config_stats)}, - {NAME("swap"), CTL(config_swap)}, - {NAME("sysv"), CTL(config_sysv)}, {NAME("tcache"), CTL(config_tcache)}, - {NAME("tiny"), CTL(config_tiny)}, {NAME("tls"), CTL(config_tls)}, + {NAME("utrace"), CTL(config_utrace)}, + {NAME("valgrind"), CTL(config_valgrind)}, {NAME("xmalloc"), CTL(config_xmalloc)} }; -static const ctl_node_t opt_node[] = { +static const ctl_named_node_t opt_node[] = { {NAME("abort"), CTL(opt_abort)}, - {NAME("lg_qspace_max"), CTL(opt_lg_qspace_max)}, - {NAME("lg_cspace_max"), CTL(opt_lg_cspace_max)}, {NAME("lg_chunk"), CTL(opt_lg_chunk)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, - {NAME("stats_print"), CTL(opt_stats_print)} -#ifdef JEMALLOC_FILL - , + {NAME("stats_print"), CTL(opt_stats_print)}, {NAME("junk"), CTL(opt_junk)}, - {NAME("zero"), CTL(opt_zero)} -#endif -#ifdef JEMALLOC_SYSV - , - {NAME("sysv"), CTL(opt_sysv)} -#endif -#ifdef JEMALLOC_XMALLOC - , - {NAME("xmalloc"), CTL(opt_xmalloc)} -#endif -#ifdef JEMALLOC_TCACHE - , + {NAME("zero"), CTL(opt_zero)}, + {NAME("quarantine"), CTL(opt_quarantine)}, + {NAME("redzone"), CTL(opt_redzone)}, + {NAME("utrace"), CTL(opt_utrace)}, + {NAME("valgrind"), CTL(opt_valgrind)}, + {NAME("xmalloc"), CTL(opt_xmalloc)}, {NAME("tcache"), CTL(opt_tcache)}, - {NAME("lg_tcache_gc_sweep"), CTL(opt_lg_tcache_gc_sweep)} -#endif -#ifdef JEMALLOC_PROF - , + {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, {NAME("prof"), CTL(opt_prof)}, {NAME("prof_prefix"), CTL(opt_prof_prefix)}, {NAME("prof_active"), CTL(opt_prof_active)}, - {NAME("lg_prof_bt_max"), CTL(opt_lg_prof_bt_max)}, {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, {NAME("prof_gdump"), CTL(opt_prof_gdump)}, + {NAME("prof_final"), CTL(opt_prof_final)}, {NAME("prof_leak"), CTL(opt_prof_leak)}, - {NAME("prof_accum"), CTL(opt_prof_accum)}, - {NAME("lg_prof_tcmax"), CTL(opt_lg_prof_tcmax)} -#endif -#ifdef JEMALLOC_SWAP - , - {NAME("overcommit"), CTL(opt_overcommit)} -#endif + {NAME("prof_accum"), CTL(opt_prof_accum)} }; -static const ctl_node_t arenas_bin_i_node[] = { +static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, {NAME("run_size"), CTL(arenas_bin_i_run_size)} }; -static const ctl_node_t super_arenas_bin_i_node[] = { - {NAME(""), CHILD(arenas_bin_i)} +static const ctl_named_node_t super_arenas_bin_i_node[] = { + {NAME(""), CHILD(named, arenas_bin_i)} }; -static const ctl_node_t arenas_bin_node[] = { +static const ctl_indexed_node_t arenas_bin_node[] = { {INDEX(arenas_bin_i)} }; -static const ctl_node_t arenas_lrun_i_node[] = { +static const ctl_named_node_t arenas_lrun_i_node[] = { {NAME("size"), CTL(arenas_lrun_i_size)} }; -static const ctl_node_t super_arenas_lrun_i_node[] = { - {NAME(""), CHILD(arenas_lrun_i)} +static const ctl_named_node_t super_arenas_lrun_i_node[] = { + {NAME(""), CHILD(named, arenas_lrun_i)} }; -static const ctl_node_t arenas_lrun_node[] = { +static const ctl_indexed_node_t arenas_lrun_node[] = { {INDEX(arenas_lrun_i)} }; -static const ctl_node_t arenas_node[] = { +static const ctl_named_node_t arenas_node[] = { {NAME("narenas"), CTL(arenas_narenas)}, {NAME("initialized"), CTL(arenas_initialized)}, {NAME("quantum"), CTL(arenas_quantum)}, - {NAME("cacheline"), CTL(arenas_cacheline)}, - {NAME("subpage"), CTL(arenas_subpage)}, - {NAME("pagesize"), CTL(arenas_pagesize)}, - {NAME("chunksize"), CTL(arenas_chunksize)}, -#ifdef JEMALLOC_TINY - {NAME("tspace_min"), CTL(arenas_tspace_min)}, - {NAME("tspace_max"), CTL(arenas_tspace_max)}, -#endif - {NAME("qspace_min"), CTL(arenas_qspace_min)}, - {NAME("qspace_max"), CTL(arenas_qspace_max)}, - {NAME("cspace_min"), CTL(arenas_cspace_min)}, - {NAME("cspace_max"), CTL(arenas_cspace_max)}, - {NAME("sspace_min"), CTL(arenas_sspace_min)}, - {NAME("sspace_max"), CTL(arenas_sspace_max)}, -#ifdef JEMALLOC_TCACHE + {NAME("page"), CTL(arenas_page)}, {NAME("tcache_max"), CTL(arenas_tcache_max)}, -#endif - {NAME("ntbins"), CTL(arenas_ntbins)}, - {NAME("nqbins"), CTL(arenas_nqbins)}, - {NAME("ncbins"), CTL(arenas_ncbins)}, - {NAME("nsbins"), CTL(arenas_nsbins)}, {NAME("nbins"), CTL(arenas_nbins)}, -#ifdef JEMALLOC_TCACHE {NAME("nhbins"), CTL(arenas_nhbins)}, -#endif - {NAME("bin"), CHILD(arenas_bin)}, + {NAME("bin"), CHILD(indexed, arenas_bin)}, {NAME("nlruns"), CTL(arenas_nlruns)}, - {NAME("lrun"), CHILD(arenas_lrun)}, + {NAME("lrun"), CHILD(indexed, arenas_lrun)}, {NAME("purge"), CTL(arenas_purge)} }; -#ifdef JEMALLOC_PROF -static const ctl_node_t prof_node[] = { +static const ctl_named_node_t prof_node[] = { {NAME("active"), CTL(prof_active)}, {NAME("dump"), CTL(prof_dump)}, {NAME("interval"), CTL(prof_interval)} }; -#endif -#ifdef JEMALLOC_STATS -static const ctl_node_t stats_chunks_node[] = { +static const ctl_named_node_t stats_chunks_node[] = { {NAME("current"), CTL(stats_chunks_current)}, {NAME("total"), CTL(stats_chunks_total)}, {NAME("high"), CTL(stats_chunks_high)} }; -static const ctl_node_t stats_huge_node[] = { +static const ctl_named_node_t stats_huge_node[] = { {NAME("allocated"), CTL(stats_huge_allocated)}, {NAME("nmalloc"), CTL(stats_huge_nmalloc)}, {NAME("ndalloc"), CTL(stats_huge_ndalloc)} }; -static const ctl_node_t stats_arenas_i_small_node[] = { +static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} }; -static const ctl_node_t stats_arenas_i_large_node[] = { +static const ctl_named_node_t stats_arenas_i_large_node[] = { {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} }; -static const ctl_node_t stats_arenas_i_bins_j_node[] = { +static const ctl_named_node_t stats_arenas_i_bins_j_node[] = { {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, -#ifdef JEMALLOC_TCACHE {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, -#endif {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, - {NAME("highruns"), CTL(stats_arenas_i_bins_j_highruns)}, {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} }; -static const ctl_node_t super_stats_arenas_i_bins_j_node[] = { - {NAME(""), CHILD(stats_arenas_i_bins_j)} +static const ctl_named_node_t super_stats_arenas_i_bins_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_bins_j)} }; -static const ctl_node_t stats_arenas_i_bins_node[] = { +static const ctl_indexed_node_t stats_arenas_i_bins_node[] = { {INDEX(stats_arenas_i_bins_j)} }; -static const ctl_node_t stats_arenas_i_lruns_j_node[] = { +static const ctl_named_node_t stats_arenas_i_lruns_j_node[] = { {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, - {NAME("highruns"), CTL(stats_arenas_i_lruns_j_highruns)}, {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} }; -static const ctl_node_t super_stats_arenas_i_lruns_j_node[] = { - {NAME(""), CHILD(stats_arenas_i_lruns_j)} +static const ctl_named_node_t super_stats_arenas_i_lruns_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_lruns_j)} }; -static const ctl_node_t stats_arenas_i_lruns_node[] = { +static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = { {INDEX(stats_arenas_i_lruns_j)} }; -#endif -static const ctl_node_t stats_arenas_i_node[] = { +static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)}, - {NAME("pdirty"), CTL(stats_arenas_i_pdirty)} -#ifdef JEMALLOC_STATS - , + {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("npurge"), CTL(stats_arenas_i_npurge)}, {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, {NAME("purged"), CTL(stats_arenas_i_purged)}, - {NAME("small"), CHILD(stats_arenas_i_small)}, - {NAME("large"), CHILD(stats_arenas_i_large)}, - {NAME("bins"), CHILD(stats_arenas_i_bins)}, - {NAME("lruns"), CHILD(stats_arenas_i_lruns)} -#endif + {NAME("small"), CHILD(named, stats_arenas_i_small)}, + {NAME("large"), CHILD(named, stats_arenas_i_large)}, + {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, + {NAME("lruns"), CHILD(indexed, stats_arenas_i_lruns)} }; -static const ctl_node_t super_stats_arenas_i_node[] = { - {NAME(""), CHILD(stats_arenas_i)} +static const ctl_named_node_t super_stats_arenas_i_node[] = { + {NAME(""), CHILD(named, stats_arenas_i)} }; -static const ctl_node_t stats_arenas_node[] = { +static const ctl_indexed_node_t stats_arenas_node[] = { {INDEX(stats_arenas_i)} }; -static const ctl_node_t stats_node[] = { -#ifdef JEMALLOC_STATS +static const ctl_named_node_t stats_node[] = { {NAME("cactive"), CTL(stats_cactive)}, {NAME("allocated"), CTL(stats_allocated)}, {NAME("active"), CTL(stats_active)}, {NAME("mapped"), CTL(stats_mapped)}, - {NAME("chunks"), CHILD(stats_chunks)}, - {NAME("huge"), CHILD(stats_huge)}, -#endif - {NAME("arenas"), CHILD(stats_arenas)} + {NAME("chunks"), CHILD(named, stats_chunks)}, + {NAME("huge"), CHILD(named, stats_huge)}, + {NAME("arenas"), CHILD(indexed, stats_arenas)} }; -#ifdef JEMALLOC_SWAP -static const ctl_node_t swap_node[] = { -# ifdef JEMALLOC_STATS - {NAME("avail"), CTL(swap_avail)}, -# endif - {NAME("prezeroed"), CTL(swap_prezeroed)}, - {NAME("nfds"), CTL(swap_nfds)}, - {NAME("fds"), CTL(swap_fds)} -}; -#endif - -static const ctl_node_t root_node[] = { +static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, -#ifdef JEMALLOC_TCACHE - {NAME("tcache"), CHILD(tcache)}, -#endif - {NAME("thread"), CHILD(thread)}, - {NAME("config"), CHILD(config)}, - {NAME("opt"), CHILD(opt)}, - {NAME("arenas"), CHILD(arenas)}, -#ifdef JEMALLOC_PROF - {NAME("prof"), CHILD(prof)}, -#endif - {NAME("stats"), CHILD(stats)} -#ifdef JEMALLOC_SWAP - , - {NAME("swap"), CHILD(swap)} -#endif + {NAME("thread"), CHILD(named, thread)}, + {NAME("config"), CHILD(named, config)}, + {NAME("opt"), CHILD(named, opt)}, + {NAME("arenas"), CHILD(named, arenas)}, + {NAME("prof"), CHILD(named, prof)}, + {NAME("stats"), CHILD(named, stats)} }; -static const ctl_node_t super_root_node[] = { - {NAME(""), CHILD(root)} +static const ctl_named_node_t super_root_node[] = { + {NAME(""), CHILD(named, root)} }; #undef NAME @@ -512,17 +402,10 @@ static const ctl_node_t super_root_node[] = { /******************************************************************************/ -#ifdef JEMALLOC_STATS static bool ctl_arena_init(ctl_arena_stats_t *astats) { - if (astats->bstats == NULL) { - astats->bstats = (malloc_bin_stats_t *)base_alloc(nbins * - sizeof(malloc_bin_stats_t)); - if (astats->bstats == NULL) - return (true); - } if (astats->lstats == NULL) { astats->lstats = (malloc_large_stats_t *)base_alloc(nlclasses * sizeof(malloc_large_stats_t)); @@ -532,7 +415,6 @@ ctl_arena_init(ctl_arena_stats_t *astats) return (false); } -#endif static void ctl_arena_clear(ctl_arena_stats_t *astats) @@ -540,18 +422,18 @@ ctl_arena_clear(ctl_arena_stats_t *astats) astats->pactive = 0; astats->pdirty = 0; -#ifdef JEMALLOC_STATS - memset(&astats->astats, 0, sizeof(arena_stats_t)); - astats->allocated_small = 0; - astats->nmalloc_small = 0; - astats->ndalloc_small = 0; - astats->nrequests_small = 0; - memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t)); - memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); -#endif + if (config_stats) { + memset(&astats->astats, 0, sizeof(arena_stats_t)); + astats->allocated_small = 0; + astats->nmalloc_small = 0; + astats->ndalloc_small = 0; + astats->nrequests_small = 0; + memset(astats->bstats, 0, NBINS * sizeof(malloc_bin_stats_t)); + memset(astats->lstats, 0, nlclasses * + sizeof(malloc_large_stats_t)); + } } -#ifdef JEMALLOC_STATS static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) { @@ -560,7 +442,7 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats); - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { cstats->allocated_small += cstats->bstats[i].allocated; cstats->nmalloc_small += cstats->bstats[i].nmalloc; cstats->ndalloc_small += cstats->bstats[i].ndalloc; @@ -595,26 +477,24 @@ ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; sstats->lstats[i].nrequests += astats->lstats[i].nrequests; - sstats->lstats[i].highruns += astats->lstats[i].highruns; sstats->lstats[i].curruns += astats->lstats[i].curruns; } - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { sstats->bstats[i].allocated += astats->bstats[i].allocated; sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sstats->bstats[i].nrequests += astats->bstats[i].nrequests; -#ifdef JEMALLOC_TCACHE - sstats->bstats[i].nfills += astats->bstats[i].nfills; - sstats->bstats[i].nflushes += astats->bstats[i].nflushes; -#endif + if (config_tcache) { + sstats->bstats[i].nfills += astats->bstats[i].nfills; + sstats->bstats[i].nflushes += + astats->bstats[i].nflushes; + } sstats->bstats[i].nruns += astats->bstats[i].nruns; sstats->bstats[i].reruns += astats->bstats[i].reruns; - sstats->bstats[i].highruns += astats->bstats[i].highruns; sstats->bstats[i].curruns += astats->bstats[i].curruns; } } -#endif static void ctl_arena_refresh(arena_t *arena, unsigned i) @@ -625,38 +505,38 @@ ctl_arena_refresh(arena_t *arena, unsigned i) ctl_arena_clear(astats); sstats->nthreads += astats->nthreads; -#ifdef JEMALLOC_STATS - ctl_arena_stats_amerge(astats, arena); - /* Merge into sum stats as well. */ - ctl_arena_stats_smerge(sstats, astats); -#else - astats->pactive += arena->nactive; - astats->pdirty += arena->ndirty; - /* Merge into sum stats as well. */ - sstats->pactive += arena->nactive; - sstats->pdirty += arena->ndirty; -#endif + if (config_stats) { + ctl_arena_stats_amerge(astats, arena); + /* Merge into sum stats as well. */ + ctl_arena_stats_smerge(sstats, astats); + } else { + astats->pactive += arena->nactive; + astats->pdirty += arena->ndirty; + /* Merge into sum stats as well. */ + sstats->pactive += arena->nactive; + sstats->pdirty += arena->ndirty; + } } static void ctl_refresh(void) { unsigned i; - arena_t *tarenas[narenas]; - -#ifdef JEMALLOC_STATS - malloc_mutex_lock(&chunks_mtx); - ctl_stats.chunks.current = stats_chunks.curchunks; - ctl_stats.chunks.total = stats_chunks.nchunks; - ctl_stats.chunks.high = stats_chunks.highchunks; - malloc_mutex_unlock(&chunks_mtx); - - malloc_mutex_lock(&huge_mtx); - ctl_stats.huge.allocated = huge_allocated; - ctl_stats.huge.nmalloc = huge_nmalloc; - ctl_stats.huge.ndalloc = huge_ndalloc; - malloc_mutex_unlock(&huge_mtx); -#endif + VARIABLE_ARRAY(arena_t *, tarenas, narenas); + + if (config_stats) { + malloc_mutex_lock(&chunks_mtx); + ctl_stats.chunks.current = stats_chunks.curchunks; + ctl_stats.chunks.total = stats_chunks.nchunks; + ctl_stats.chunks.high = stats_chunks.highchunks; + malloc_mutex_unlock(&chunks_mtx); + + malloc_mutex_lock(&huge_mtx); + ctl_stats.huge.allocated = huge_allocated; + ctl_stats.huge.nmalloc = huge_nmalloc; + ctl_stats.huge.ndalloc = huge_ndalloc; + malloc_mutex_unlock(&huge_mtx); + } /* * Clear sum stats, since they will be merged into by @@ -682,20 +562,14 @@ ctl_refresh(void) ctl_arena_refresh(tarenas[i], i); } -#ifdef JEMALLOC_STATS - ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small - + ctl_stats.arenas[narenas].astats.allocated_large - + ctl_stats.huge.allocated; - ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT) - + ctl_stats.huge.allocated; - ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); - -# ifdef JEMALLOC_SWAP - malloc_mutex_lock(&swap_mtx); - ctl_stats.swap_avail = swap_avail; - malloc_mutex_unlock(&swap_mtx); -# endif -#endif + if (config_stats) { + ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small + + ctl_stats.arenas[narenas].astats.allocated_large + + ctl_stats.huge.allocated; + ctl_stats.active = (ctl_stats.arenas[narenas].pactive << + LG_PAGE) + ctl_stats.huge.allocated; + ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); + } ctl_epoch++; } @@ -707,10 +581,6 @@ ctl_init(void) malloc_mutex_lock(&ctl_mtx); if (ctl_initialized == false) { -#ifdef JEMALLOC_STATS - unsigned i; -#endif - /* * Allocate space for one extra arena stats element, which * contains summed stats across all arenas. @@ -719,7 +589,7 @@ ctl_init(void) (narenas + 1) * sizeof(ctl_arena_stats_t)); if (ctl_stats.arenas == NULL) { ret = true; - goto RETURN; + goto label_return; } memset(ctl_stats.arenas, 0, (narenas + 1) * sizeof(ctl_arena_stats_t)); @@ -729,14 +599,15 @@ ctl_init(void) * ever get used. Lazy initialization would allow errors to * cause inconsistent state to be viewable by the application. */ -#ifdef JEMALLOC_STATS - for (i = 0; i <= narenas; i++) { - if (ctl_arena_init(&ctl_stats.arenas[i])) { - ret = true; - goto RETURN; + if (config_stats) { + unsigned i; + for (i = 0; i <= narenas; i++) { + if (ctl_arena_init(&ctl_stats.arenas[i])) { + ret = true; + goto label_return; + } } } -#endif ctl_stats.arenas[narenas].initialized = true; ctl_epoch = 0; @@ -745,7 +616,7 @@ ctl_init(void) } ret = false; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } @@ -757,7 +628,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, int ret; const char *elm, *tdot, *dot; size_t elen, i, j; - const ctl_node_t *node; + const ctl_named_node_t *node; elm = name; /* Equivalent to strchrnul(). */ @@ -765,54 +636,53 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); if (elen == 0) { ret = ENOENT; - goto RETURN; + goto label_return; } node = super_root_node; for (i = 0; i < *depthp; i++) { - assert(node->named); - assert(node->u.named.nchildren > 0); - if (node->u.named.children[0].named) { - const ctl_node_t *pnode = node; + assert(node); + assert(node->nchildren > 0); + if (ctl_named_node(node->children) != NULL) { + const ctl_named_node_t *pnode = node; /* Children are named. */ - for (j = 0; j < node->u.named.nchildren; j++) { - const ctl_node_t *child = - &node->u.named.children[j]; - if (strlen(child->u.named.name) == elen - && strncmp(elm, child->u.named.name, - elen) == 0) { + for (j = 0; j < node->nchildren; j++) { + const ctl_named_node_t *child = + ctl_named_children(node, j); + if (strlen(child->name) == elen && + strncmp(elm, child->name, elen) == 0) { node = child; if (nodesp != NULL) - nodesp[i] = node; + nodesp[i] = + (const ctl_node_t *)node; mibp[i] = j; break; } } if (node == pnode) { ret = ENOENT; - goto RETURN; + goto label_return; } } else { - unsigned long index; - const ctl_node_t *inode; + uintmax_t index; + const ctl_indexed_node_t *inode; /* Children are indexed. */ - index = strtoul(elm, NULL, 10); - if (index == ULONG_MAX) { + index = malloc_strtoumax(elm, NULL, 10); + if (index == UINTMAX_MAX || index > SIZE_T_MAX) { ret = ENOENT; - goto RETURN; + goto label_return; } - inode = &node->u.named.children[0]; - node = inode->u.indexed.index(mibp, *depthp, - index); + inode = ctl_indexed_node(node->children); + node = inode->index(mibp, *depthp, (size_t)index); if (node == NULL) { ret = ENOENT; - goto RETURN; + goto label_return; } if (nodesp != NULL) - nodesp[i] = node; + nodesp[i] = (const ctl_node_t *)node; mibp[i] = (size_t)index; } @@ -824,7 +694,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, * in this path through the tree. */ ret = ENOENT; - goto RETURN; + goto label_return; } /* Complete lookup successful. */ *depthp = i + 1; @@ -835,7 +705,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, if (*dot == '\0') { /* No more elements. */ ret = ENOENT; - goto RETURN; + goto label_return; } elm = &dot[1]; dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : @@ -844,7 +714,7 @@ ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, } ret = 0; -RETURN: +label_return: return (ret); } @@ -856,25 +726,27 @@ ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t depth; ctl_node_t const *nodes[CTL_MAX_DEPTH]; size_t mib[CTL_MAX_DEPTH]; + const ctl_named_node_t *node; if (ctl_initialized == false && ctl_init()) { ret = EAGAIN; - goto RETURN; + goto label_return; } depth = CTL_MAX_DEPTH; ret = ctl_lookup(name, nodes, mib, &depth); if (ret != 0) - goto RETURN; + goto label_return; - if (nodes[depth-1]->ctl == NULL) { + node = ctl_named_node(nodes[depth-1]); + if (node != NULL && node->ctl) + ret = node->ctl(mib, depth, oldp, oldlenp, newp, newlen); + else { /* The name refers to a partial path through the ctl tree. */ ret = ENOENT; - goto RETURN; } - ret = nodes[depth-1]->ctl(mib, depth, oldp, oldlenp, newp, newlen); -RETURN: +label_return: return(ret); } @@ -885,11 +757,11 @@ ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp) if (ctl_initialized == false && ctl_init()) { ret = EAGAIN; - goto RETURN; + goto label_return; } ret = ctl_lookup(name, NULL, mibp, miblenp); -RETURN: +label_return: return(ret); } @@ -898,46 +770,48 @@ ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - const ctl_node_t *node; + const ctl_named_node_t *node; size_t i; if (ctl_initialized == false && ctl_init()) { ret = EAGAIN; - goto RETURN; + goto label_return; } /* Iterate down the tree. */ node = super_root_node; for (i = 0; i < miblen; i++) { - if (node->u.named.children[0].named) { + assert(node); + assert(node->nchildren > 0); + if (ctl_named_node(node->children) != NULL) { /* Children are named. */ - if (node->u.named.nchildren <= mib[i]) { + if (node->nchildren <= mib[i]) { ret = ENOENT; - goto RETURN; + goto label_return; } - node = &node->u.named.children[mib[i]]; + node = ctl_named_children(node, mib[i]); } else { - const ctl_node_t *inode; + const ctl_indexed_node_t *inode; /* Indexed element. */ - inode = &node->u.named.children[0]; - node = inode->u.indexed.index(mib, miblen, mib[i]); + inode = ctl_indexed_node(node->children); + node = inode->index(mib, miblen, mib[i]); if (node == NULL) { ret = ENOENT; - goto RETURN; + goto label_return; } } } /* Call the ctl function. */ - if (node->ctl == NULL) { + if (node && node->ctl) + ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); + else { /* Partial MIB. */ ret = ENOENT; - goto RETURN; } - ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); -RETURN: +label_return: return(ret); } @@ -959,22 +833,17 @@ ctl_boot(void) #define READONLY() do { \ if (newp != NULL || newlen != 0) { \ ret = EPERM; \ - goto RETURN; \ + goto label_return; \ } \ } while (0) #define WRITEONLY() do { \ if (oldp != NULL || oldlenp != NULL) { \ ret = EPERM; \ - goto RETURN; \ + goto label_return; \ } \ } while (0) -#define VOID() do { \ - READONLY(); \ - WRITEONLY(); \ -} while (0) - #define READ(v, t) do { \ if (oldp != NULL && oldlenp != NULL) { \ if (*oldlenp != sizeof(t)) { \ @@ -982,7 +851,7 @@ ctl_boot(void) ? sizeof(t) : *oldlenp; \ memcpy(oldp, (void *)&v, copylen); \ ret = EINVAL; \ - goto RETURN; \ + goto label_return; \ } else \ *(t *)oldp = v; \ } \ @@ -992,12 +861,60 @@ ctl_boot(void) if (newp != NULL) { \ if (newlen != sizeof(t)) { \ ret = EINVAL; \ - goto RETURN; \ + goto label_return; \ } \ v = *(t *)newp; \ } \ } while (0) +/* + * There's a lot of code duplication in the following macros due to limitations + * in how nested cpp macros are expanded. + */ +#define CTL_RO_CLGEN(c, l, n, v, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + \ + if ((c) == false) \ + return (ENOENT); \ + if (l) \ + malloc_mutex_lock(&ctl_mtx); \ + READONLY(); \ + oldval = v; \ + READ(oldval, t); \ + \ + ret = 0; \ +label_return: \ + if (l) \ + malloc_mutex_unlock(&ctl_mtx); \ + return (ret); \ +} + +#define CTL_RO_CGEN(c, n, v, t) \ +static int \ +n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ + void *newp, size_t newlen) \ +{ \ + int ret; \ + t oldval; \ + \ + if ((c) == false) \ + return (ENOENT); \ + malloc_mutex_lock(&ctl_mtx); \ + READONLY(); \ + oldval = v; \ + READ(oldval, t); \ + \ + ret = 0; \ +label_return: \ + malloc_mutex_unlock(&ctl_mtx); \ + return (ret); \ +} + #define CTL_RO_GEN(n, v, t) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ @@ -1012,7 +929,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ READ(oldval, t); \ \ ret = 0; \ -RETURN: \ +label_return: \ malloc_mutex_unlock(&ctl_mtx); \ return (ret); \ } @@ -1021,7 +938,7 @@ RETURN: \ * ctl_mtx is not acquired, under the assumption that no pertinent data will * mutate during the call. */ -#define CTL_RO_NL_GEN(n, v, t) \ +#define CTL_RO_NL_CGEN(c, n, v, t) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ void *newp, size_t newlen) \ @@ -1029,33 +946,35 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ int ret; \ t oldval; \ \ + if ((c) == false) \ + return (ENOENT); \ READONLY(); \ oldval = v; \ READ(oldval, t); \ \ ret = 0; \ -RETURN: \ +label_return: \ return (ret); \ } -#define CTL_RO_TRUE_GEN(n) \ +#define CTL_RO_NL_GEN(n, v, t) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ void *newp, size_t newlen) \ { \ int ret; \ - bool oldval; \ + t oldval; \ \ READONLY(); \ - oldval = true; \ - READ(oldval, bool); \ + oldval = v; \ + READ(oldval, t); \ \ ret = 0; \ -RETURN: \ +label_return: \ return (ret); \ } -#define CTL_RO_FALSE_GEN(n) \ +#define CTL_RO_BOOL_CONFIG_GEN(n) \ static int \ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ void *newp, size_t newlen) \ @@ -1064,11 +983,11 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ bool oldval; \ \ READONLY(); \ - oldval = false; \ + oldval = n; \ READ(oldval, bool); \ \ ret = 0; \ -RETURN: \ +label_return: \ return (ret); \ } @@ -1082,41 +1001,60 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, uint64_t newval; malloc_mutex_lock(&ctl_mtx); - newval = 0; WRITE(newval, uint64_t); - if (newval != 0) + if (newp != NULL) ctl_refresh(); READ(ctl_epoch, uint64_t); ret = 0; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } -#ifdef JEMALLOC_TCACHE static int -tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) +thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { int ret; - tcache_t *tcache; + bool oldval; - VOID(); + if (config_tcache == false) + return (ENOENT); - tcache = TCACHE_GET(); - if (tcache == NULL) { - ret = 0; - goto RETURN; + oldval = tcache_enabled_get(); + if (newp != NULL) { + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + tcache_enabled_set(*(bool *)newp); } - tcache_destroy(tcache); - TCACHE_SET(NULL); + READ(oldval, bool); +label_return: ret = 0; -RETURN: return (ret); } -#endif + +static int +thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + + if (config_tcache == false) + return (ENOENT); + + READONLY(); + WRITEONLY(); + + tcache_flush(); + + ret = 0; +label_return: + return (ret); +} static int thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, @@ -1125,7 +1063,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; unsigned newind, oldind; - newind = oldind = choose_arena()->ind; + newind = oldind = choose_arena(NULL)->ind; WRITE(newind, unsigned); READ(oldind, unsigned); if (newind != oldind) { @@ -1134,191 +1072,108 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, if (newind >= narenas) { /* New arena index is out of range. */ ret = EFAULT; - goto RETURN; + goto label_return; } /* Initialize arena if necessary. */ malloc_mutex_lock(&arenas_lock); - if ((arena = arenas[newind]) == NULL) - arena = arenas_extend(newind); + if ((arena = arenas[newind]) == NULL && (arena = + arenas_extend(newind)) == NULL) { + malloc_mutex_unlock(&arenas_lock); + ret = EAGAIN; + goto label_return; + } + assert(arena == arenas[newind]); arenas[oldind]->nthreads--; arenas[newind]->nthreads++; malloc_mutex_unlock(&arenas_lock); - if (arena == NULL) { - ret = EAGAIN; - goto RETURN; - } /* Set new arena association. */ - ARENA_SET(arena); -#ifdef JEMALLOC_TCACHE - { - tcache_t *tcache = TCACHE_GET(); - if (tcache != NULL) - tcache->arena = arena; + if (config_tcache) { + tcache_t *tcache; + if ((uintptr_t)(tcache = *tcache_tsd_get()) > + (uintptr_t)TCACHE_STATE_MAX) { + tcache_arena_dissociate(tcache); + tcache_arena_associate(tcache, arena); + } } -#endif + arenas_tsd_set(&arena); } ret = 0; -RETURN: +label_return: return (ret); } -#ifdef JEMALLOC_STATS -CTL_RO_NL_GEN(thread_allocated, ALLOCATED_GET(), uint64_t); -CTL_RO_NL_GEN(thread_allocatedp, ALLOCATEDP_GET(), uint64_t *); -CTL_RO_NL_GEN(thread_deallocated, DEALLOCATED_GET(), uint64_t); -CTL_RO_NL_GEN(thread_deallocatedp, DEALLOCATEDP_GET(), uint64_t *); -#endif +CTL_RO_NL_CGEN(config_stats, thread_allocated, + thread_allocated_tsd_get()->allocated, uint64_t) +CTL_RO_NL_CGEN(config_stats, thread_allocatedp, + &thread_allocated_tsd_get()->allocated, uint64_t *) +CTL_RO_NL_CGEN(config_stats, thread_deallocated, + thread_allocated_tsd_get()->deallocated, uint64_t) +CTL_RO_NL_CGEN(config_stats, thread_deallocatedp, + &thread_allocated_tsd_get()->deallocated, uint64_t *) /******************************************************************************/ -#ifdef JEMALLOC_DEBUG -CTL_RO_TRUE_GEN(config_debug) -#else -CTL_RO_FALSE_GEN(config_debug) -#endif - -#ifdef JEMALLOC_DSS -CTL_RO_TRUE_GEN(config_dss) -#else -CTL_RO_FALSE_GEN(config_dss) -#endif - -#ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT -CTL_RO_TRUE_GEN(config_dynamic_page_shift) -#else -CTL_RO_FALSE_GEN(config_dynamic_page_shift) -#endif - -#ifdef JEMALLOC_FILL -CTL_RO_TRUE_GEN(config_fill) -#else -CTL_RO_FALSE_GEN(config_fill) -#endif - -#ifdef JEMALLOC_LAZY_LOCK -CTL_RO_TRUE_GEN(config_lazy_lock) -#else -CTL_RO_FALSE_GEN(config_lazy_lock) -#endif - -#ifdef JEMALLOC_PROF -CTL_RO_TRUE_GEN(config_prof) -#else -CTL_RO_FALSE_GEN(config_prof) -#endif - -#ifdef JEMALLOC_PROF_LIBGCC -CTL_RO_TRUE_GEN(config_prof_libgcc) -#else -CTL_RO_FALSE_GEN(config_prof_libgcc) -#endif - -#ifdef JEMALLOC_PROF_LIBUNWIND -CTL_RO_TRUE_GEN(config_prof_libunwind) -#else -CTL_RO_FALSE_GEN(config_prof_libunwind) -#endif - -#ifdef JEMALLOC_STATS -CTL_RO_TRUE_GEN(config_stats) -#else -CTL_RO_FALSE_GEN(config_stats) -#endif - -#ifdef JEMALLOC_SWAP -CTL_RO_TRUE_GEN(config_swap) -#else -CTL_RO_FALSE_GEN(config_swap) -#endif - -#ifdef JEMALLOC_SYSV -CTL_RO_TRUE_GEN(config_sysv) -#else -CTL_RO_FALSE_GEN(config_sysv) -#endif - -#ifdef JEMALLOC_TCACHE -CTL_RO_TRUE_GEN(config_tcache) -#else -CTL_RO_FALSE_GEN(config_tcache) -#endif - -#ifdef JEMALLOC_TINY -CTL_RO_TRUE_GEN(config_tiny) -#else -CTL_RO_FALSE_GEN(config_tiny) -#endif - -#ifdef JEMALLOC_TLS -CTL_RO_TRUE_GEN(config_tls) -#else -CTL_RO_FALSE_GEN(config_tls) -#endif - -#ifdef JEMALLOC_XMALLOC -CTL_RO_TRUE_GEN(config_xmalloc) -#else -CTL_RO_FALSE_GEN(config_xmalloc) -#endif +CTL_RO_BOOL_CONFIG_GEN(config_debug) +CTL_RO_BOOL_CONFIG_GEN(config_dss) +CTL_RO_BOOL_CONFIG_GEN(config_fill) +CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) +CTL_RO_BOOL_CONFIG_GEN(config_mremap) +CTL_RO_BOOL_CONFIG_GEN(config_munmap) +CTL_RO_BOOL_CONFIG_GEN(config_prof) +CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) +CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind) +CTL_RO_BOOL_CONFIG_GEN(config_stats) +CTL_RO_BOOL_CONFIG_GEN(config_tcache) +CTL_RO_BOOL_CONFIG_GEN(config_tls) +CTL_RO_BOOL_CONFIG_GEN(config_utrace) +CTL_RO_BOOL_CONFIG_GEN(config_valgrind) +CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) /******************************************************************************/ CTL_RO_NL_GEN(opt_abort, opt_abort, bool) -CTL_RO_NL_GEN(opt_lg_qspace_max, opt_lg_qspace_max, size_t) -CTL_RO_NL_GEN(opt_lg_cspace_max, opt_lg_cspace_max, size_t) CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) -#ifdef JEMALLOC_FILL -CTL_RO_NL_GEN(opt_junk, opt_junk, bool) -CTL_RO_NL_GEN(opt_zero, opt_zero, bool) -#endif -#ifdef JEMALLOC_SYSV -CTL_RO_NL_GEN(opt_sysv, opt_sysv, bool) -#endif -#ifdef JEMALLOC_XMALLOC -CTL_RO_NL_GEN(opt_xmalloc, opt_xmalloc, bool) -#endif -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) -CTL_RO_NL_GEN(opt_lg_tcache_gc_sweep, opt_lg_tcache_gc_sweep, ssize_t) -#endif -#ifdef JEMALLOC_PROF -CTL_RO_NL_GEN(opt_prof, opt_prof, bool) -CTL_RO_NL_GEN(opt_prof_prefix, opt_prof_prefix, const char *) -CTL_RO_GEN(opt_prof_active, opt_prof_active, bool) /* Mutable. */ -CTL_RO_NL_GEN(opt_lg_prof_bt_max, opt_lg_prof_bt_max, size_t) -CTL_RO_NL_GEN(opt_lg_prof_sample, opt_lg_prof_sample, size_t) -CTL_RO_NL_GEN(opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) -CTL_RO_NL_GEN(opt_prof_gdump, opt_prof_gdump, bool) -CTL_RO_NL_GEN(opt_prof_leak, opt_prof_leak, bool) -CTL_RO_NL_GEN(opt_prof_accum, opt_prof_accum, bool) -CTL_RO_NL_GEN(opt_lg_prof_tcmax, opt_lg_prof_tcmax, ssize_t) -#endif -#ifdef JEMALLOC_SWAP -CTL_RO_NL_GEN(opt_overcommit, opt_overcommit, bool) -#endif +CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool) +CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) +CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) +CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) +CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) +CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool) +CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) +CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) +CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) +CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) +CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */ +CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) +CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) +CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) /******************************************************************************/ CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) -const ctl_node_t * +const ctl_named_node_t * arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) { - if (i > nbins) + if (i > NBINS) return (NULL); return (super_arenas_bin_i_node); } -CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << PAGE_SHIFT), size_t) -const ctl_node_t * +CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) +const ctl_named_node_t * arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) { @@ -1350,37 +1205,16 @@ arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, for (i = 0; i < nread; i++) ((bool *)oldp)[i] = ctl_stats.arenas[i].initialized; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) -CTL_RO_NL_GEN(arenas_cacheline, CACHELINE, size_t) -CTL_RO_NL_GEN(arenas_subpage, SUBPAGE, size_t) -CTL_RO_NL_GEN(arenas_pagesize, PAGE_SIZE, size_t) -CTL_RO_NL_GEN(arenas_chunksize, chunksize, size_t) -#ifdef JEMALLOC_TINY -CTL_RO_NL_GEN(arenas_tspace_min, (1U << LG_TINY_MIN), size_t) -CTL_RO_NL_GEN(arenas_tspace_max, (qspace_min >> 1), size_t) -#endif -CTL_RO_NL_GEN(arenas_qspace_min, qspace_min, size_t) -CTL_RO_NL_GEN(arenas_qspace_max, qspace_max, size_t) -CTL_RO_NL_GEN(arenas_cspace_min, cspace_min, size_t) -CTL_RO_NL_GEN(arenas_cspace_max, cspace_max, size_t) -CTL_RO_NL_GEN(arenas_sspace_min, sspace_min, size_t) -CTL_RO_NL_GEN(arenas_sspace_max, sspace_max, size_t) -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -#endif -CTL_RO_NL_GEN(arenas_ntbins, ntbins, unsigned) -CTL_RO_NL_GEN(arenas_nqbins, nqbins, unsigned) -CTL_RO_NL_GEN(arenas_ncbins, ncbins, unsigned) -CTL_RO_NL_GEN(arenas_nsbins, nsbins, unsigned) -CTL_RO_NL_GEN(arenas_nbins, nbins, unsigned) -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) -#endif +CTL_RO_NL_GEN(arenas_page, PAGE, size_t) +CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) +CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) +CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned) CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) static int @@ -1395,9 +1229,9 @@ arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, WRITE(arena, unsigned); if (newp != NULL && arena >= narenas) { ret = EFAULT; - goto RETURN; + goto label_return; } else { - arena_t *tarenas[narenas]; + VARIABLE_ARRAY(arena_t *, tarenas, narenas); malloc_mutex_lock(&arenas_lock); memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); @@ -1417,13 +1251,12 @@ arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, } ret = 0; -RETURN: +label_return: return (ret); } /******************************************************************************/ -#ifdef JEMALLOC_PROF static int prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) @@ -1431,6 +1264,9 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; bool oldval; + if (config_prof == false) + return (ENOENT); + malloc_mutex_lock(&ctl_mtx); /* Protect opt_prof_active. */ oldval = opt_prof_active; if (newp != NULL) { @@ -1445,7 +1281,7 @@ prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, READ(oldval, bool); ret = 0; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } @@ -1457,92 +1293,88 @@ prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; const char *filename = NULL; + if (config_prof == false) + return (ENOENT); + WRITEONLY(); WRITE(filename, const char *); if (prof_mdump(filename)) { ret = EFAULT; - goto RETURN; + goto label_return; } ret = 0; -RETURN: +label_return: return (ret); } -CTL_RO_NL_GEN(prof_interval, prof_interval, uint64_t) -#endif +CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) /******************************************************************************/ -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_chunks_current, ctl_stats.chunks.current, size_t) -CTL_RO_GEN(stats_chunks_total, ctl_stats.chunks.total, uint64_t) -CTL_RO_GEN(stats_chunks_high, ctl_stats.chunks.high, size_t) -CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t) -CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t) -CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_allocated, +CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, + size_t) +CTL_RO_CGEN(config_stats, stats_chunks_total, ctl_stats.chunks.total, uint64_t) +CTL_RO_CGEN(config_stats, stats_chunks_high, ctl_stats.chunks.high, size_t) +CTL_RO_CGEN(config_stats, stats_huge_allocated, huge_allocated, size_t) +CTL_RO_CGEN(config_stats, stats_huge_nmalloc, huge_nmalloc, uint64_t) +CTL_RO_CGEN(config_stats, stats_huge_ndalloc, huge_ndalloc, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, ctl_stats.arenas[mib[2]].allocated_small, size_t) -CTL_RO_GEN(stats_arenas_i_small_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_small_ndalloc, ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_small_nrequests, ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_allocated, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, ctl_stats.arenas[mib[2]].astats.allocated_large, size_t) -CTL_RO_GEN(stats_arenas_i_large_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_allocated, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_allocated, ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nmalloc, ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_ndalloc, ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nrequests, ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) -#ifdef JEMALLOC_TCACHE -CTL_RO_GEN(stats_arenas_i_bins_j_nfills, +CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nfills, ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nflushes, +CTL_RO_CGEN(config_stats && config_tcache, stats_arenas_i_bins_j_nflushes, ctl_stats.arenas[mib[2]].bstats[mib[4]].nflushes, uint64_t) -#endif -CTL_RO_GEN(stats_arenas_i_bins_j_nruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].nruns, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nreruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].reruns, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_highruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].highruns, size_t) -CTL_RO_GEN(stats_arenas_i_bins_j_curruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) -const ctl_node_t * +const ctl_named_node_t * stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) { - if (j > nbins) + if (j > NBINS) return (NULL); return (super_stats_arenas_i_bins_j_node); } -CTL_RO_GEN(stats_arenas_i_lruns_j_nmalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nmalloc, ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_ndalloc, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_ndalloc, ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_nrequests, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nrequests, ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_curruns, +CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns, ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_highruns, - ctl_stats.arenas[mib[2]].lstats[mib[4]].highruns, size_t) -const ctl_node_t * +const ctl_named_node_t * stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) { @@ -1551,120 +1383,36 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) return (super_stats_arenas_i_lruns_j_node); } -#endif CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_arenas_i_mapped, ctl_stats.arenas[mib[2]].astats.mapped, - size_t) -CTL_RO_GEN(stats_arenas_i_npurge, ctl_stats.arenas[mib[2]].astats.npurge, - uint64_t) -CTL_RO_GEN(stats_arenas_i_nmadvise, ctl_stats.arenas[mib[2]].astats.nmadvise, - uint64_t) -CTL_RO_GEN(stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, - uint64_t) -#endif - -const ctl_node_t * +CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, + ctl_stats.arenas[mib[2]].astats.mapped, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_npurge, + ctl_stats.arenas[mib[2]].astats.npurge, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, + ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_purged, + ctl_stats.arenas[mib[2]].astats.purged, uint64_t) + +const ctl_named_node_t * stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) { - const ctl_node_t * ret; + const ctl_named_node_t * ret; malloc_mutex_lock(&ctl_mtx); if (ctl_stats.arenas[i].initialized == false) { ret = NULL; - goto RETURN; + goto label_return; } ret = super_stats_arenas_i_node; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_cactive, &stats_cactive, size_t *) -CTL_RO_GEN(stats_allocated, ctl_stats.allocated, size_t) -CTL_RO_GEN(stats_active, ctl_stats.active, size_t) -CTL_RO_GEN(stats_mapped, ctl_stats.mapped, size_t) -#endif - -/******************************************************************************/ - -#ifdef JEMALLOC_SWAP -# ifdef JEMALLOC_STATS -CTL_RO_GEN(swap_avail, ctl_stats.swap_avail, size_t) -# endif - -static int -swap_prezeroed_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - int ret; - - malloc_mutex_lock(&ctl_mtx); - if (swap_enabled) { - READONLY(); - } else { - /* - * swap_prezeroed isn't actually used by the swap code until it - * is set during a successful chunk_swap_enabled() call. We - * use it here to store the value that we'll pass to - * chunk_swap_enable() in a swap.fds mallctl(). This is not - * very clean, but the obvious alternatives are even worse. - */ - WRITE(swap_prezeroed, bool); - } - - READ(swap_prezeroed, bool); - - ret = 0; -RETURN: +label_return: malloc_mutex_unlock(&ctl_mtx); return (ret); } -CTL_RO_GEN(swap_nfds, swap_nfds, size_t) - -static int -swap_fds_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - - malloc_mutex_lock(&ctl_mtx); - if (swap_enabled) { - READONLY(); - } else if (newp != NULL) { - size_t nfds = newlen / sizeof(int); - - { - int fds[nfds]; - - memcpy(fds, newp, nfds * sizeof(int)); - if (chunk_swap_enable(fds, nfds, swap_prezeroed)) { - ret = EFAULT; - goto RETURN; - } - } - } - - if (oldp != NULL && oldlenp != NULL) { - if (*oldlenp != swap_nfds * sizeof(int)) { - size_t copylen = (swap_nfds * sizeof(int) <= *oldlenp) - ? swap_nfds * sizeof(int) : *oldlenp; - - memcpy(oldp, swap_fds, copylen); - ret = EINVAL; - goto RETURN; - } else - memcpy(oldp, swap_fds, *oldlenp); - } - - ret = 0; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} -#endif +CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) +CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) +CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) +CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) diff --git a/deps/jemalloc/src/extent.c b/deps/jemalloc/src/extent.c index 3c04d3aa5d1..8c09b486ed8 100644 --- a/deps/jemalloc/src/extent.c +++ b/deps/jemalloc/src/extent.c @@ -3,7 +3,6 @@ /******************************************************************************/ -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) static inline int extent_szad_comp(extent_node_t *a, extent_node_t *b) { @@ -25,7 +24,6 @@ extent_szad_comp(extent_node_t *a, extent_node_t *b) /* Generate red-black tree functions. */ rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad, extent_szad_comp) -#endif static inline int extent_ad_comp(extent_node_t *a, extent_node_t *b) diff --git a/deps/jemalloc/src/huge.c b/deps/jemalloc/src/huge.c index a4f9b054ed5..8a4ec942410 100644 --- a/deps/jemalloc/src/huge.c +++ b/deps/jemalloc/src/huge.c @@ -4,11 +4,9 @@ /******************************************************************************/ /* Data. */ -#ifdef JEMALLOC_STATS uint64_t huge_nmalloc; uint64_t huge_ndalloc; size_t huge_allocated; -#endif malloc_mutex_t huge_mtx; @@ -19,10 +17,18 @@ static extent_tree_t huge; void * huge_malloc(size_t size, bool zero) +{ + + return (huge_palloc(size, chunksize, zero)); +} + +void * +huge_palloc(size_t size, size_t alignment, bool zero) { void *ret; size_t csize; extent_node_t *node; + bool is_zeroed; /* Allocate one or more contiguous chunks for this request. */ @@ -37,7 +43,12 @@ huge_malloc(size_t size, bool zero) if (node == NULL) return (NULL); - ret = chunk_alloc(csize, false, &zero); + /* + * Copy zero into is_zeroed and pass the copy to chunk_alloc(), so that + * it is possible to make correct junk/zero fill decisions below. + */ + is_zeroed = zero; + ret = chunk_alloc(csize, alignment, false, &is_zeroed); if (ret == NULL) { base_node_dealloc(node); return (NULL); @@ -49,106 +60,19 @@ huge_malloc(size_t size, bool zero) malloc_mutex_lock(&huge_mtx); extent_tree_ad_insert(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_add(csize); - huge_nmalloc++; - huge_allocated += csize; -#endif + if (config_stats) { + stats_cactive_add(csize); + huge_nmalloc++; + huge_allocated += csize; + } malloc_mutex_unlock(&huge_mtx); -#ifdef JEMALLOC_FILL - if (zero == false) { + if (config_fill && zero == false) { if (opt_junk) memset(ret, 0xa5, csize); - else if (opt_zero) + else if (opt_zero && is_zeroed == false) memset(ret, 0, csize); } -#endif - - return (ret); -} - -/* Only handles large allocations that require more than chunk alignment. */ -void * -huge_palloc(size_t size, size_t alignment, bool zero) -{ - void *ret; - size_t alloc_size, chunk_size, offset; - extent_node_t *node; - - /* - * This allocation requires alignment that is even larger than chunk - * alignment. This means that huge_malloc() isn't good enough. - * - * Allocate almost twice as many chunks as are demanded by the size or - * alignment, in order to assure the alignment can be achieved, then - * unmap leading and trailing chunks. - */ - assert(alignment > chunksize); - - chunk_size = CHUNK_CEILING(size); - - if (size >= alignment) - alloc_size = chunk_size + alignment - chunksize; - else - alloc_size = (alignment << 1) - chunksize; - - /* Allocate an extent node with which to track the chunk. */ - node = base_node_alloc(); - if (node == NULL) - return (NULL); - - ret = chunk_alloc(alloc_size, false, &zero); - if (ret == NULL) { - base_node_dealloc(node); - return (NULL); - } - - offset = (uintptr_t)ret & (alignment - 1); - assert((offset & chunksize_mask) == 0); - assert(offset < alloc_size); - if (offset == 0) { - /* Trim trailing space. */ - chunk_dealloc((void *)((uintptr_t)ret + chunk_size), alloc_size - - chunk_size, true); - } else { - size_t trailsize; - - /* Trim leading space. */ - chunk_dealloc(ret, alignment - offset, true); - - ret = (void *)((uintptr_t)ret + (alignment - offset)); - - trailsize = alloc_size - (alignment - offset) - chunk_size; - if (trailsize != 0) { - /* Trim trailing space. */ - assert(trailsize < alloc_size); - chunk_dealloc((void *)((uintptr_t)ret + chunk_size), - trailsize, true); - } - } - - /* Insert node into huge. */ - node->addr = ret; - node->size = chunk_size; - - malloc_mutex_lock(&huge_mtx); - extent_tree_ad_insert(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_add(chunk_size); - huge_nmalloc++; - huge_allocated += chunk_size; -#endif - malloc_mutex_unlock(&huge_mtx); - -#ifdef JEMALLOC_FILL - if (zero == false) { - if (opt_junk) - memset(ret, 0xa5, chunk_size); - else if (opt_zero) - memset(ret, 0, chunk_size); - } -#endif return (ret); } @@ -164,12 +88,10 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { assert(CHUNK_CEILING(oldsize) == oldsize); -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { + if (config_fill && opt_junk && size < oldsize) { memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - size); } -#endif return (ptr); } @@ -218,20 +140,13 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, */ copysize = (size < oldsize) ? size : oldsize; +#ifdef JEMALLOC_MREMAP /* * Use mremap(2) if this is a huge-->huge reallocation, and neither the - * source nor the destination are in swap or dss. + * source nor the destination are in dss. */ -#ifdef JEMALLOC_MREMAP_FIXED - if (oldsize >= chunksize -# ifdef JEMALLOC_SWAP - && (swap_enabled == false || (chunk_in_swap(ptr) == false && - chunk_in_swap(ret) == false)) -# endif -# ifdef JEMALLOC_DSS - && chunk_in_dss(ptr) == false && chunk_in_dss(ret) == false -# endif - ) { + if (oldsize >= chunksize && (config_dss == false || (chunk_in_dss(ptr) + == false && chunk_in_dss(ret) == false))) { size_t newsize = huge_salloc(ret); /* @@ -253,10 +168,9 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, */ char buf[BUFERROR_BUF]; - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in mremap(): "); - malloc_write(buf); - malloc_write("\n"); + buferror(buf, sizeof(buf)); + malloc_printf(": Error in mremap(): %s\n", + buf); if (opt_abort) abort(); memcpy(ret, ptr, copysize); @@ -266,7 +180,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, #endif { memcpy(ret, ptr, copysize); - idalloc(ptr); + iqalloc(ptr); } return (ret); } @@ -285,23 +199,16 @@ huge_dalloc(void *ptr, bool unmap) assert(node->addr == ptr); extent_tree_ad_remove(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_sub(node->size); - huge_ndalloc++; - huge_allocated -= node->size; -#endif + if (config_stats) { + stats_cactive_sub(node->size); + huge_ndalloc++; + huge_allocated -= node->size; + } malloc_mutex_unlock(&huge_mtx); - if (unmap) { - /* Unmap chunk. */ -#ifdef JEMALLOC_FILL -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) - if (opt_junk) - memset(node->addr, 0x5a, node->size); -#endif -#endif - } + if (unmap && config_fill && config_dss && opt_junk) + memset(node->addr, 0x5a, node->size); chunk_dealloc(node->addr, node->size, unmap); @@ -328,7 +235,6 @@ huge_salloc(const void *ptr) return (ret); } -#ifdef JEMALLOC_PROF prof_ctx_t * huge_prof_ctx_get(const void *ptr) { @@ -365,7 +271,6 @@ huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) malloc_mutex_unlock(&huge_mtx); } -#endif bool huge_boot(void) @@ -376,11 +281,32 @@ huge_boot(void) return (true); extent_tree_ad_new(&huge); -#ifdef JEMALLOC_STATS - huge_nmalloc = 0; - huge_ndalloc = 0; - huge_allocated = 0; -#endif + if (config_stats) { + huge_nmalloc = 0; + huge_ndalloc = 0; + huge_allocated = 0; + } return (false); } + +void +huge_prefork(void) +{ + + malloc_mutex_prefork(&huge_mtx); +} + +void +huge_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&huge_mtx); +} + +void +huge_postfork_child(void) +{ + + malloc_mutex_postfork_child(&huge_mtx); +} diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c index a161c2e26e1..bc54cd7ca0d 100644 --- a/deps/jemalloc/src/jemalloc.c +++ b/deps/jemalloc/src/jemalloc.c @@ -4,111 +4,108 @@ /******************************************************************************/ /* Data. */ -malloc_mutex_t arenas_lock; -arena_t **arenas; -unsigned narenas; - -pthread_key_t arenas_tsd; -#ifndef NO_TLS -__thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); -#endif +malloc_tsd_data(, arenas, arena_t *, NULL) +malloc_tsd_data(, thread_allocated, thread_allocated_t, + THREAD_ALLOCATED_INITIALIZER) -#ifdef JEMALLOC_STATS -# ifndef NO_TLS -__thread thread_allocated_t thread_allocated_tls; +/* Runtime configuration options. */ +const char *je_malloc_conf; +#ifdef JEMALLOC_DEBUG +bool opt_abort = true; +# ifdef JEMALLOC_FILL +bool opt_junk = true; # else -pthread_key_t thread_allocated_tsd; +bool opt_junk = false; # endif +#else +bool opt_abort = false; +bool opt_junk = false; #endif +size_t opt_quarantine = ZU(0); +bool opt_redzone = false; +bool opt_utrace = false; +bool opt_valgrind = false; +bool opt_xmalloc = false; +bool opt_zero = false; +size_t opt_narenas = 0; + +unsigned ncpus; + +malloc_mutex_t arenas_lock; +arena_t **arenas; +unsigned narenas; /* Set to true once the allocator has been initialized. */ static bool malloc_initialized = false; +#ifdef JEMALLOC_THREADED_INIT /* Used to let the initializing thread recursively allocate. */ -static pthread_t malloc_initializer = (unsigned long)0; - -/* Used to avoid initialization races. */ -static malloc_mutex_t init_lock = -#ifdef JEMALLOC_OSSPIN - 0 +# define NO_INITIALIZER ((unsigned long)0) +# define INITIALIZER pthread_self() +# define IS_INITIALIZER (malloc_initializer == pthread_self()) +static pthread_t malloc_initializer = NO_INITIALIZER; #else - MALLOC_MUTEX_INITIALIZER +# define NO_INITIALIZER false +# define INITIALIZER true +# define IS_INITIALIZER malloc_initializer +static bool malloc_initializer = NO_INITIALIZER; #endif - ; -#ifdef DYNAMIC_PAGE_SHIFT -size_t pagesize; -size_t pagesize_mask; -size_t lg_pagesize; -#endif +/* Used to avoid initialization races. */ +#ifdef _WIN32 +static malloc_mutex_t init_lock; -unsigned ncpus; +JEMALLOC_ATTR(constructor) +static void WINAPI +_init_init_lock(void) +{ -/* Runtime configuration options. */ -const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default")); -#ifdef JEMALLOC_DEBUG -bool opt_abort = true; -# ifdef JEMALLOC_FILL -bool opt_junk = true; -# endif -#else -bool opt_abort = false; -# ifdef JEMALLOC_FILL -bool opt_junk = false; -# endif -#endif -#ifdef JEMALLOC_SYSV -bool opt_sysv = false; -#endif -#ifdef JEMALLOC_XMALLOC -bool opt_xmalloc = false; + malloc_mutex_init(&init_lock); +} + +#ifdef _MSC_VER +# pragma section(".CRT$XCU", read) +JEMALLOC_SECTION(".CRT$XCU") JEMALLOC_ATTR(used) +static const void (WINAPI *init_init_lock)(void) = _init_init_lock; #endif -#ifdef JEMALLOC_FILL -bool opt_zero = false; + +#else +static malloc_mutex_t init_lock = MALLOC_MUTEX_INITIALIZER; +#endif + +typedef struct { + void *p; /* Input pointer (as in realloc(p, s)). */ + size_t s; /* Request size. */ + void *r; /* Result pointer. */ +} malloc_utrace_t; + +#ifdef JEMALLOC_UTRACE +# define UTRACE(a, b, c) do { \ + if (opt_utrace) { \ + malloc_utrace_t ut; \ + ut.p = (a); \ + ut.s = (b); \ + ut.r = (c); \ + utrace(&ut, sizeof(ut)); \ + } \ +} while (0) +#else +# define UTRACE(a, b, c) #endif -size_t opt_narenas = 0; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static void wrtmessage(void *cbopaque, const char *s); static void stats_print_atexit(void); static unsigned malloc_ncpus(void); -static void arenas_cleanup(void *arg); -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -static void thread_allocated_cleanup(void *arg); -#endif static bool malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p); static void malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, size_t vlen); static void malloc_conf_init(void); static bool malloc_init_hard(void); -static int imemalign(void **memptr, size_t alignment, size_t size); - -/******************************************************************************/ -/* malloc_message() setup. */ - -#ifdef JEMALLOC_HAVE_ATTR -JEMALLOC_ATTR(visibility("hidden")) -#else -static -#endif -void -wrtmessage(void *cbopaque, const char *s) -{ -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - write(STDERR_FILENO, s, strlen(s)); -#ifdef JEMALLOC_CC_SILENCE - if (result < 0) - result = errno; -#endif -} - -void (*JEMALLOC_P(malloc_message))(void *, const char *s) - JEMALLOC_ATTR(visibility("default")) = wrtmessage; +static int imemalign(void **memptr, size_t alignment, size_t size, + size_t min_alignment); /******************************************************************************/ /* @@ -121,9 +118,7 @@ arenas_extend(unsigned ind) { arena_t *ret; - /* Allocate enough space for trailing bins. */ - ret = (arena_t *)base_alloc(offsetof(arena_t, bins) - + (sizeof(arena_bin_t) * nbins)); + ret = (arena_t *)base_alloc(sizeof(arena_t)); if (ret != NULL && arena_new(ret, ind) == false) { arenas[ind] = ret; return (ret); @@ -143,10 +138,7 @@ arenas_extend(unsigned ind) return (arenas[0]); } -/* - * Choose an arena based on a per-thread value (slow-path code only, called - * only by choose_arena()). - */ +/* Slow path, called only by choose_arena(). */ arena_t * choose_arena_hard(void) { @@ -182,7 +174,7 @@ choose_arena_hard(void) } } - if (arenas[choose] == 0 || first_null == narenas) { + if (arenas[choose]->nthreads == 0 || first_null == narenas) { /* * Use an unloaded arena, or the least loaded arena if * all arenas are already initialized. @@ -201,85 +193,46 @@ choose_arena_hard(void) malloc_mutex_unlock(&arenas_lock); } - ARENA_SET(ret); + arenas_tsd_set(&ret); return (ret); } -/* - * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so - * provide a wrapper. - */ -int -buferror(int errnum, char *buf, size_t buflen) -{ -#ifdef _GNU_SOURCE - char *b = strerror_r(errno, buf, buflen); - if (b != buf) { - strncpy(buf, b, buflen); - buf[buflen-1] = '\0'; - } - return (0); -#else - return (strerror_r(errno, buf, buflen)); -#endif -} - static void stats_print_atexit(void) { -#if (defined(JEMALLOC_TCACHE) && defined(JEMALLOC_STATS)) - unsigned i; + if (config_tcache && config_stats) { + unsigned i; - /* - * Merge stats from extant threads. This is racy, since individual - * threads do not lock when recording tcache stats events. As a - * consequence, the final stats may be slightly out of date by the time - * they are reported, if other threads continue to allocate. - */ - for (i = 0; i < narenas; i++) { - arena_t *arena = arenas[i]; - if (arena != NULL) { - tcache_t *tcache; + /* + * Merge stats from extant threads. This is racy, since + * individual threads do not lock when recording tcache stats + * events. As a consequence, the final stats may be slightly + * out of date by the time they are reported, if other threads + * continue to allocate. + */ + for (i = 0; i < narenas; i++) { + arena_t *arena = arenas[i]; + if (arena != NULL) { + tcache_t *tcache; - /* - * tcache_stats_merge() locks bins, so if any code is - * introduced that acquires both arena and bin locks in - * the opposite order, deadlocks may result. - */ - malloc_mutex_lock(&arena->lock); - ql_foreach(tcache, &arena->tcache_ql, link) { - tcache_stats_merge(tcache, arena); + /* + * tcache_stats_merge() locks bins, so if any + * code is introduced that acquires both arena + * and bin locks in the opposite order, + * deadlocks may result. + */ + malloc_mutex_lock(&arena->lock); + ql_foreach(tcache, &arena->tcache_ql, link) { + tcache_stats_merge(tcache, arena); + } + malloc_mutex_unlock(&arena->lock); } - malloc_mutex_unlock(&arena->lock); } } -#endif - JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL); -} - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -thread_allocated_t * -thread_allocated_get_hard(void) -{ - thread_allocated_t *thread_allocated = (thread_allocated_t *) - imalloc(sizeof(thread_allocated_t)); - if (thread_allocated == NULL) { - static thread_allocated_t static_thread_allocated = {0, 0}; - malloc_write(": Error allocating TSD;" - " mallctl(\"thread.{de,}allocated[p]\", ...)" - " will be inaccurate\n"); - if (opt_abort) - abort(); - return (&static_thread_allocated); - } - pthread_setspecific(thread_allocated_tsd, thread_allocated); - thread_allocated->allocated = 0; - thread_allocated->deallocated = 0; - return (thread_allocated); + je_malloc_stats_print(NULL, NULL, NULL); } -#endif /* * End miscellaneous support functions. @@ -295,42 +248,32 @@ malloc_ncpus(void) unsigned ret; long result; +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + result = si.dwNumberOfProcessors; +#else result = sysconf(_SC_NPROCESSORS_ONLN); if (result == -1) { /* Error. */ ret = 1; } +#endif ret = (unsigned)result; return (ret); } -static void +void arenas_cleanup(void *arg) { - arena_t *arena = (arena_t *)arg; + arena_t *arena = *(arena_t **)arg; malloc_mutex_lock(&arenas_lock); arena->nthreads--; malloc_mutex_unlock(&arenas_lock); } -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -static void -thread_allocated_cleanup(void *arg) -{ - uint64_t *allocated = (uint64_t *)arg; - - if (allocated != NULL) - idalloc(allocated); -} -#endif - -/* - * FreeBSD's pthreads implementation calls malloc(3), so the malloc - * implementation has to take pains to avoid infinite recursion during - * initialization. - */ static inline bool malloc_init(void) { @@ -352,68 +295,64 @@ malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, for (accept = false; accept == false;) { switch (*opts) { - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': case 'G': case 'H': case 'I': case 'J': - case 'K': case 'L': case 'M': case 'N': case 'O': - case 'P': case 'Q': case 'R': case 'S': case 'T': - case 'U': case 'V': case 'W': case 'X': case 'Y': - case 'Z': - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': case 'g': case 'h': case 'i': case 'j': - case 'k': case 'l': case 'm': case 'n': case 'o': - case 'p': case 'q': case 'r': case 's': case 't': - case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - case '_': - opts++; - break; - case ':': - opts++; - *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; - *v_p = opts; - accept = true; - break; - case '\0': - if (opts != *opts_p) { - malloc_write(": Conf string " - "ends with key\n"); - } - return (true); - default: - malloc_write(": Malformed conf " - "string\n"); - return (true); + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': + case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': + case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': + case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': + case 'Y': case 'Z': + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': + case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': + case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': + case 's': case 't': case 'u': case 'v': case 'w': case 'x': + case 'y': case 'z': + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': + case '_': + opts++; + break; + case ':': + opts++; + *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; + *v_p = opts; + accept = true; + break; + case '\0': + if (opts != *opts_p) { + malloc_write(": Conf string ends " + "with key\n"); + } + return (true); + default: + malloc_write(": Malformed conf string\n"); + return (true); } } for (accept = false; accept == false;) { switch (*opts) { - case ',': - opts++; - /* - * Look ahead one character here, because the - * next time this function is called, it will - * assume that end of input has been cleanly - * reached if no input remains, but we have - * optimistically already consumed the comma if - * one exists. - */ - if (*opts == '\0') { - malloc_write(": Conf string " - "ends with comma\n"); - } - *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; - accept = true; - break; - case '\0': - *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; - accept = true; - break; - default: - opts++; - break; + case ',': + opts++; + /* + * Look ahead one character here, because the next time + * this function is called, it will assume that end of + * input has been cleanly reached if no input remains, + * but we have optimistically already consumed the + * comma if one exists. + */ + if (*opts == '\0') { + malloc_write(": Conf string ends " + "with comma\n"); + } + *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; + accept = true; + break; + case '\0': + *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; + accept = true; + break; + default: + opts++; + break; } } @@ -425,17 +364,9 @@ static void malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, size_t vlen) { - char buf[PATH_MAX + 1]; - malloc_write(": "); - malloc_write(msg); - malloc_write(": "); - memcpy(buf, k, klen); - memcpy(&buf[klen], ":", 1); - memcpy(&buf[klen+1], v, vlen); - buf[klen+1+vlen] = '\0'; - malloc_write(buf); - malloc_write("\n"); + malloc_printf(": %s: %.*s:%.*s\n", msg, (int)klen, k, + (int)vlen, v); } static void @@ -450,12 +381,12 @@ malloc_conf_init(void) /* Get runtime configuration. */ switch (i) { case 0: - if (JEMALLOC_P(malloc_conf) != NULL) { + if (je_malloc_conf != NULL) { /* * Use options that were compiled into the * program. */ - opts = JEMALLOC_P(malloc_conf); + opts = je_malloc_conf; } else { /* No configuration specified. */ buf[0] = '\0'; @@ -463,13 +394,14 @@ malloc_conf_init(void) } break; case 1: { +#ifndef _WIN32 int linklen; const char *linkname = -#ifdef JEMALLOC_PREFIX +# ifdef JEMALLOC_PREFIX "/etc/"JEMALLOC_PREFIX"malloc.conf" -#else +# else "/etc/malloc.conf" -#endif +# endif ; if ((linklen = readlink(linkname, buf, @@ -480,14 +412,15 @@ malloc_conf_init(void) */ buf[linklen] = '\0'; opts = buf; - } else { + } else +#endif + { /* No configuration specified. */ buf[0] = '\0'; opts = buf; } break; - } - case 2: { + } case 2: { const char *envname = #ifdef JEMALLOC_PREFIX JEMALLOC_CPREFIX"MALLOC_CONF" @@ -508,8 +441,7 @@ malloc_conf_init(void) opts = buf; } break; - } - default: + } default: /* NOTREACHED */ assert(false); buf[0] = '\0'; @@ -518,52 +450,59 @@ malloc_conf_init(void) while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, &vlen) == false) { -#define CONF_HANDLE_BOOL(n) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_BOOL_HIT(o, n, hit) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ if (strncmp("true", v, vlen) == 0 && \ vlen == sizeof("true")-1) \ - opt_##n = true; \ + o = true; \ else if (strncmp("false", v, vlen) == \ 0 && vlen == sizeof("false")-1) \ - opt_##n = false; \ + o = false; \ else { \ malloc_conf_error( \ "Invalid conf value", \ k, klen, v, vlen); \ } \ + hit = true; \ + } else \ + hit = false; +#define CONF_HANDLE_BOOL(o, n) { \ + bool hit; \ + CONF_HANDLE_BOOL_HIT(o, n, hit); \ + if (hit) \ continue; \ - } -#define CONF_HANDLE_SIZE_T(n, min, max) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +} +#define CONF_HANDLE_SIZE_T(o, n, min, max) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ - unsigned long ul; \ + uintmax_t um; \ char *end; \ \ - errno = 0; \ - ul = strtoul(v, &end, 0); \ - if (errno != 0 || (uintptr_t)end - \ + set_errno(0); \ + um = malloc_strtoumax(v, &end, 0); \ + if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ malloc_conf_error( \ "Invalid conf value", \ k, klen, v, vlen); \ - } else if (ul < min || ul > max) { \ + } else if (um < min || um > max) { \ malloc_conf_error( \ "Out-of-range conf value", \ k, klen, v, vlen); \ } else \ - opt_##n = ul; \ + o = um; \ continue; \ } -#define CONF_HANDLE_SSIZE_T(n, min, max) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_SSIZE_T(o, n, min, max) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ long l; \ char *end; \ \ - errno = 0; \ + set_errno(0); \ l = strtol(v, &end, 0); \ - if (errno != 0 || (uintptr_t)end - \ + if (get_errno() != 0 || (uintptr_t)end -\ (uintptr_t)v != vlen) { \ malloc_conf_error( \ "Invalid conf value", \ @@ -574,70 +513,86 @@ malloc_conf_init(void) "Out-of-range conf value", \ k, klen, v, vlen); \ } else \ - opt_##n = l; \ + o = l; \ continue; \ } -#define CONF_HANDLE_CHAR_P(n, d) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ +#define CONF_HANDLE_CHAR_P(o, n, d) \ + if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ size_t cpylen = (vlen <= \ - sizeof(opt_##n)-1) ? vlen : \ - sizeof(opt_##n)-1; \ - strncpy(opt_##n, v, cpylen); \ - opt_##n[cpylen] = '\0'; \ + sizeof(o)-1) ? vlen : \ + sizeof(o)-1; \ + strncpy(o, v, cpylen); \ + o[cpylen] = '\0'; \ continue; \ } - CONF_HANDLE_BOOL(abort) - CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM, - PAGE_SHIFT-1) - CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM, - PAGE_SHIFT-1) + CONF_HANDLE_BOOL(opt_abort, "abort") /* - * Chunks always require at least one * header page, - * plus one data page. + * Chunks always require at least one header page, plus + * one data page in the absence of redzones, or three + * pages in the presence of redzones. In order to + * simplify options processing, fix the limit based on + * config_fill. */ - CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX) - CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_BOOL(stats_print) -#ifdef JEMALLOC_FILL - CONF_HANDLE_BOOL(junk) - CONF_HANDLE_BOOL(zero) -#endif -#ifdef JEMALLOC_SYSV - CONF_HANDLE_BOOL(sysv) -#endif -#ifdef JEMALLOC_XMALLOC - CONF_HANDLE_BOOL(xmalloc) -#endif -#ifdef JEMALLOC_TCACHE - CONF_HANDLE_BOOL(tcache) - CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SSIZE_T(lg_tcache_max, -1, - (sizeof(size_t) << 3) - 1) -#endif -#ifdef JEMALLOC_PROF - CONF_HANDLE_BOOL(prof) - CONF_HANDLE_CHAR_P(prof_prefix, "jeprof") - CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0, LG_PROF_BT_MAX) - CONF_HANDLE_BOOL(prof_active) - CONF_HANDLE_SSIZE_T(lg_prof_sample, 0, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(prof_accum) - CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SSIZE_T(lg_prof_interval, -1, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(prof_gdump) - CONF_HANDLE_BOOL(prof_leak) -#endif -#ifdef JEMALLOC_SWAP - CONF_HANDLE_BOOL(overcommit) -#endif + CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + + (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1) + CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, + SIZE_T_MAX) + CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", + -1, (sizeof(size_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_stats_print, "stats_print") + if (config_fill) { + CONF_HANDLE_BOOL(opt_junk, "junk") + CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", + 0, SIZE_T_MAX) + CONF_HANDLE_BOOL(opt_redzone, "redzone") + CONF_HANDLE_BOOL(opt_zero, "zero") + } + if (config_utrace) { + CONF_HANDLE_BOOL(opt_utrace, "utrace") + } + if (config_valgrind) { + bool hit; + CONF_HANDLE_BOOL_HIT(opt_valgrind, + "valgrind", hit) + if (config_fill && opt_valgrind && hit) { + opt_junk = false; + opt_zero = false; + if (opt_quarantine == 0) { + opt_quarantine = + JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; + } + opt_redzone = true; + } + if (hit) + continue; + } + if (config_xmalloc) { + CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") + } + if (config_tcache) { + CONF_HANDLE_BOOL(opt_tcache, "tcache") + CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, + "lg_tcache_max", -1, + (sizeof(size_t) << 3) - 1) + } + if (config_prof) { + CONF_HANDLE_BOOL(opt_prof, "prof") + CONF_HANDLE_CHAR_P(opt_prof_prefix, + "prof_prefix", "jeprof") + CONF_HANDLE_BOOL(opt_prof_active, "prof_active") + CONF_HANDLE_SSIZE_T(opt_lg_prof_sample, + "lg_prof_sample", 0, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_prof_accum, "prof_accum") + CONF_HANDLE_SSIZE_T(opt_lg_prof_interval, + "lg_prof_interval", -1, + (sizeof(uint64_t) << 3) - 1) + CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") + CONF_HANDLE_BOOL(opt_prof_final, "prof_final") + CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + } malloc_conf_error("Invalid conf pair", k, klen, v, vlen); #undef CONF_HANDLE_BOOL @@ -645,14 +600,6 @@ malloc_conf_init(void) #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P } - - /* Validate configuration of options that are inter-related. */ - if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) { - malloc_write(": Invalid lg_[qc]space_max " - "relationship; restoring defaults\n"); - opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; - opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; - } } } @@ -662,7 +609,7 @@ malloc_init_hard(void) arena_t *init_arenas[1]; malloc_mutex_lock(&init_lock); - if (malloc_initialized || malloc_initializer == pthread_self()) { + if (malloc_initialized || IS_INITIALIZER) { /* * Another thread initialized the allocator before this one * acquired init_lock, or this thread is the initializing @@ -671,7 +618,8 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); return (false); } - if (malloc_initializer != (unsigned long)0) { +#ifdef JEMALLOC_THREADED_INIT + if (malloc_initializer != NO_INITIALIZER && IS_INITIALIZER == false) { /* Busy-wait until the initializing thread completes. */ do { malloc_mutex_unlock(&init_lock); @@ -681,44 +629,25 @@ malloc_init_hard(void) malloc_mutex_unlock(&init_lock); return (false); } - -#ifdef DYNAMIC_PAGE_SHIFT - /* Get page size. */ - { - long result; - - result = sysconf(_SC_PAGESIZE); - assert(result != -1); - pagesize = (size_t)result; - - /* - * We assume that pagesize is a power of 2 when calculating - * pagesize_mask and lg_pagesize. - */ - assert(((result - 1) & result) == 0); - pagesize_mask = result - 1; - lg_pagesize = ffs((int)result) - 1; - } #endif + malloc_initializer = INITIALIZER; -#ifdef JEMALLOC_PROF - prof_boot0(); -#endif + malloc_tsd_boot(); + if (config_prof) + prof_boot0(); malloc_conf_init(); +#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ + && !defined(_WIN32)) /* Register fork handlers. */ - if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, - jemalloc_postfork) != 0) { + if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, + jemalloc_postfork_child) != 0) { malloc_write(": Error in pthread_atfork()\n"); if (opt_abort) abort(); } - - if (ctl_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } +#endif if (opt_stats_print) { /* Print statistics at exit. */ @@ -729,54 +658,39 @@ malloc_init_hard(void) } } - if (chunk_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - if (base_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#ifdef JEMALLOC_PROF - prof_boot1(); -#endif - - if (arena_boot()) { + if (chunk_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#ifdef JEMALLOC_TCACHE - if (tcache_boot()) { + if (ctl_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#endif - if (huge_boot()) { + if (config_prof) + prof_boot1(); + + arena_boot(); + + if (config_tcache && tcache_boot0()) { malloc_mutex_unlock(&init_lock); return (true); } -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) - /* Initialize allocation counters before any allocations can occur. */ - if (pthread_key_create(&thread_allocated_tsd, thread_allocated_cleanup) - != 0) { + if (huge_boot()) { malloc_mutex_unlock(&init_lock); return (true); } -#endif if (malloc_mutex_init(&arenas_lock)) return (true); - if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) { - malloc_mutex_unlock(&init_lock); - return (true); - } - /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). @@ -795,27 +709,42 @@ malloc_init_hard(void) return (true); } - /* - * Assign the initial arena to the initial thread, in order to avoid - * spurious creation of an extra arena if the application switches to - * threaded mode. - */ - ARENA_SET(arenas[0]); - arenas[0]->nthreads++; + /* Initialize allocation counters before any allocations can occur. */ + if (config_stats && thread_allocated_tsd_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } -#ifdef JEMALLOC_PROF - if (prof_boot2()) { + if (arenas_tsd_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (config_tcache && tcache_boot1()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (config_fill && quarantine_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + + if (config_prof && prof_boot2()) { malloc_mutex_unlock(&init_lock); return (true); } -#endif /* Get number of CPUs. */ - malloc_initializer = pthread_self(); malloc_mutex_unlock(&init_lock); ncpus = malloc_ncpus(); malloc_mutex_lock(&init_lock); + if (mutex_boot()) { + malloc_mutex_unlock(&init_lock); + return (true); + } + if (opt_narenas == 0) { /* * For SMP systems, create more than one arena per CPU by @@ -833,12 +762,9 @@ malloc_init_hard(void) * machinery will fail to allocate memory at far lower limits. */ if (narenas > chunksize / sizeof(arena_t *)) { - char buf[UMAX2S_BUFSIZE]; - narenas = chunksize / sizeof(arena_t *); - malloc_write(": Reducing narenas to limit ("); - malloc_write(u2s(narenas, 10, buf)); - malloc_write(")\n"); + malloc_printf(": Reducing narenas to limit (%d)\n", + narenas); } /* Allocate and initialize arenas. */ @@ -855,34 +781,11 @@ malloc_init_hard(void) /* Copy the pointer to the one arena that was already initialized. */ arenas[0] = init_arenas[0]; -#ifdef JEMALLOC_ZONE - /* Register the custom zone. */ - malloc_zone_register(create_zone()); - - /* - * Convert the default szone to an "overlay zone" that is capable of - * deallocating szone-allocated objects, but allocating new objects - * from jemalloc. - */ - szone2ozone(malloc_default_zone()); -#endif - malloc_initialized = true; malloc_mutex_unlock(&init_lock); return (false); } -#ifdef JEMALLOC_ZONE -JEMALLOC_ATTR(constructor) -void -jemalloc_darwin_init(void) -{ - - if (malloc_init_hard()) - abort(); -} -#endif - /* * End initialization functions. */ @@ -891,191 +794,118 @@ jemalloc_darwin_init(void) * Begin malloc(3)-compatible functions. */ -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(malloc)(size_t size) +je_malloc(size_t size) { void *ret; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + size_t usize JEMALLOC_CC_SILENCE_INIT(0); + prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); if (malloc_init()) { ret = NULL; - goto OOM; + goto label_oom; } - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { -# ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in malloc(): " - "invalid size 0\n"); - abort(); - } -# endif - ret = NULL; - goto RETURN; - } -#endif - } + if (size == 0) + size = 1; -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { usize = s2u(size); PROF_ALLOC_PREP(1, usize, cnt); if (cnt == NULL) { ret = NULL; - goto OOM; + goto label_oom; } if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { - ret = imalloc(small_maxclass+1); + SMALL_MAXCLASS) { + ret = imalloc(SMALL_MAXCLASS+1); if (ret != NULL) arena_prof_promoted(ret, usize); } else ret = imalloc(size); - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif + } else { + if (config_stats || (config_valgrind && opt_valgrind)) + usize = s2u(size); ret = imalloc(size); } -OOM: +label_oom: if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in malloc(): " "out of memory\n"); abort(); } -#endif - errno = ENOMEM; + set_errno(ENOMEM); } - -#ifdef JEMALLOC_SYSV -RETURN: -#endif -#ifdef JEMALLOC_PROF - if (opt_prof && ret != NULL) + if (config_prof && opt_prof && ret != NULL) prof_malloc(ret, usize, cnt); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, 0); + if (config_stats && ret != NULL) { + assert(usize == isalloc(ret, config_prof)); + thread_allocated_tsd_get()->allocated += usize; } -#endif + UTRACE(0, size, ret); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, false); return (ret); } JEMALLOC_ATTR(nonnull(1)) #ifdef JEMALLOC_PROF /* - * Avoid any uncertainty as to how many backtrace frames to ignore in + * Avoid any uncertainty as to how many backtrace frames to ignore in * PROF_ALLOC_PREP(). */ JEMALLOC_ATTR(noinline) #endif static int -imemalign(void **memptr, size_t alignment, size_t size) +imemalign(void **memptr, size_t alignment, size_t size, + size_t min_alignment) { int ret; - size_t usize -#ifdef JEMALLOC_CC_SILENCE - = 0 -#endif - ; + size_t usize; void *result; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); + + assert(min_alignment != 0); if (malloc_init()) result = NULL; else { - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { -# ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in " - "posix_memalign(): invalid size " - "0\n"); - abort(); - } -# endif - result = NULL; - *memptr = NULL; - ret = 0; - goto RETURN; - } -#endif - } + if (size == 0) + size = 1; /* Make sure that alignment is a large enough power of 2. */ if (((alignment - 1) & alignment) != 0 - || alignment < sizeof(void *)) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in " - "posix_memalign(): invalid alignment\n"); + || (alignment < min_alignment)) { + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error allocating " + "aligned memory: invalid alignment\n"); abort(); } -#endif result = NULL; ret = EINVAL; - goto RETURN; + goto label_return; } - usize = sa2u(size, alignment, NULL); + usize = sa2u(size, alignment); if (usize == 0) { result = NULL; ret = ENOMEM; - goto RETURN; + goto label_return; } -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { PROF_ALLOC_PREP(2, usize, cnt); if (cnt == NULL) { result = NULL; ret = EINVAL; } else { if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= small_maxclass) { - assert(sa2u(small_maxclass+1, - alignment, NULL) != 0); - result = ipalloc(sa2u(small_maxclass+1, - alignment, NULL), alignment, false); + (uintptr_t)1U && usize <= SMALL_MAXCLASS) { + assert(sa2u(SMALL_MAXCLASS+1, + alignment) != 0); + result = ipalloc(sa2u(SMALL_MAXCLASS+1, + alignment), alignment, false); if (result != NULL) { arena_prof_promoted(result, usize); @@ -1086,88 +916,79 @@ imemalign(void **memptr, size_t alignment, size_t size) } } } else -#endif result = ipalloc(usize, alignment, false); } if (result == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in posix_memalign(): " - "out of memory\n"); + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error allocating aligned " + "memory: out of memory\n"); abort(); } -#endif ret = ENOMEM; - goto RETURN; + goto label_return; } *memptr = result; ret = 0; -RETURN: -#ifdef JEMALLOC_STATS - if (result != NULL) { - assert(usize == isalloc(result)); - ALLOCATED_ADD(usize, 0); +label_return: + if (config_stats && result != NULL) { + assert(usize == isalloc(result, config_prof)); + thread_allocated_tsd_get()->allocated += usize; } -#endif -#ifdef JEMALLOC_PROF - if (opt_prof && result != NULL) + if (config_prof && opt_prof && result != NULL) prof_malloc(result, usize, cnt); -#endif + UTRACE(0, size, result); return (ret); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) +je_posix_memalign(void **memptr, size_t alignment, size_t size) +{ + int ret = imemalign(memptr, alignment, size, sizeof(void *)); + JEMALLOC_VALGRIND_MALLOC(ret == 0, *memptr, isalloc(*memptr, + config_prof), false); + return (ret); +} + +void * +je_aligned_alloc(size_t alignment, size_t size) { + void *ret; + int err; - return imemalign(memptr, alignment, size); + if ((err = imemalign(&ret, alignment, size, 1)) != 0) { + ret = NULL; + set_errno(err); + } + JEMALLOC_VALGRIND_MALLOC(err == 0, ret, isalloc(ret, config_prof), + false); + return (ret); } -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(calloc)(size_t num, size_t size) +je_calloc(size_t num, size_t size) { void *ret; size_t num_size; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + size_t usize JEMALLOC_CC_SILENCE_INIT(0); + prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); if (malloc_init()) { num_size = 0; ret = NULL; - goto RETURN; + goto label_return; } num_size = num * size; if (num_size == 0) { -#ifdef JEMALLOC_SYSV - if ((opt_sysv == false) && ((num == 0) || (size == 0))) -#endif + if (num == 0 || size == 0) num_size = 1; -#ifdef JEMALLOC_SYSV else { ret = NULL; - goto RETURN; + goto label_return; } -#endif /* * Try to avoid division here. We know that it isn't possible to * overflow during multiplication if neither operand uses any of the @@ -1177,135 +998,113 @@ JEMALLOC_P(calloc)(size_t num, size_t size) && (num_size / size != num)) { /* size_t overflow. */ ret = NULL; - goto RETURN; + goto label_return; } -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { usize = s2u(num_size); PROF_ALLOC_PREP(1, usize, cnt); if (cnt == NULL) { ret = NULL; - goto RETURN; + goto label_return; } if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize - <= small_maxclass) { - ret = icalloc(small_maxclass+1); + <= SMALL_MAXCLASS) { + ret = icalloc(SMALL_MAXCLASS+1); if (ret != NULL) arena_prof_promoted(ret, usize); } else ret = icalloc(num_size); - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(num_size); -#endif + } else { + if (config_stats || (config_valgrind && opt_valgrind)) + usize = s2u(num_size); ret = icalloc(num_size); } -RETURN: +label_return: if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in calloc(): out of " "memory\n"); abort(); } -#endif - errno = ENOMEM; + set_errno(ENOMEM); } -#ifdef JEMALLOC_PROF - if (opt_prof && ret != NULL) + if (config_prof && opt_prof && ret != NULL) prof_malloc(ret, usize, cnt); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, 0); + if (config_stats && ret != NULL) { + assert(usize == isalloc(ret, config_prof)); + thread_allocated_tsd_get()->allocated += usize; } -#endif + UTRACE(0, num_size, ret); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, usize, true); return (ret); } -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(realloc)(void *ptr, size_t size) +je_realloc(void *ptr, size_t size) { void *ret; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; + size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_size = 0; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; - prof_ctx_t *old_ctx -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif + size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); + prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); + prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { - if (ptr != NULL) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - old_size = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { - old_ctx = prof_ctx_get(ptr); - cnt = NULL; - } -#endif - idalloc(ptr); + if (ptr != NULL) { + /* realloc(ptr, 0) is equivalent to free(p). */ + if (config_prof) { + old_size = isalloc(ptr, true); + if (config_valgrind && opt_valgrind) + old_rzsize = p2rz(ptr); + } else if (config_stats) { + old_size = isalloc(ptr, false); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_size); + } else if (config_valgrind && opt_valgrind) { + old_size = isalloc(ptr, false); + old_rzsize = u2rz(old_size); } -#ifdef JEMALLOC_PROF - else if (opt_prof) { - old_ctx = NULL; + if (config_prof && opt_prof) { + old_ctx = prof_ctx_get(ptr); cnt = NULL; } -#endif + iqalloc(ptr); ret = NULL; - goto RETURN; - } -#endif + goto label_return; + } else + size = 1; } if (ptr != NULL) { - assert(malloc_initialized || malloc_initializer == - pthread_self()); - -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - old_size = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { + assert(malloc_initialized || IS_INITIALIZER); + + if (config_prof) { + old_size = isalloc(ptr, true); + if (config_valgrind && opt_valgrind) + old_rzsize = p2rz(ptr); + } else if (config_stats) { + old_size = isalloc(ptr, false); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_size); + } else if (config_valgrind && opt_valgrind) { + old_size = isalloc(ptr, false); + old_rzsize = u2rz(old_size); + } + if (config_prof && opt_prof) { usize = s2u(size); old_ctx = prof_ctx_get(ptr); PROF_ALLOC_PREP(1, usize, cnt); if (cnt == NULL) { old_ctx = NULL; ret = NULL; - goto OOM; + goto label_oom; } if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && - usize <= small_maxclass) { - ret = iralloc(ptr, small_maxclass+1, 0, 0, + usize <= SMALL_MAXCLASS) { + ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0, false, false); if (ret != NULL) arena_prof_promoted(ret, usize); @@ -1316,42 +1115,31 @@ JEMALLOC_P(realloc)(void *ptr, size_t size) if (ret == NULL) old_ctx = NULL; } - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif + } else { + if (config_stats || (config_valgrind && opt_valgrind)) + usize = s2u(size); ret = iralloc(ptr, size, 0, 0, false, false); } -#ifdef JEMALLOC_PROF -OOM: -#endif +label_oom: if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in realloc(): " "out of memory\n"); abort(); } -#endif - errno = ENOMEM; + set_errno(ENOMEM); } } else { -#ifdef JEMALLOC_PROF - if (opt_prof) + /* realloc(NULL, size) is equivalent to malloc(size). */ + if (config_prof && opt_prof) old_ctx = NULL; -#endif if (malloc_init()) { -#ifdef JEMALLOC_PROF - if (opt_prof) + if (config_prof && opt_prof) cnt = NULL; -#endif ret = NULL; } else { -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { usize = s2u(size); PROF_ALLOC_PREP(1, usize, cnt); if (cnt == NULL) @@ -1359,8 +1147,8 @@ JEMALLOC_P(realloc)(void *ptr, size_t size) else { if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { - ret = imalloc(small_maxclass+1); + SMALL_MAXCLASS) { + ret = imalloc(SMALL_MAXCLASS+1); if (ret != NULL) { arena_prof_promoted(ret, usize); @@ -1368,72 +1156,61 @@ JEMALLOC_P(realloc)(void *ptr, size_t size) } else ret = imalloc(size); } - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif + } else { + if (config_stats || (config_valgrind && + opt_valgrind)) + usize = s2u(size); ret = imalloc(size); } } if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in realloc(): " "out of memory\n"); abort(); } -#endif - errno = ENOMEM; + set_errno(ENOMEM); } } -#ifdef JEMALLOC_SYSV -RETURN: -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) +label_return: + if (config_prof && opt_prof) prof_realloc(ret, usize, cnt, old_size, old_ctx); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, old_size); + if (config_stats && ret != NULL) { + thread_allocated_t *ta; + assert(usize == isalloc(ret, config_prof)); + ta = thread_allocated_tsd_get(); + ta->allocated += usize; + ta->deallocated += old_size; } -#endif + UTRACE(ptr, size, ret); + JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); return (ret); } -JEMALLOC_ATTR(visibility("default")) void -JEMALLOC_P(free)(void *ptr) +je_free(void *ptr) { + UTRACE(ptr, 0, 0); if (ptr != NULL) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) size_t usize; -#endif + size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - assert(malloc_initialized || malloc_initializer == - pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); -#ifdef JEMALLOC_STATS - usize = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { -# ifndef JEMALLOC_STATS - usize = isalloc(ptr); -# endif + if (config_prof && opt_prof) { + usize = isalloc(ptr, config_prof); prof_free(ptr, usize); - } -#endif -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(0, usize); -#endif - idalloc(ptr); + } else if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); + if (config_stats) + thread_allocated_tsd_get()->deallocated += usize; + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloc(ptr); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); } } @@ -1443,51 +1220,56 @@ JEMALLOC_P(free)(void *ptr) /******************************************************************************/ /* * Begin non-standard override functions. - * - * These overrides are omitted if the JEMALLOC_PREFIX is defined, since the - * entire point is to avoid accidental mixed allocator usage. */ -#ifndef JEMALLOC_PREFIX #ifdef JEMALLOC_OVERRIDE_MEMALIGN -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(memalign)(size_t alignment, size_t size) +je_memalign(size_t alignment, size_t size) { - void *ret; -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - imemalign(&ret, alignment, size); -#ifdef JEMALLOC_CC_SILENCE - if (result != 0) - return (NULL); -#endif + void *ret JEMALLOC_CC_SILENCE_INIT(NULL); + imemalign(&ret, alignment, size, 1); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } #endif #ifdef JEMALLOC_OVERRIDE_VALLOC -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) void * -JEMALLOC_P(valloc)(size_t size) +je_valloc(size_t size) { - void *ret; -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - imemalign(&ret, PAGE_SIZE, size); -#ifdef JEMALLOC_CC_SILENCE - if (result != 0) - return (NULL); -#endif + void *ret JEMALLOC_CC_SILENCE_INIT(NULL); + imemalign(&ret, PAGE, size, 1); + JEMALLOC_VALGRIND_MALLOC(ret != NULL, ret, size, false); return (ret); } #endif -#endif /* JEMALLOC_PREFIX */ +/* + * is_malloc(je_malloc) is some macro magic to detect if jemalloc_defs.h has + * #define je_malloc malloc + */ +#define malloc_is_malloc 1 +#define is_malloc_(a) malloc_is_ ## a +#define is_malloc(a) is_malloc_(a) + +#if ((is_malloc(je_malloc) == 1) && defined(__GLIBC__) && !defined(__UCLIBC__)) +/* + * glibc provides the RTLD_DEEPBIND flag for dlopen which can make it possible + * to inconsistently reference libc's malloc(3)-compatible functions + * (https://bugzilla.mozilla.org/show_bug.cgi?id=493541). + * + * These definitions interpose hooks in glibc. The functions are actually + * passed an extra argument for the caller return address, which will be + * ignored. + */ +JEMALLOC_EXPORT void (* const __free_hook)(void *ptr) = je_free; +JEMALLOC_EXPORT void *(* const __malloc_hook)(size_t size) = je_malloc; +JEMALLOC_EXPORT void *(* const __realloc_hook)(void *ptr, size_t size) = + je_realloc; +JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) = + je_memalign; +#endif + /* * End non-standard override functions. */ @@ -1496,36 +1278,31 @@ JEMALLOC_P(valloc)(size_t size) * Begin non-standard functions. */ -JEMALLOC_ATTR(visibility("default")) size_t -JEMALLOC_P(malloc_usable_size)(const void *ptr) +je_malloc_usable_size(const void *ptr) { size_t ret; - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); -#ifdef JEMALLOC_IVSALLOC - ret = ivsalloc(ptr); -#else - assert(ptr != NULL); - ret = isalloc(ptr); -#endif + if (config_ivsalloc) + ret = ivsalloc(ptr, config_prof); + else + ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; return (ret); } -JEMALLOC_ATTR(visibility("default")) void -JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), - void *cbopaque, const char *opts) +je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, + const char *opts) { stats_print(write_cb, cbopaque, opts); } -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, +je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { @@ -1535,9 +1312,8 @@ JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, return (ctl_byname(name, oldp, oldlenp, newp, newlen)); } -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) +je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { if (malloc_init()) @@ -1546,10 +1322,9 @@ JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) return (ctl_nametomib(name, mibp, miblenp)); } -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) +je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { if (malloc_init()) @@ -1558,12 +1333,21 @@ JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); } +/* + * End non-standard functions. + */ +/******************************************************************************/ +/* + * Begin experimental functions. + */ +#ifdef JEMALLOC_EXPERIMENTAL + JEMALLOC_INLINE void * iallocm(size_t usize, size_t alignment, bool zero) { - assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment, - NULL))); + assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, + alignment))); if (alignment != 0) return (ipalloc(usize, alignment, zero)); @@ -1573,116 +1357,96 @@ iallocm(size_t usize, size_t alignment, bool zero) return (imalloc(usize)); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) +je_allocm(void **ptr, size_t *rsize, size_t size, int flags) { void *p; size_t usize; size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); bool zero = flags & ALLOCM_ZERO; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt; -#endif assert(ptr != NULL); assert(size != 0); if (malloc_init()) - goto OOM; + goto label_oom; - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL); + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); if (usize == 0) - goto OOM; + goto label_oom; + + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; -#ifdef JEMALLOC_PROF - if (opt_prof) { PROF_ALLOC_PREP(1, usize, cnt); if (cnt == NULL) - goto OOM; + goto label_oom; if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { + SMALL_MAXCLASS) { size_t usize_promoted = (alignment == 0) ? - s2u(small_maxclass+1) : sa2u(small_maxclass+1, - alignment, NULL); + s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, + alignment); assert(usize_promoted != 0); p = iallocm(usize_promoted, alignment, zero); if (p == NULL) - goto OOM; + goto label_oom; arena_prof_promoted(p, usize); } else { p = iallocm(usize, alignment, zero); if (p == NULL) - goto OOM; + goto label_oom; } prof_malloc(p, usize, cnt); - if (rsize != NULL) - *rsize = usize; - } else -#endif - { + } else { p = iallocm(usize, alignment, zero); if (p == NULL) - goto OOM; -#ifndef JEMALLOC_STATS - if (rsize != NULL) -#endif - { -#ifdef JEMALLOC_STATS - if (rsize != NULL) -#endif - *rsize = usize; - } + goto label_oom; } + if (rsize != NULL) + *rsize = usize; *ptr = p; -#ifdef JEMALLOC_STATS - assert(usize == isalloc(p)); - ALLOCATED_ADD(usize, 0); -#endif + if (config_stats) { + assert(usize == isalloc(p, config_prof)); + thread_allocated_tsd_get()->allocated += usize; + } + UTRACE(0, size, p); + JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); return (ALLOCM_SUCCESS); -OOM: -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { +label_oom: + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in allocm(): " "out of memory\n"); abort(); } -#endif *ptr = NULL; + UTRACE(0, size, 0); return (ALLOCM_ERR_OOM); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, - int flags) +je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) { void *p, *q; size_t usize; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) size_t old_size; -#endif + size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); bool zero = flags & ALLOCM_ZERO; bool no_move = flags & ALLOCM_NO_MOVE; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt; -#endif assert(ptr != NULL); assert(*ptr != NULL); assert(size != 0); assert(SIZE_T_MAX - size >= extra); - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); p = *ptr; -#ifdef JEMALLOC_PROF - if (opt_prof) { + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + /* * usize isn't knowable before iralloc() returns when extra is * non-zero. Therefore, compute its maximum possible value and @@ -1691,191 +1455,284 @@ JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, * decide whether to sample. */ size_t max_usize = (alignment == 0) ? s2u(size+extra) : - sa2u(size+extra, alignment, NULL); + sa2u(size+extra, alignment); prof_ctx_t *old_ctx = prof_ctx_get(p); - old_size = isalloc(p); + old_size = isalloc(p, true); + if (config_valgrind && opt_valgrind) + old_rzsize = p2rz(p); PROF_ALLOC_PREP(1, max_usize, cnt); if (cnt == NULL) - goto OOM; + goto label_oom; /* * Use minimum usize to determine whether promotion may happen. */ if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U - && ((alignment == 0) ? s2u(size) : sa2u(size, - alignment, NULL)) <= small_maxclass) { - q = iralloc(p, small_maxclass+1, (small_maxclass+1 >= - size+extra) ? 0 : size+extra - (small_maxclass+1), + && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) + <= SMALL_MAXCLASS) { + q = iralloc(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= + size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), alignment, zero, no_move); if (q == NULL) - goto ERR; - if (max_usize < PAGE_SIZE) { + goto label_err; + if (max_usize < PAGE) { usize = max_usize; arena_prof_promoted(q, usize); } else - usize = isalloc(q); + usize = isalloc(q, config_prof); } else { q = iralloc(p, size, extra, alignment, zero, no_move); if (q == NULL) - goto ERR; - usize = isalloc(q); + goto label_err; + usize = isalloc(q, config_prof); } prof_realloc(q, usize, cnt, old_size, old_ctx); if (rsize != NULL) *rsize = usize; - } else -#endif - { -#ifdef JEMALLOC_STATS - old_size = isalloc(p); -#endif + } else { + if (config_stats) { + old_size = isalloc(p, false); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_size); + } else if (config_valgrind && opt_valgrind) { + old_size = isalloc(p, false); + old_rzsize = u2rz(old_size); + } q = iralloc(p, size, extra, alignment, zero, no_move); if (q == NULL) - goto ERR; -#ifndef JEMALLOC_STATS - if (rsize != NULL) -#endif - { - usize = isalloc(q); -#ifdef JEMALLOC_STATS - if (rsize != NULL) -#endif - *rsize = usize; + goto label_err; + if (config_stats) + usize = isalloc(q, config_prof); + if (rsize != NULL) { + if (config_stats == false) + usize = isalloc(q, config_prof); + *rsize = usize; } } *ptr = q; -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(usize, old_size); -#endif + if (config_stats) { + thread_allocated_t *ta; + ta = thread_allocated_tsd_get(); + ta->allocated += usize; + ta->deallocated += old_size; + } + UTRACE(p, size, q); + JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); return (ALLOCM_SUCCESS); -ERR: - if (no_move) +label_err: + if (no_move) { + UTRACE(p, size, q); return (ALLOCM_ERR_NOT_MOVED); -#ifdef JEMALLOC_PROF -OOM: -#endif -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { + } +label_oom: + if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in rallocm(): " "out of memory\n"); abort(); } -#endif + UTRACE(p, size, 0); return (ALLOCM_ERR_OOM); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) +je_sallocm(const void *ptr, size_t *rsize, int flags) { size_t sz; - assert(malloc_initialized || malloc_initializer == pthread_self()); + assert(malloc_initialized || IS_INITIALIZER); -#ifdef JEMALLOC_IVSALLOC - sz = ivsalloc(ptr); -#else - assert(ptr != NULL); - sz = isalloc(ptr); -#endif + if (config_ivsalloc) + sz = ivsalloc(ptr, config_prof); + else { + assert(ptr != NULL); + sz = isalloc(ptr, config_prof); + } assert(rsize != NULL); *rsize = sz; return (ALLOCM_SUCCESS); } -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) int -JEMALLOC_P(dallocm)(void *ptr, int flags) +je_dallocm(void *ptr, int flags) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) size_t usize; -#endif + size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); assert(ptr != NULL); - assert(malloc_initialized || malloc_initializer == pthread_self()); - -#ifdef JEMALLOC_STATS - usize = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { -# ifndef JEMALLOC_STATS - usize = isalloc(ptr); -# endif + assert(malloc_initialized || IS_INITIALIZER); + + UTRACE(ptr, 0, 0); + if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); + if (config_prof && opt_prof) { + if (config_stats == false && config_valgrind == false) + usize = isalloc(ptr, config_prof); prof_free(ptr, usize); } -#endif -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(0, usize); -#endif - idalloc(ptr); + if (config_stats) + thread_allocated_tsd_get()->deallocated += usize; + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloc(ptr); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); return (ALLOCM_SUCCESS); } +int +je_nallocm(size_t *rsize, size_t size, int flags) +{ + size_t usize; + size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + & (SIZE_T_MAX-1)); + + assert(size != 0); + + if (malloc_init()) + return (ALLOCM_ERR_OOM); + + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + if (usize == 0) + return (ALLOCM_ERR_OOM); + + if (rsize != NULL) + *rsize = usize; + return (ALLOCM_SUCCESS); +} + +#endif /* - * End non-standard functions. + * End experimental functions. */ /******************************************************************************/ - /* * The following functions are used by threading libraries for protection of * malloc during fork(). */ +#ifndef JEMALLOC_MUTEX_INIT_CB void jemalloc_prefork(void) +#else +JEMALLOC_EXPORT void +_malloc_prefork(void) +#endif { unsigned i; - /* Acquire all mutexes in a safe order. */ +#ifdef JEMALLOC_MUTEX_INIT_CB + if (malloc_initialized == false) + return; +#endif + assert(malloc_initialized); - malloc_mutex_lock(&arenas_lock); + /* Acquire all mutexes in a safe order. */ + malloc_mutex_prefork(&arenas_lock); for (i = 0; i < narenas; i++) { if (arenas[i] != NULL) - malloc_mutex_lock(&arenas[i]->lock); + arena_prefork(arenas[i]); } + base_prefork(); + huge_prefork(); + chunk_dss_prefork(); +} - malloc_mutex_lock(&base_mtx); - - malloc_mutex_lock(&huge_mtx); - -#ifdef JEMALLOC_DSS - malloc_mutex_lock(&dss_mtx); +#ifndef JEMALLOC_MUTEX_INIT_CB +void +jemalloc_postfork_parent(void) +#else +JEMALLOC_EXPORT void +_malloc_postfork(void) #endif +{ + unsigned i; -#ifdef JEMALLOC_SWAP - malloc_mutex_lock(&swap_mtx); +#ifdef JEMALLOC_MUTEX_INIT_CB + if (malloc_initialized == false) + return; #endif + assert(malloc_initialized); + + /* Release all mutexes, now that fork() has completed. */ + chunk_dss_postfork_parent(); + huge_postfork_parent(); + base_postfork_parent(); + for (i = 0; i < narenas; i++) { + if (arenas[i] != NULL) + arena_postfork_parent(arenas[i]); + } + malloc_mutex_postfork_parent(&arenas_lock); } void -jemalloc_postfork(void) +jemalloc_postfork_child(void) { unsigned i; + assert(malloc_initialized); + /* Release all mutexes, now that fork() has completed. */ + chunk_dss_postfork_child(); + huge_postfork_child(); + base_postfork_child(); + for (i = 0; i < narenas; i++) { + if (arenas[i] != NULL) + arena_postfork_child(arenas[i]); + } + malloc_mutex_postfork_child(&arenas_lock); +} -#ifdef JEMALLOC_SWAP - malloc_mutex_unlock(&swap_mtx); -#endif +/******************************************************************************/ +/* + * The following functions are used for TLS allocation/deallocation in static + * binaries on FreeBSD. The primary difference between these and i[mcd]alloc() + * is that these avoid accessing TLS variables. + */ -#ifdef JEMALLOC_DSS - malloc_mutex_unlock(&dss_mtx); -#endif +static void * +a0alloc(size_t size, bool zero) +{ + + if (malloc_init()) + return (NULL); - malloc_mutex_unlock(&huge_mtx); + if (size == 0) + size = 1; - malloc_mutex_unlock(&base_mtx); + if (size <= arena_maxclass) + return (arena_malloc(arenas[0], size, zero, false)); + else + return (huge_malloc(size, zero)); +} - for (i = 0; i < narenas; i++) { - if (arenas[i] != NULL) - malloc_mutex_unlock(&arenas[i]->lock); - } - malloc_mutex_unlock(&arenas_lock); +void * +a0malloc(size_t size) +{ + + return (a0alloc(size, false)); +} + +void * +a0calloc(size_t num, size_t size) +{ + + return (a0alloc(num * size, true)); +} + +void +a0free(void *ptr) +{ + arena_chunk_t *chunk; + + if (ptr == NULL) + return; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + if (chunk != ptr) + arena_dalloc(chunk->arena, chunk, ptr, false); + else + huge_dalloc(ptr, true); } /******************************************************************************/ diff --git a/deps/jemalloc/src/mutex.c b/deps/jemalloc/src/mutex.c index ca89ef1c962..37a843e6e44 100644 --- a/deps/jemalloc/src/mutex.c +++ b/deps/jemalloc/src/mutex.c @@ -1,14 +1,26 @@ #define JEMALLOC_MUTEX_C_ #include "jemalloc/internal/jemalloc_internal.h" +#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) +#include +#endif + +#ifndef _CRT_SPINCOUNT +#define _CRT_SPINCOUNT 4000 +#endif + /******************************************************************************/ /* Data. */ #ifdef JEMALLOC_LAZY_LOCK bool isthreaded = false; #endif +#ifdef JEMALLOC_MUTEX_INIT_CB +static bool postpone_init = true; +static malloc_mutex_t *postponed_mutexes = NULL; +#endif -#ifdef JEMALLOC_LAZY_LOCK +#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) static void pthread_create_once(void); #endif @@ -18,7 +30,7 @@ static void pthread_create_once(void); * process goes multi-threaded. */ -#ifdef JEMALLOC_LAZY_LOCK +#if defined(JEMALLOC_LAZY_LOCK) && !defined(_WIN32) static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, void *(*)(void *), void *__restrict); @@ -36,8 +48,7 @@ pthread_create_once(void) isthreaded = true; } -JEMALLOC_ATTR(visibility("default")) -int +JEMALLOC_EXPORT int pthread_create(pthread_t *__restrict thread, const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), void *__restrict arg) @@ -52,39 +63,87 @@ pthread_create(pthread_t *__restrict thread, /******************************************************************************/ +#ifdef JEMALLOC_MUTEX_INIT_CB +int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, + void *(calloc_cb)(size_t, size_t)); +#endif + bool malloc_mutex_init(malloc_mutex_t *mutex) { -#ifdef JEMALLOC_OSSPIN - *mutex = 0; + +#ifdef _WIN32 + if (!InitializeCriticalSectionAndSpinCount(&mutex->lock, + _CRT_SPINCOUNT)) + return (true); +#elif (defined(JEMALLOC_OSSPIN)) + mutex->lock = 0; +#elif (defined(JEMALLOC_MUTEX_INIT_CB)) + if (postpone_init) { + mutex->postponed_next = postponed_mutexes; + postponed_mutexes = mutex; + } else { + if (_pthread_mutex_init_calloc_cb(&mutex->lock, base_calloc) != + 0) + return (true); + } #else pthread_mutexattr_t attr; if (pthread_mutexattr_init(&attr) != 0) return (true); -#ifdef PTHREAD_MUTEX_ADAPTIVE_NP - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP); -#else - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); -#endif - if (pthread_mutex_init(mutex, &attr) != 0) { + pthread_mutexattr_settype(&attr, MALLOC_MUTEX_TYPE); + if (pthread_mutex_init(&mutex->lock, &attr) != 0) { pthread_mutexattr_destroy(&attr); return (true); } pthread_mutexattr_destroy(&attr); - #endif return (false); } void -malloc_mutex_destroy(malloc_mutex_t *mutex) +malloc_mutex_prefork(malloc_mutex_t *mutex) { -#ifndef JEMALLOC_OSSPIN - if (pthread_mutex_destroy(mutex) != 0) { - malloc_write(": Error in pthread_mutex_destroy()\n"); - abort(); + malloc_mutex_lock(mutex); +} + +void +malloc_mutex_postfork_parent(malloc_mutex_t *mutex) +{ + + malloc_mutex_unlock(mutex); +} + +void +malloc_mutex_postfork_child(malloc_mutex_t *mutex) +{ + +#ifdef JEMALLOC_MUTEX_INIT_CB + malloc_mutex_unlock(mutex); +#else + if (malloc_mutex_init(mutex)) { + malloc_printf(": Error re-initializing mutex in " + "child\n"); + if (opt_abort) + abort(); } #endif } + +bool +mutex_boot(void) +{ + +#ifdef JEMALLOC_MUTEX_INIT_CB + postpone_init = false; + while (postponed_mutexes != NULL) { + if (_pthread_mutex_init_calloc_cb(&postponed_mutexes->lock, + base_calloc) != 0) + return (true); + postponed_mutexes = postponed_mutexes->postponed_next; + } +#endif + return (false); +} diff --git a/deps/jemalloc/src/prof.c b/deps/jemalloc/src/prof.c index 8a144b4e46c..de1d392993e 100644 --- a/deps/jemalloc/src/prof.c +++ b/deps/jemalloc/src/prof.c @@ -1,6 +1,5 @@ #define JEMALLOC_PROF_C_ #include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_PROF /******************************************************************************/ #ifdef JEMALLOC_PROF_LIBUNWIND @@ -15,27 +14,30 @@ /******************************************************************************/ /* Data. */ +malloc_tsd_data(, prof_tdata, prof_tdata_t *, NULL) + bool opt_prof = false; bool opt_prof_active = true; -size_t opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT; size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; bool opt_prof_gdump = false; +bool opt_prof_final = true; bool opt_prof_leak = false; -bool opt_prof_accum = true; -ssize_t opt_lg_prof_tcmax = LG_PROF_TCMAX_DEFAULT; +bool opt_prof_accum = false; char opt_prof_prefix[PATH_MAX + 1]; uint64_t prof_interval; bool prof_promote; -unsigned prof_bt_max; - -#ifndef NO_TLS -__thread prof_tdata_t *prof_tdata_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -#endif -pthread_key_t prof_tdata_tsd; +/* + * Table of mutexes that are shared among ctx's. These are leaf locks, so + * there is no problem with using them for more than one ctx at the same time. + * The primary motivation for this sharing though is that ctx's are ephemeral, + * and destroying mutexes causes complications for systems that allocate when + * creating/destroying mutexes. + */ +static malloc_mutex_t *ctx_locks; +static unsigned cum_ctxs; /* Atomic counter. */ /* * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data @@ -55,18 +57,13 @@ static uint64_t prof_dump_useq; * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since * it must be locked anyway during dumping. */ -static char prof_dump_buf[PROF_DUMP_BUF_SIZE]; +static char prof_dump_buf[PROF_DUMP_BUFSIZE]; static unsigned prof_dump_buf_end; static int prof_dump_fd; /* Do not dump any profiles until bootstrapping is complete. */ static bool prof_booted = false; -static malloc_mutex_t enq_mtx; -static bool enq; -static bool enq_idump; -static bool enq_gdump; - /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -79,22 +76,24 @@ static _Unwind_Reason_Code prof_unwind_callback( struct _Unwind_Context *context, void *arg); #endif static bool prof_flush(bool propagate_err); -static bool prof_write(const char *s, bool propagate_err); +static bool prof_write(bool propagate_err, const char *s); +static bool prof_printf(bool propagate_err, const char *format, ...) + JEMALLOC_ATTR(format(printf, 2, 3)); static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx); static void prof_ctx_destroy(prof_ctx_t *ctx); static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); -static bool prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, - bool propagate_err); +static bool prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, + prof_bt_t *bt); static bool prof_dump_maps(bool propagate_err); -static bool prof_dump(const char *filename, bool leakcheck, - bool propagate_err); +static bool prof_dump(bool propagate_err, const char *filename, + bool leakcheck); static void prof_dump_filename(char *filename, char v, int64_t vseq); static void prof_fdump(void); static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2); static bool prof_bt_keycomp(const void *k1, const void *k2); -static void prof_tdata_cleanup(void *arg); +static malloc_mutex_t *prof_ctx_mutex_choose(void); /******************************************************************************/ @@ -102,6 +101,8 @@ void bt_init(prof_bt_t *bt, void **vec) { + cassert(config_prof); + bt->vec = vec; bt->len = 0; } @@ -110,6 +111,8 @@ static void bt_destroy(prof_bt_t *bt) { + cassert(config_prof); + idalloc(bt); } @@ -118,6 +121,8 @@ bt_dup(prof_bt_t *bt) { prof_bt_t *ret; + cassert(config_prof); + /* * Create a single allocation that has space for vec immediately * following the prof_bt_t structure. The backtraces that get @@ -138,30 +143,32 @@ bt_dup(prof_bt_t *bt) } static inline void -prof_enter(void) +prof_enter(prof_tdata_t *prof_tdata) { - malloc_mutex_lock(&enq_mtx); - enq = true; - malloc_mutex_unlock(&enq_mtx); + cassert(config_prof); + + assert(prof_tdata->enq == false); + prof_tdata->enq = true; malloc_mutex_lock(&bt2ctx_mtx); } static inline void -prof_leave(void) +prof_leave(prof_tdata_t *prof_tdata) { bool idump, gdump; + cassert(config_prof); + malloc_mutex_unlock(&bt2ctx_mtx); - malloc_mutex_lock(&enq_mtx); - enq = false; - idump = enq_idump; - enq_idump = false; - gdump = enq_gdump; - enq_gdump = false; - malloc_mutex_unlock(&enq_mtx); + assert(prof_tdata->enq); + prof_tdata->enq = false; + idump = prof_tdata->enq_idump; + prof_tdata->enq_idump = false; + gdump = prof_tdata->enq_gdump; + prof_tdata->enq_gdump = false; if (idump) prof_idump(); @@ -171,16 +178,16 @@ prof_leave(void) #ifdef JEMALLOC_PROF_LIBUNWIND void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +prof_backtrace(prof_bt_t *bt, unsigned nignore) { unw_context_t uc; unw_cursor_t cursor; unsigned i; int err; + cassert(config_prof); assert(bt->len == 0); assert(bt->vec != NULL); - assert(max <= (1U << opt_lg_prof_bt_max)); unw_getcontext(&uc); unw_init_local(&cursor, &uc); @@ -196,7 +203,7 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) * Iterate over stack frames until there are no more, or until no space * remains in bt. */ - for (i = 0; i < max; i++) { + for (i = 0; i < PROF_BT_MAX; i++) { unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); bt->len++; err = unw_step(&cursor); @@ -204,12 +211,13 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) break; } } -#endif -#ifdef JEMALLOC_PROF_LIBGCC +#elif (defined(JEMALLOC_PROF_LIBGCC)) static _Unwind_Reason_Code prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) { + cassert(config_prof); + return (_URC_NO_REASON); } @@ -218,6 +226,8 @@ prof_unwind_callback(struct _Unwind_Context *context, void *arg) { prof_unwind_data_t *data = (prof_unwind_data_t *)arg; + cassert(config_prof); + if (data->nignore > 0) data->nignore--; else { @@ -231,19 +241,20 @@ prof_unwind_callback(struct _Unwind_Context *context, void *arg) } void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +prof_backtrace(prof_bt_t *bt, unsigned nignore) { - prof_unwind_data_t data = {bt, nignore, max}; + prof_unwind_data_t data = {bt, nignore, PROF_BT_MAX}; + + cassert(config_prof); _Unwind_Backtrace(prof_unwind_callback, &data); } -#endif -#ifdef JEMALLOC_PROF_GCC +#elif (defined(JEMALLOC_PROF_GCC)) void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) +prof_backtrace(prof_bt_t *bt, unsigned nignore) { #define BT_FRAME(i) \ - if ((i) < nignore + max) { \ + if ((i) < nignore + PROF_BT_MAX) { \ void *p; \ if (__builtin_frame_address(i) == 0) \ return; \ @@ -257,8 +268,8 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) } else \ return; + cassert(config_prof); assert(nignore <= 3); - assert(max <= (1U << opt_lg_prof_bt_max)); BT_FRAME(0) BT_FRAME(1) @@ -407,6 +418,14 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) BT_FRAME(130) #undef BT_FRAME } +#else +void +prof_backtrace(prof_bt_t *bt, unsigned nignore) +{ + + cassert(config_prof); + assert(false); +} #endif prof_thr_cnt_t * @@ -418,12 +437,11 @@ prof_lookup(prof_bt_t *bt) } ret; prof_tdata_t *prof_tdata; - prof_tdata = PROF_TCACHE_GET(); - if (prof_tdata == NULL) { - prof_tdata = prof_tdata_init(); - if (prof_tdata == NULL) - return (NULL); - } + cassert(config_prof); + + prof_tdata = prof_tdata_get(); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (NULL); if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { union { @@ -440,62 +458,51 @@ prof_lookup(prof_bt_t *bt) * This thread's cache lacks bt. Look for it in the global * cache. */ - prof_enter(); + prof_enter(prof_tdata); if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { /* bt has never been seen before. Insert it. */ ctx.v = imalloc(sizeof(prof_ctx_t)); if (ctx.v == NULL) { - prof_leave(); + prof_leave(prof_tdata); return (NULL); } btkey.p = bt_dup(bt); if (btkey.v == NULL) { - prof_leave(); + prof_leave(prof_tdata); idalloc(ctx.v); return (NULL); } ctx.p->bt = btkey.p; - if (malloc_mutex_init(&ctx.p->lock)) { - prof_leave(); - idalloc(btkey.v); - idalloc(ctx.v); - return (NULL); - } + ctx.p->lock = prof_ctx_mutex_choose(); + /* + * Set nlimbo to 1, in order to avoid a race condition + * with prof_ctx_merge()/prof_ctx_destroy(). + */ + ctx.p->nlimbo = 1; memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); ql_new(&ctx.p->cnts_ql); if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { /* OOM. */ - prof_leave(); - malloc_mutex_destroy(&ctx.p->lock); + prof_leave(prof_tdata); idalloc(btkey.v); idalloc(ctx.v); return (NULL); } - /* - * Artificially raise curobjs, in order to avoid a race - * condition with prof_ctx_merge()/prof_ctx_destroy(). - * - * No locking is necessary for ctx here because no other - * threads have had the opportunity to fetch it from - * bt2ctx yet. - */ - ctx.p->cnt_merged.curobjs++; new_ctx = true; } else { /* - * Artificially raise curobjs, in order to avoid a race - * condition with prof_ctx_merge()/prof_ctx_destroy(). + * Increment nlimbo, in order to avoid a race condition + * with prof_ctx_merge()/prof_ctx_destroy(). */ - malloc_mutex_lock(&ctx.p->lock); - ctx.p->cnt_merged.curobjs++; - malloc_mutex_unlock(&ctx.p->lock); + malloc_mutex_lock(ctx.p->lock); + ctx.p->nlimbo++; + malloc_mutex_unlock(ctx.p->lock); new_ctx = false; } - prof_leave(); + prof_leave(prof_tdata); /* Link a prof_thd_cnt_t into ctx for this thread. */ - if (opt_lg_prof_tcmax >= 0 && ckh_count(&prof_tdata->bt2cnt) - == (ZU(1) << opt_lg_prof_tcmax)) { + if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { assert(ckh_count(&prof_tdata->bt2cnt) > 0); /* * Flush the least recently used cnt in order to keep @@ -510,9 +517,7 @@ prof_lookup(prof_bt_t *bt) prof_ctx_merge(ret.p->ctx, ret.p); /* ret can now be re-used. */ } else { - assert(opt_lg_prof_tcmax < 0 || - ckh_count(&prof_tdata->bt2cnt) < (ZU(1) << - opt_lg_prof_tcmax)); + assert(ckh_count(&prof_tdata->bt2cnt) < PROF_TCMAX); /* Allocate and partially initialize a new cnt. */ ret.v = imalloc(sizeof(prof_thr_cnt_t)); if (ret.p == NULL) { @@ -534,10 +539,10 @@ prof_lookup(prof_bt_t *bt) return (NULL); } ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); - malloc_mutex_lock(&ctx.p->lock); + malloc_mutex_lock(ctx.p->lock); ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); - ctx.p->cnt_merged.curobjs--; - malloc_mutex_unlock(&ctx.p->lock); + ctx.p->nlimbo--; + malloc_mutex_unlock(ctx.p->lock); } else { /* Move ret to the front of the LRU. */ ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); @@ -553,6 +558,8 @@ prof_flush(bool propagate_err) bool ret = false; ssize_t err; + cassert(config_prof); + err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); if (err == -1) { if (propagate_err == false) { @@ -569,24 +576,26 @@ prof_flush(bool propagate_err) } static bool -prof_write(const char *s, bool propagate_err) +prof_write(bool propagate_err, const char *s) { unsigned i, slen, n; + cassert(config_prof); + i = 0; slen = strlen(s); while (i < slen) { /* Flush the buffer if it is full. */ - if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) + if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) if (prof_flush(propagate_err) && propagate_err) return (true); - if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) { + if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { /* Finish writing. */ n = slen - i; } else { /* Write as much of s as will fit. */ - n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end; + n = PROF_DUMP_BUFSIZE - prof_dump_buf_end; } memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); prof_dump_buf_end += n; @@ -596,13 +605,31 @@ prof_write(const char *s, bool propagate_err) return (false); } +JEMALLOC_ATTR(format(printf, 2, 3)) +static bool +prof_printf(bool propagate_err, const char *format, ...) +{ + bool ret; + va_list ap; + char buf[PROF_PRINTF_BUFSIZE]; + + va_start(ap, format); + malloc_vsnprintf(buf, sizeof(buf), format, ap); + va_end(ap); + ret = prof_write(propagate_err, buf); + + return (ret); +} + static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) { prof_thr_cnt_t *thr_cnt; prof_cnt_t tcnt; - malloc_mutex_lock(&ctx->lock); + cassert(config_prof); + + malloc_mutex_lock(ctx->lock); memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { @@ -641,43 +668,48 @@ prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) cnt_all->accumbytes += ctx->cnt_summed.accumbytes; } - malloc_mutex_unlock(&ctx->lock); + malloc_mutex_unlock(ctx->lock); } static void prof_ctx_destroy(prof_ctx_t *ctx) { + prof_tdata_t *prof_tdata; + + cassert(config_prof); /* * Check that ctx is still unused by any thread cache before destroying - * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in - * order to avoid a race condition with this function, as does - * prof_ctx_merge() in order to avoid a race between the main body of - * prof_ctx_merge() and entry into this function. + * it. prof_lookup() increments ctx->nlimbo in order to avoid a race + * condition with this function, as does prof_ctx_merge() in order to + * avoid a race between the main body of prof_ctx_merge() and entry + * into this function. */ - prof_enter(); - malloc_mutex_lock(&ctx->lock); - if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { + prof_tdata = *prof_tdata_tsd_get(); + assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX); + prof_enter(prof_tdata); + malloc_mutex_lock(ctx->lock); + if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 0 && + ctx->nlimbo == 1) { assert(ctx->cnt_merged.curbytes == 0); assert(ctx->cnt_merged.accumobjs == 0); assert(ctx->cnt_merged.accumbytes == 0); /* Remove ctx from bt2ctx. */ if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) assert(false); - prof_leave(); + prof_leave(prof_tdata); /* Destroy ctx. */ - malloc_mutex_unlock(&ctx->lock); + malloc_mutex_unlock(ctx->lock); bt_destroy(ctx->bt); - malloc_mutex_destroy(&ctx->lock); idalloc(ctx); } else { /* * Compensate for increment in prof_ctx_merge() or * prof_lookup(). */ - ctx->cnt_merged.curobjs--; - malloc_mutex_unlock(&ctx->lock); - prof_leave(); + ctx->nlimbo--; + malloc_mutex_unlock(ctx->lock); + prof_leave(prof_tdata); } } @@ -686,20 +718,22 @@ prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) { bool destroy; + cassert(config_prof); + /* Merge cnt stats and detach from ctx. */ - malloc_mutex_lock(&ctx->lock); + malloc_mutex_lock(ctx->lock); ctx->cnt_merged.curobjs += cnt->cnts.curobjs; ctx->cnt_merged.curbytes += cnt->cnts.curbytes; ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; ql_remove(&ctx->cnts_ql, cnt, cnts_link); if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && - ctx->cnt_merged.curobjs == 0) { + ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { /* - * Artificially raise ctx->cnt_merged.curobjs in order to keep - * another thread from winning the race to destroy ctx while - * this one has ctx->lock dropped. Without this, it would be - * possible for another thread to: + * Increment ctx->nlimbo in order to keep another thread from + * winning the race to destroy ctx while this one has ctx->lock + * dropped. Without this, it would be possible for another + * thread to: * * 1) Sample an allocation associated with ctx. * 2) Deallocate the sampled object. @@ -708,49 +742,51 @@ prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) * The result would be that ctx no longer exists by the time * this thread accesses it in prof_ctx_destroy(). */ - ctx->cnt_merged.curobjs++; + ctx->nlimbo++; destroy = true; } else destroy = false; - malloc_mutex_unlock(&ctx->lock); + malloc_mutex_unlock(ctx->lock); if (destroy) prof_ctx_destroy(ctx); } static bool -prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, bool propagate_err) +prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, prof_bt_t *bt) { - char buf[UMAX2S_BUFSIZE]; unsigned i; - if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { + cassert(config_prof); + + /* + * Current statistics can sum to 0 as a result of unmerged per thread + * statistics. Additionally, interval- and growth-triggered dumps can + * occur between the time a ctx is created and when its statistics are + * filled in. Avoid dumping any ctx that is an artifact of either + * implementation detail. + */ + if ((opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) || + (opt_prof_accum && ctx->cnt_summed.accumobjs == 0)) { + assert(ctx->cnt_summed.curobjs == 0); assert(ctx->cnt_summed.curbytes == 0); assert(ctx->cnt_summed.accumobjs == 0); assert(ctx->cnt_summed.accumbytes == 0); return (false); } - if (prof_write(u2s(ctx->cnt_summed.curobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(ctx->cnt_summed.curbytes, 10, buf), - propagate_err) - || prof_write(" [", propagate_err) - || prof_write(u2s(ctx->cnt_summed.accumobjs, 10, buf), - propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(ctx->cnt_summed.accumbytes, 10, buf), - propagate_err) - || prof_write("] @", propagate_err)) + if (prof_printf(propagate_err, "%"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @", + ctx->cnt_summed.curobjs, ctx->cnt_summed.curbytes, + ctx->cnt_summed.accumobjs, ctx->cnt_summed.accumbytes)) return (true); for (i = 0; i < bt->len; i++) { - if (prof_write(" 0x", propagate_err) - || prof_write(u2s((uintptr_t)bt->vec[i], 16, buf), - propagate_err)) + if (prof_printf(propagate_err, " %#"PRIxPTR, + (uintptr_t)bt->vec[i])) return (true); } - if (prof_write("\n", propagate_err)) + if (prof_write(propagate_err, "\n")) return (true); return (false); @@ -760,49 +796,29 @@ static bool prof_dump_maps(bool propagate_err) { int mfd; - char buf[UMAX2S_BUFSIZE]; - char *s; - unsigned i, slen; - /* /proc//maps\0 */ - char mpath[6 + UMAX2S_BUFSIZE - + 5 + 1]; + char filename[PATH_MAX + 1]; - i = 0; + cassert(config_prof); - s = "/proc/"; - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - s = u2s(getpid(), 10, buf); - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - s = "/maps"; - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - mpath[i] = '\0'; - - mfd = open(mpath, O_RDONLY); + malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps", + (int)getpid()); + mfd = open(filename, O_RDONLY); if (mfd != -1) { ssize_t nread; - if (prof_write("\nMAPPED_LIBRARIES:\n", propagate_err) && + if (prof_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && propagate_err) return (true); nread = 0; do { prof_dump_buf_end += nread; - if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) { + if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { /* Make space in prof_dump_buf before read(). */ if (prof_flush(propagate_err) && propagate_err) return (true); } nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], - PROF_DUMP_BUF_SIZE - prof_dump_buf_end); + PROF_DUMP_BUFSIZE - prof_dump_buf_end); } while (nread > 0); close(mfd); } else @@ -812,8 +828,9 @@ prof_dump_maps(bool propagate_err) } static bool -prof_dump(const char *filename, bool leakcheck, bool propagate_err) +prof_dump(bool propagate_err, const char *filename, bool leakcheck) { + prof_tdata_t *prof_tdata; prof_cnt_t cnt_all; size_t tabind; union { @@ -824,20 +841,24 @@ prof_dump(const char *filename, bool leakcheck, bool propagate_err) prof_ctx_t *p; void *v; } ctx; - char buf[UMAX2S_BUFSIZE]; size_t leak_nctx; - prof_enter(); + cassert(config_prof); + + prof_tdata = prof_tdata_get(); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (true); + prof_enter(prof_tdata); prof_dump_fd = creat(filename, 0644); if (prof_dump_fd == -1) { if (propagate_err == false) { - malloc_write(": creat(\""); - malloc_write(filename); - malloc_write("\", 0644) failed\n"); + malloc_printf( + ": creat(\"%s\"), 0644) failed\n", + filename); if (opt_abort) abort(); } - goto ERROR; + goto label_error; } /* Merge per thread profile stats, and sum them in cnt_all. */ @@ -847,131 +868,75 @@ prof_dump(const char *filename, bool leakcheck, bool propagate_err) prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); /* Dump profile header. */ - if (prof_write("heap profile: ", propagate_err) - || prof_write(u2s(cnt_all.curobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(cnt_all.curbytes, 10, buf), propagate_err) - || prof_write(" [", propagate_err) - || prof_write(u2s(cnt_all.accumobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(cnt_all.accumbytes, 10, buf), propagate_err)) - goto ERROR; - if (opt_lg_prof_sample == 0) { - if (prof_write("] @ heapprofile\n", propagate_err)) - goto ERROR; + if (prof_printf(propagate_err, + "heap profile: %"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @ heapprofile\n", + cnt_all.curobjs, cnt_all.curbytes, + cnt_all.accumobjs, cnt_all.accumbytes)) + goto label_error; } else { - if (prof_write("] @ heap_v2/", propagate_err) - || prof_write(u2s((uint64_t)1U << opt_lg_prof_sample, 10, - buf), propagate_err) - || prof_write("\n", propagate_err)) - goto ERROR; + if (prof_printf(propagate_err, + "heap profile: %"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @ heap_v2/%"PRIu64"\n", + cnt_all.curobjs, cnt_all.curbytes, + cnt_all.accumobjs, cnt_all.accumbytes, + ((uint64_t)1U << opt_lg_prof_sample))) + goto label_error; } /* Dump per ctx profile stats. */ for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) == false;) { - if (prof_dump_ctx(ctx.p, bt.p, propagate_err)) - goto ERROR; + if (prof_dump_ctx(propagate_err, ctx.p, bt.p)) + goto label_error; } /* Dump /proc//maps if possible. */ if (prof_dump_maps(propagate_err)) - goto ERROR; + goto label_error; if (prof_flush(propagate_err)) - goto ERROR; + goto label_error; close(prof_dump_fd); - prof_leave(); + prof_leave(prof_tdata); if (leakcheck && cnt_all.curbytes != 0) { - malloc_write(": Leak summary: "); - malloc_write(u2s(cnt_all.curbytes, 10, buf)); - malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, "); - malloc_write(u2s(cnt_all.curobjs, 10, buf)); - malloc_write((cnt_all.curobjs != 1) ? " objects, " : - " object, "); - malloc_write(u2s(leak_nctx, 10, buf)); - malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n"); - malloc_write(": Run pprof on \""); - malloc_write(filename); - malloc_write("\" for leak detail\n"); + malloc_printf(": Leak summary: %"PRId64" byte%s, %" + PRId64" object%s, %zu context%s\n", + cnt_all.curbytes, (cnt_all.curbytes != 1) ? "s" : "", + cnt_all.curobjs, (cnt_all.curobjs != 1) ? "s" : "", + leak_nctx, (leak_nctx != 1) ? "s" : ""); + malloc_printf( + ": Run pprof on \"%s\" for leak detail\n", + filename); } return (false); -ERROR: - prof_leave(); +label_error: + prof_leave(prof_tdata); return (true); } -#define DUMP_FILENAME_BUFSIZE (PATH_MAX+ UMAX2S_BUFSIZE \ - + 1 \ - + UMAX2S_BUFSIZE \ - + 2 \ - + UMAX2S_BUFSIZE \ - + 5 + 1) +#define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) static void prof_dump_filename(char *filename, char v, int64_t vseq) { - char buf[UMAX2S_BUFSIZE]; - char *s; - unsigned i, slen; - - /* - * Construct a filename of the form: - * - * ...v.heap\0 - */ - - i = 0; - s = opt_prof_prefix; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = u2s(getpid(), 10, buf); - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; + cassert(config_prof); - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = u2s(prof_dump_seq, 10, buf); - prof_dump_seq++; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - filename[i] = v; - i++; - - if (vseq != 0xffffffffffffffffLLU) { - s = u2s(vseq, 10, buf); - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; + if (vseq != UINT64_C(0xffffffffffffffff)) { + /* "...v.heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%d.%"PRIu64".%c%"PRId64".heap", + opt_prof_prefix, (int)getpid(), prof_dump_seq, v, vseq); + } else { + /* "....heap" */ + malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, + "%s.%d.%"PRIu64".%c.heap", + opt_prof_prefix, (int)getpid(), prof_dump_seq, v); } - - s = ".heap"; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - filename[i] = '\0'; + prof_dump_seq++; } static void @@ -979,38 +944,47 @@ prof_fdump(void) { char filename[DUMP_FILENAME_BUFSIZE]; + cassert(config_prof); + if (prof_booted == false) return; - if (opt_prof_prefix[0] != '\0') { + if (opt_prof_final && opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'f', 0xffffffffffffffffLLU); + prof_dump_filename(filename, 'f', UINT64_C(0xffffffffffffffff)); malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, opt_prof_leak, false); + prof_dump(false, filename, opt_prof_leak); } } void prof_idump(void) { - char filename[DUMP_FILENAME_BUFSIZE]; + prof_tdata_t *prof_tdata; + char filename[PATH_MAX + 1]; + + cassert(config_prof); if (prof_booted == false) return; - malloc_mutex_lock(&enq_mtx); - if (enq) { - enq_idump = true; - malloc_mutex_unlock(&enq_mtx); + /* + * Don't call prof_tdata_get() here, because it could cause recursive + * allocation. + */ + prof_tdata = *prof_tdata_tsd_get(); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return; + if (prof_tdata->enq) { + prof_tdata->enq_idump = true; return; } - malloc_mutex_unlock(&enq_mtx); if (opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'i', prof_dump_iseq); prof_dump_iseq++; malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, false, false); + prof_dump(false, filename, false); } } @@ -1019,6 +993,8 @@ prof_mdump(const char *filename) { char filename_buf[DUMP_FILENAME_BUFSIZE]; + cassert(config_prof); + if (opt_prof == false || prof_booted == false) return (true); @@ -1032,30 +1008,37 @@ prof_mdump(const char *filename) malloc_mutex_unlock(&prof_dump_seq_mtx); filename = filename_buf; } - return (prof_dump(filename, false, true)); + return (prof_dump(true, filename, false)); } void prof_gdump(void) { + prof_tdata_t *prof_tdata; char filename[DUMP_FILENAME_BUFSIZE]; + cassert(config_prof); + if (prof_booted == false) return; - malloc_mutex_lock(&enq_mtx); - if (enq) { - enq_gdump = true; - malloc_mutex_unlock(&enq_mtx); + /* + * Don't call prof_tdata_get() here, because it could cause recursive + * allocation. + */ + prof_tdata = *prof_tdata_tsd_get(); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return; + if (prof_tdata->enq) { + prof_tdata->enq_gdump = true; return; } - malloc_mutex_unlock(&enq_mtx); if (opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); prof_dump_filename(filename, 'u', prof_dump_useq); prof_dump_useq++; malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, false, false); + prof_dump(false, filename, false); } } @@ -1066,11 +1049,13 @@ prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) uint64_t h; prof_bt_t *bt = (prof_bt_t *)key; + cassert(config_prof); assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); assert(hash1 != NULL); assert(hash2 != NULL); - h = hash(bt->vec, bt->len * sizeof(void *), 0x94122f335b332aeaLLU); + h = hash(bt->vec, bt->len * sizeof(void *), + UINT64_C(0x94122f335b332aea)); if (minbits <= 32) { /* * Avoid doing multiple hashes, since a single hash provides @@ -1081,7 +1066,7 @@ prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) } else { ret1 = h; ret2 = hash(bt->vec, bt->len * sizeof(void *), - 0x8432a476666bbc13LLU); + UINT64_C(0x8432a476666bbc13)); } *hash1 = ret1; @@ -1094,16 +1079,28 @@ prof_bt_keycomp(const void *k1, const void *k2) const prof_bt_t *bt1 = (prof_bt_t *)k1; const prof_bt_t *bt2 = (prof_bt_t *)k2; + cassert(config_prof); + if (bt1->len != bt2->len) return (false); return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } +static malloc_mutex_t * +prof_ctx_mutex_choose(void) +{ + unsigned nctxs = atomic_add_u(&cum_ctxs, 1); + + return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); +} + prof_tdata_t * prof_tdata_init(void) { prof_tdata_t *prof_tdata; + cassert(config_prof); + /* Initialize an empty cache for this thread. */ prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); if (prof_tdata == NULL) @@ -1116,51 +1113,77 @@ prof_tdata_init(void) } ql_new(&prof_tdata->lru_ql); - prof_tdata->vec = imalloc(sizeof(void *) * prof_bt_max); + prof_tdata->vec = imalloc(sizeof(void *) * PROF_BT_MAX); if (prof_tdata->vec == NULL) { ckh_delete(&prof_tdata->bt2cnt); idalloc(prof_tdata); return (NULL); } - prof_tdata->prn_state = 0; + prof_tdata->prng_state = 0; prof_tdata->threshold = 0; prof_tdata->accum = 0; - PROF_TCACHE_SET(prof_tdata); + prof_tdata->enq = false; + prof_tdata->enq_idump = false; + prof_tdata->enq_gdump = false; + + prof_tdata_tsd_set(&prof_tdata); return (prof_tdata); } -static void +void prof_tdata_cleanup(void *arg) { prof_thr_cnt_t *cnt; - prof_tdata_t *prof_tdata = (prof_tdata_t *)arg; + prof_tdata_t *prof_tdata = *(prof_tdata_t **)arg; - /* - * Delete the hash table. All of its contents can still be iterated - * over via the LRU. - */ - ckh_delete(&prof_tdata->bt2cnt); + cassert(config_prof); - /* Iteratively merge cnt's into the global stats and delete them. */ - while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { - ql_remove(&prof_tdata->lru_ql, cnt, lru_link); - prof_ctx_merge(cnt->ctx, cnt); - idalloc(cnt); + if (prof_tdata == PROF_TDATA_STATE_REINCARNATED) { + /* + * Another destructor deallocated memory after this destructor + * was called. Reset prof_tdata to PROF_TDATA_STATE_PURGATORY + * in order to receive another callback. + */ + prof_tdata = PROF_TDATA_STATE_PURGATORY; + prof_tdata_tsd_set(&prof_tdata); + } else if (prof_tdata == PROF_TDATA_STATE_PURGATORY) { + /* + * The previous time this destructor was called, we set the key + * to PROF_TDATA_STATE_PURGATORY so that other destructors + * wouldn't cause re-creation of the prof_tdata. This time, do + * nothing, so that the destructor will not be called again. + */ + } else if (prof_tdata != NULL) { + /* + * Delete the hash table. All of its contents can still be + * iterated over via the LRU. + */ + ckh_delete(&prof_tdata->bt2cnt); + /* + * Iteratively merge cnt's into the global stats and delete + * them. + */ + while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { + ql_remove(&prof_tdata->lru_ql, cnt, lru_link); + prof_ctx_merge(cnt->ctx, cnt); + idalloc(cnt); + } + idalloc(prof_tdata->vec); + idalloc(prof_tdata); + prof_tdata = PROF_TDATA_STATE_PURGATORY; + prof_tdata_tsd_set(&prof_tdata); } - - idalloc(prof_tdata->vec); - - idalloc(prof_tdata); - PROF_TCACHE_SET(NULL); } void prof_boot0(void) { + cassert(config_prof); + memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, sizeof(PROF_PREFIX_DEFAULT)); } @@ -1169,6 +1192,8 @@ void prof_boot1(void) { + cassert(config_prof); + /* * opt_prof and prof_promote must be in their final state before any * arenas are initialized, so this function must be executed early. @@ -1190,41 +1215,46 @@ prof_boot1(void) prof_interval = 0; } - prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT); + prof_promote = (opt_prof && opt_lg_prof_sample > LG_PAGE); } bool prof_boot2(void) { + cassert(config_prof); + if (opt_prof) { + unsigned i; + if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, prof_bt_keycomp)) return (true); if (malloc_mutex_init(&bt2ctx_mtx)) return (true); - if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup) - != 0) { + if (prof_tdata_tsd_boot()) { malloc_write( ": Error in pthread_key_create()\n"); abort(); } - prof_bt_max = (1U << opt_lg_prof_bt_max); if (malloc_mutex_init(&prof_dump_seq_mtx)) return (true); - if (malloc_mutex_init(&enq_mtx)) - return (true); - enq = false; - enq_idump = false; - enq_gdump = false; - if (atexit(prof_fdump) != 0) { malloc_write(": Error in atexit()\n"); if (opt_abort) abort(); } + + ctx_locks = (malloc_mutex_t *)base_alloc(PROF_NCTX_LOCKS * + sizeof(malloc_mutex_t)); + if (ctx_locks == NULL) + return (true); + for (i = 0; i < PROF_NCTX_LOCKS; i++) { + if (malloc_mutex_init(&ctx_locks[i])) + return (true); + } } #ifdef JEMALLOC_PROF_LIBGCC @@ -1241,4 +1271,3 @@ prof_boot2(void) } /******************************************************************************/ -#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc/src/quarantine.c b/deps/jemalloc/src/quarantine.c new file mode 100644 index 00000000000..9005ab3ba06 --- /dev/null +++ b/deps/jemalloc/src/quarantine.c @@ -0,0 +1,210 @@ +#include "jemalloc/internal/jemalloc_internal.h" + +/* + * quarantine pointers close to NULL are used to encode state information that + * is used for cleaning up during thread shutdown. + */ +#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) +#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) +#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY + +/******************************************************************************/ +/* Data. */ + +typedef struct quarantine_obj_s quarantine_obj_t; +typedef struct quarantine_s quarantine_t; + +struct quarantine_obj_s { + void *ptr; + size_t usize; +}; + +struct quarantine_s { + size_t curbytes; + size_t curobjs; + size_t first; +#define LG_MAXOBJS_INIT 10 + size_t lg_maxobjs; + quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */ +}; + +static void quarantine_cleanup(void *arg); + +malloc_tsd_data(static, quarantine, quarantine_t *, NULL) +malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL, + quarantine_cleanup) + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static quarantine_t *quarantine_init(size_t lg_maxobjs); +static quarantine_t *quarantine_grow(quarantine_t *quarantine); +static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); + +/******************************************************************************/ + +static quarantine_t * +quarantine_init(size_t lg_maxobjs) +{ + quarantine_t *quarantine; + + quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + + ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); + if (quarantine == NULL) + return (NULL); + quarantine->curbytes = 0; + quarantine->curobjs = 0; + quarantine->first = 0; + quarantine->lg_maxobjs = lg_maxobjs; + + quarantine_tsd_set(&quarantine); + + return (quarantine); +} + +static quarantine_t * +quarantine_grow(quarantine_t *quarantine) +{ + quarantine_t *ret; + + ret = quarantine_init(quarantine->lg_maxobjs + 1); + if (ret == NULL) + return (quarantine); + + ret->curbytes = quarantine->curbytes; + ret->curobjs = quarantine->curobjs; + if (quarantine->first + quarantine->curobjs <= (ZU(1) << + quarantine->lg_maxobjs)) { + /* objs ring buffer data are contiguous. */ + memcpy(ret->objs, &quarantine->objs[quarantine->first], + quarantine->curobjs * sizeof(quarantine_obj_t)); + } else { + /* objs ring buffer data wrap around. */ + size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - + quarantine->first; + size_t ncopy_b = quarantine->curobjs - ncopy_a; + + memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a + * sizeof(quarantine_obj_t)); + memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * + sizeof(quarantine_obj_t)); + } + + return (ret); +} + +static void +quarantine_drain(quarantine_t *quarantine, size_t upper_bound) +{ + + while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) { + quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; + assert(obj->usize == isalloc(obj->ptr, config_prof)); + idalloc(obj->ptr); + quarantine->curbytes -= obj->usize; + quarantine->curobjs--; + quarantine->first = (quarantine->first + 1) & ((ZU(1) << + quarantine->lg_maxobjs) - 1); + } +} + +void +quarantine(void *ptr) +{ + quarantine_t *quarantine; + size_t usize = isalloc(ptr, config_prof); + + cassert(config_fill); + assert(opt_quarantine); + + quarantine = *quarantine_tsd_get(); + if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { + if (quarantine == NULL) { + if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) == + NULL) { + idalloc(ptr); + return; + } + } else { + if (quarantine == QUARANTINE_STATE_PURGATORY) { + /* + * Make a note that quarantine() was called + * after quarantine_cleanup() was called. + */ + quarantine = QUARANTINE_STATE_REINCARNATED; + quarantine_tsd_set(&quarantine); + } + idalloc(ptr); + return; + } + } + /* + * Drain one or more objects if the quarantine size limit would be + * exceeded by appending ptr. + */ + if (quarantine->curbytes + usize > opt_quarantine) { + size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine + - usize : 0; + quarantine_drain(quarantine, upper_bound); + } + /* Grow the quarantine ring buffer if it's full. */ + if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) + quarantine = quarantine_grow(quarantine); + /* quarantine_grow() must free a slot if it fails to grow. */ + assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); + /* Append ptr if its size doesn't exceed the quarantine size. */ + if (quarantine->curbytes + usize <= opt_quarantine) { + size_t offset = (quarantine->first + quarantine->curobjs) & + ((ZU(1) << quarantine->lg_maxobjs) - 1); + quarantine_obj_t *obj = &quarantine->objs[offset]; + obj->ptr = ptr; + obj->usize = usize; + quarantine->curbytes += usize; + quarantine->curobjs++; + if (opt_junk) + memset(ptr, 0x5a, usize); + } else { + assert(quarantine->curbytes == 0); + idalloc(ptr); + } +} + +static void +quarantine_cleanup(void *arg) +{ + quarantine_t *quarantine = *(quarantine_t **)arg; + + if (quarantine == QUARANTINE_STATE_REINCARNATED) { + /* + * Another destructor deallocated memory after this destructor + * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY + * in order to receive another callback. + */ + quarantine = QUARANTINE_STATE_PURGATORY; + quarantine_tsd_set(&quarantine); + } else if (quarantine == QUARANTINE_STATE_PURGATORY) { + /* + * The previous time this destructor was called, we set the key + * to QUARANTINE_STATE_PURGATORY so that other destructors + * wouldn't cause re-creation of the quarantine. This time, do + * nothing, so that the destructor will not be called again. + */ + } else if (quarantine != NULL) { + quarantine_drain(quarantine, 0); + idalloc(quarantine); + quarantine = QUARANTINE_STATE_PURGATORY; + quarantine_tsd_set(&quarantine); + } +} + +bool +quarantine_boot(void) +{ + + cassert(config_fill); + + if (quarantine_tsd_boot()) + return (true); + + return (false); +} diff --git a/deps/jemalloc/src/stats.c b/deps/jemalloc/src/stats.c index dc172e425c0..433b80d128d 100644 --- a/deps/jemalloc/src/stats.c +++ b/deps/jemalloc/src/stats.c @@ -39,140 +39,40 @@ bool opt_stats_print = false; -#ifdef JEMALLOC_STATS size_t stats_cactive = 0; -#endif /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -#ifdef JEMALLOC_STATS -static void malloc_vcprintf(void (*write_cb)(void *, const char *), - void *cbopaque, const char *format, va_list ap); static void stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i); static void stats_arena_print(void (*write_cb)(void *, const char *), - void *cbopaque, unsigned i); -#endif + void *cbopaque, unsigned i, bool bins, bool large); /******************************************************************************/ -/* - * We don't want to depend on vsnprintf() for production builds, since that can - * cause unnecessary bloat for static binaries. u2s() provides minimal integer - * printing functionality, so that malloc_printf() use can be limited to - * JEMALLOC_STATS code. - */ -char * -u2s(uint64_t x, unsigned base, char *s) -{ - unsigned i; - - i = UMAX2S_BUFSIZE - 1; - s[i] = '\0'; - switch (base) { - case 10: - do { - i--; - s[i] = "0123456789"[x % (uint64_t)10]; - x /= (uint64_t)10; - } while (x > 0); - break; - case 16: - do { - i--; - s[i] = "0123456789abcdef"[x & 0xf]; - x >>= 4; - } while (x > 0); - break; - default: - do { - i--; - s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % - (uint64_t)base]; - x /= (uint64_t)base; - } while (x > 0); - } - - return (&s[i]); -} - -#ifdef JEMALLOC_STATS -static void -malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, - const char *format, va_list ap) -{ - char buf[4096]; - - if (write_cb == NULL) { - /* - * The caller did not provide an alternate write_cb callback - * function, so use the default one. malloc_write() is an - * inline function, so use malloc_message() directly here. - */ - write_cb = JEMALLOC_P(malloc_message); - cbopaque = NULL; - } - - vsnprintf(buf, sizeof(buf), format, ap); - write_cb(cbopaque, buf); -} - -/* - * Print to a callback function in such a way as to (hopefully) avoid memory - * allocation. - */ -JEMALLOC_ATTR(format(printf, 3, 4)) -void -malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, - const char *format, ...) -{ - va_list ap; - - va_start(ap, format); - malloc_vcprintf(write_cb, cbopaque, format, ap); - va_end(ap); -} - -/* - * Print to stderr in such a way as to (hopefully) avoid memory allocation. - */ -JEMALLOC_ATTR(format(printf, 1, 2)) -void -malloc_printf(const char *format, ...) -{ - va_list ap; - - va_start(ap, format); - malloc_vcprintf(NULL, NULL, format, ap); - va_end(ap); -} -#endif - -#ifdef JEMALLOC_STATS static void stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i) { - size_t pagesize; + size_t page; bool config_tcache; unsigned nbins, j, gap_start; - CTL_GET("arenas.pagesize", &pagesize, size_t); + CTL_GET("arenas.page", &page, size_t); CTL_GET("config.tcache", &config_tcache, bool); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" + "bins: bin size regs pgs allocated nmalloc" " ndalloc nrequests nfills nflushes" - " newruns reruns maxruns curruns\n"); + " newruns reruns curruns\n"); } else { malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" - " ndalloc newruns reruns maxruns" - " curruns\n"); + "bins: bin size regs pgs allocated nmalloc" + " ndalloc newruns reruns curruns\n"); } CTL_GET("arenas.nbins", &nbins, unsigned); for (j = 0, gap_start = UINT_MAX; j < nbins; j++) { @@ -183,12 +83,11 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, if (gap_start == UINT_MAX) gap_start = j; } else { - unsigned ntbins_, nqbins, ncbins, nsbins; size_t reg_size, run_size, allocated; uint32_t nregs; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t reruns; - size_t highruns, curruns; + size_t curruns; if (gap_start != UINT_MAX) { if (j > gap_start + 1) { @@ -203,10 +102,6 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } gap_start = UINT_MAX; } - CTL_GET("arenas.ntbins", &ntbins_, unsigned); - CTL_GET("arenas.nqbins", &nqbins, unsigned); - CTL_GET("arenas.ncbins", &ncbins, unsigned); - CTL_GET("arenas.nsbins", &nsbins, unsigned); CTL_J_GET("arenas.bin.0.size", ®_size, size_t); CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); @@ -226,36 +121,25 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, } CTL_IJ_GET("stats.arenas.0.bins.0.nreruns", &reruns, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.highruns", &highruns, - size_t); CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns, size_t); if (config_tcache) { malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 + "%13u %5zu %4u %3zu %12zu %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - j, - j < ntbins_ ? "T" : j < ntbins_ + nqbins ? - "Q" : j < ntbins_ + nqbins + ncbins ? "C" : - "S", - reg_size, nregs, run_size / pagesize, + " %12zu\n", + j, reg_size, nregs, run_size / page, allocated, nmalloc, ndalloc, nrequests, - nfills, nflushes, nruns, reruns, highruns, - curruns); + nfills, nflushes, nruns, reruns, curruns); } else { malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 + "%13u %5zu %4u %3zu %12zu %12"PRIu64 " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - j, - j < ntbins_ ? "T" : j < ntbins_ + nqbins ? - "Q" : j < ntbins_ + nqbins + ncbins ? "C" : - "S", - reg_size, nregs, run_size / pagesize, + " %12zu\n", + j, reg_size, nregs, run_size / page, allocated, nmalloc, ndalloc, nruns, reruns, - highruns, curruns); + curruns); } } } @@ -275,18 +159,18 @@ static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i) { - size_t pagesize, nlruns, j; + size_t page, nlruns, j; ssize_t gap_start; - CTL_GET("arenas.pagesize", &pagesize, size_t); + CTL_GET("arenas.page", &page, size_t); malloc_cprintf(write_cb, cbopaque, "large: size pages nmalloc ndalloc nrequests" - " maxruns curruns\n"); + " curruns\n"); CTL_GET("arenas.nlruns", &nlruns, size_t); for (j = 0, gap_start = -1; j < nlruns; j++) { uint64_t nmalloc, ndalloc, nrequests; - size_t run_size, highruns, curruns; + size_t run_size, curruns; CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc, uint64_t); @@ -299,8 +183,6 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, gap_start = j; } else { CTL_J_GET("arenas.lrun.0.size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.highruns", &highruns, - size_t); CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns, size_t); if (gap_start != -1) { @@ -310,9 +192,9 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, } malloc_cprintf(write_cb, cbopaque, "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - run_size, run_size / pagesize, nmalloc, ndalloc, - nrequests, highruns, curruns); + " %12zu\n", + run_size, run_size / page, nmalloc, ndalloc, + nrequests, curruns); } } if (gap_start != -1) @@ -321,17 +203,17 @@ stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, static void stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, - unsigned i) + unsigned i, bool bins, bool large) { unsigned nthreads; - size_t pagesize, pactive, pdirty, mapped; + size_t page, pactive, pdirty, mapped; uint64_t npurge, nmadvise, purged; size_t small_allocated; uint64_t small_nmalloc, small_ndalloc, small_nrequests; size_t large_allocated; uint64_t large_nmalloc, large_ndalloc, large_nrequests; - CTL_GET("arenas.pagesize", &pagesize, size_t); + CTL_GET("arenas.page", &page, size_t); CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned); malloc_cprintf(write_cb, cbopaque, @@ -369,15 +251,15 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, small_nmalloc + large_nmalloc, small_ndalloc + large_ndalloc, small_nrequests + large_nrequests); - malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", - pactive * pagesize ); + malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", pactive * page); CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); - stats_arena_bins_print(write_cb, cbopaque, i); - stats_arena_lruns_print(write_cb, cbopaque, i); + if (bins) + stats_arena_bins_print(write_cb, cbopaque, i); + if (large) + stats_arena_lruns_print(write_cb, cbopaque, i); } -#endif void stats_print(void (*write_cb)(void *, const char *), void *cbopaque, @@ -386,7 +268,6 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, int err; uint64_t epoch; size_t u64sz; - char s[UMAX2S_BUFSIZE]; bool general = true; bool merged = true; bool unmerged = true; @@ -402,8 +283,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, * */ epoch = 1; u64sz = sizeof(uint64_t); - err = JEMALLOC_P(mallctl)("epoch", &epoch, &u64sz, &epoch, - sizeof(uint64_t)); + err = je_mallctl("epoch", &epoch, &u64sz, &epoch, sizeof(uint64_t)); if (err != 0) { if (err == EAGAIN) { malloc_write(": Memory allocation failure in " @@ -415,42 +295,33 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, abort(); } - if (write_cb == NULL) { - /* - * The caller did not provide an alternate write_cb callback - * function, so use the default one. malloc_write() is an - * inline function, so use malloc_message() directly here. - */ - write_cb = JEMALLOC_P(malloc_message); - cbopaque = NULL; - } - if (opts != NULL) { unsigned i; for (i = 0; opts[i] != '\0'; i++) { switch (opts[i]) { - case 'g': - general = false; - break; - case 'm': - merged = false; - break; - case 'a': - unmerged = false; - break; - case 'b': - bins = false; - break; - case 'l': - large = false; - break; - default:; + case 'g': + general = false; + break; + case 'm': + merged = false; + break; + case 'a': + unmerged = false; + break; + case 'b': + bins = false; + break; + case 'l': + large = false; + break; + default:; } } } - write_cb(cbopaque, "___ Begin jemalloc statistics ___\n"); + malloc_cprintf(write_cb, cbopaque, + "___ Begin jemalloc statistics ___\n"); if (general) { int err; const char *cpv; @@ -465,229 +336,126 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cpsz = sizeof(const char *); CTL_GET("version", &cpv, const char *); - write_cb(cbopaque, "Version: "); - write_cb(cbopaque, cpv); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Version: %s\n", cpv); CTL_GET("config.debug", &bv, bool); - write_cb(cbopaque, "Assertions "); - write_cb(cbopaque, bv ? "enabled" : "disabled"); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Assertions %s\n", + bv ? "enabled" : "disabled"); #define OPT_WRITE_BOOL(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &bv, &bsz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, bv ? "true" : "false"); \ - write_cb(cbopaque, "\n"); \ + if ((err = je_mallctl("opt."#n, &bv, &bsz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %s\n", bv ? "true" : "false"); \ } #define OPT_WRITE_SIZE_T(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &sv, &ssz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, u2s(sv, 10, s)); \ - write_cb(cbopaque, "\n"); \ + if ((err = je_mallctl("opt."#n, &sv, &ssz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %zu\n", sv); \ } #define OPT_WRITE_SSIZE_T(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &ssv, &sssz, \ - NULL, 0)) == 0) { \ - if (ssv >= 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, u2s(ssv, 10, s)); \ - } else { \ - write_cb(cbopaque, " opt."#n": -"); \ - write_cb(cbopaque, u2s(-ssv, 10, s)); \ - } \ - write_cb(cbopaque, "\n"); \ + if ((err = je_mallctl("opt."#n, &ssv, &sssz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": %zd\n", ssv); \ } #define OPT_WRITE_CHAR_P(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &cpv, &cpsz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": \""); \ - write_cb(cbopaque, cpv); \ - write_cb(cbopaque, "\"\n"); \ + if ((err = je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0)) \ + == 0) { \ + malloc_cprintf(write_cb, cbopaque, \ + " opt."#n": \"%s\"\n", cpv); \ } - write_cb(cbopaque, "Run-time option settings:\n"); + malloc_cprintf(write_cb, cbopaque, + "Run-time option settings:\n"); OPT_WRITE_BOOL(abort) - OPT_WRITE_SIZE_T(lg_qspace_max) - OPT_WRITE_SIZE_T(lg_cspace_max) OPT_WRITE_SIZE_T(lg_chunk) OPT_WRITE_SIZE_T(narenas) OPT_WRITE_SSIZE_T(lg_dirty_mult) OPT_WRITE_BOOL(stats_print) OPT_WRITE_BOOL(junk) + OPT_WRITE_SIZE_T(quarantine) + OPT_WRITE_BOOL(redzone) OPT_WRITE_BOOL(zero) - OPT_WRITE_BOOL(sysv) + OPT_WRITE_BOOL(utrace) + OPT_WRITE_BOOL(valgrind) OPT_WRITE_BOOL(xmalloc) OPT_WRITE_BOOL(tcache) - OPT_WRITE_SSIZE_T(lg_tcache_gc_sweep) OPT_WRITE_SSIZE_T(lg_tcache_max) OPT_WRITE_BOOL(prof) OPT_WRITE_CHAR_P(prof_prefix) - OPT_WRITE_SIZE_T(lg_prof_bt_max) OPT_WRITE_BOOL(prof_active) OPT_WRITE_SSIZE_T(lg_prof_sample) OPT_WRITE_BOOL(prof_accum) - OPT_WRITE_SSIZE_T(lg_prof_tcmax) OPT_WRITE_SSIZE_T(lg_prof_interval) OPT_WRITE_BOOL(prof_gdump) + OPT_WRITE_BOOL(prof_final) OPT_WRITE_BOOL(prof_leak) - OPT_WRITE_BOOL(overcommit) #undef OPT_WRITE_BOOL #undef OPT_WRITE_SIZE_T #undef OPT_WRITE_SSIZE_T #undef OPT_WRITE_CHAR_P - write_cb(cbopaque, "CPUs: "); - write_cb(cbopaque, u2s(ncpus, 10, s)); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "CPUs: %u\n", ncpus); CTL_GET("arenas.narenas", &uv, unsigned); - write_cb(cbopaque, "Max arenas: "); - write_cb(cbopaque, u2s(uv, 10, s)); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Max arenas: %u\n", uv); - write_cb(cbopaque, "Pointer size: "); - write_cb(cbopaque, u2s(sizeof(void *), 10, s)); - write_cb(cbopaque, "\n"); + malloc_cprintf(write_cb, cbopaque, "Pointer size: %zu\n", + sizeof(void *)); CTL_GET("arenas.quantum", &sv, size_t); - write_cb(cbopaque, "Quantum size: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.cacheline", &sv, size_t); - write_cb(cbopaque, "Cacheline size (assumed): "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.subpage", &sv, size_t); - write_cb(cbopaque, "Subpage spacing: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz, - NULL, 0)) == 0) { - write_cb(cbopaque, "Tiny 2^n-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - - CTL_GET("arenas.tspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - } + malloc_cprintf(write_cb, cbopaque, "Quantum size: %zu\n", sv); - CTL_GET("arenas.qspace_min", &sv, size_t); - write_cb(cbopaque, "Quantum-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.qspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("arenas.cspace_min", &sv, size_t); - write_cb(cbopaque, "Cacheline-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.cspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("arenas.sspace_min", &sv, size_t); - write_cb(cbopaque, "Subpage-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.sspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); + CTL_GET("arenas.page", &sv, size_t); + malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv); CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t); if (ssv >= 0) { - write_cb(cbopaque, - "Min active:dirty page ratio per arena: "); - write_cb(cbopaque, u2s((1U << ssv), 10, s)); - write_cb(cbopaque, ":1\n"); + malloc_cprintf(write_cb, cbopaque, + "Min active:dirty page ratio per arena: %u:1\n", + (1U << ssv)); } else { - write_cb(cbopaque, + malloc_cprintf(write_cb, cbopaque, "Min active:dirty page ratio per arena: N/A\n"); } - if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv, - &ssz, NULL, 0)) == 0) { - write_cb(cbopaque, - "Maximum thread-cached size class: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - } - if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv, - &ssz, NULL, 0)) == 0) { - size_t tcache_gc_sweep = (1U << ssv); - bool tcache_enabled; - CTL_GET("opt.tcache", &tcache_enabled, bool); - write_cb(cbopaque, "Thread cache GC sweep interval: "); - write_cb(cbopaque, tcache_enabled && ssv >= 0 ? - u2s(tcache_gc_sweep, 10, s) : "N/A"); - write_cb(cbopaque, "\n"); + if ((err = je_mallctl("arenas.tcache_max", &sv, &ssz, NULL, 0)) + == 0) { + malloc_cprintf(write_cb, cbopaque, + "Maximum thread-cached size class: %zu\n", sv); } - if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0)) - == 0 && bv) { - CTL_GET("opt.lg_prof_bt_max", &sv, size_t); - write_cb(cbopaque, "Maximum profile backtrace depth: "); - write_cb(cbopaque, u2s((1U << sv), 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("opt.lg_prof_tcmax", &ssv, ssize_t); - write_cb(cbopaque, - "Maximum per thread backtrace cache: "); - if (ssv >= 0) { - write_cb(cbopaque, u2s((1U << ssv), 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(ssv, 10, s)); - write_cb(cbopaque, ")\n"); - } else - write_cb(cbopaque, "N/A\n"); - + if ((err = je_mallctl("opt.prof", &bv, &bsz, NULL, 0)) == 0 && + bv) { CTL_GET("opt.lg_prof_sample", &sv, size_t); - write_cb(cbopaque, "Average profile sample interval: "); - write_cb(cbopaque, u2s((((uint64_t)1U) << sv), 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ")\n"); + malloc_cprintf(write_cb, cbopaque, + "Average profile sample interval: %"PRIu64 + " (2^%zu)\n", (((uint64_t)1U) << sv), sv); CTL_GET("opt.lg_prof_interval", &ssv, ssize_t); - write_cb(cbopaque, "Average profile dump interval: "); if (ssv >= 0) { - write_cb(cbopaque, u2s((((uint64_t)1U) << ssv), - 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(ssv, 10, s)); - write_cb(cbopaque, ")\n"); - } else - write_cb(cbopaque, "N/A\n"); + malloc_cprintf(write_cb, cbopaque, + "Average profile dump interval: %"PRIu64 + " (2^%zd)\n", + (((uint64_t)1U) << ssv), ssv); + } else { + malloc_cprintf(write_cb, cbopaque, + "Average profile dump interval: N/A\n"); + } } - CTL_GET("arenas.chunksize", &sv, size_t); - write_cb(cbopaque, "Chunk size: "); - write_cb(cbopaque, u2s(sv, 10, s)); CTL_GET("opt.lg_chunk", &sv, size_t); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ")\n"); + malloc_cprintf(write_cb, cbopaque, "Chunk size: %zu (2^%zu)\n", + (ZU(1) << sv), sv); } -#ifdef JEMALLOC_STATS - { - int err; - size_t sszp, ssz; + if (config_stats) { size_t *cactive; size_t allocated, active, mapped; - size_t chunks_current, chunks_high, swap_avail; + size_t chunks_current, chunks_high; uint64_t chunks_total; size_t huge_allocated; uint64_t huge_nmalloc, huge_ndalloc; - sszp = sizeof(size_t *); - ssz = sizeof(size_t); - CTL_GET("stats.cactive", &cactive, size_t *); CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); @@ -702,24 +470,10 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("stats.chunks.total", &chunks_total, uint64_t); CTL_GET("stats.chunks.high", &chunks_high, size_t); CTL_GET("stats.chunks.current", &chunks_current, size_t); - if ((err = JEMALLOC_P(mallctl)("swap.avail", &swap_avail, &ssz, - NULL, 0)) == 0) { - size_t lg_chunk; - - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks swap_avail\n"); - CTL_GET("opt.lg_chunk", &lg_chunk, size_t); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64"%13zu%13zu%13zu\n", - chunks_total, chunks_high, chunks_current, - swap_avail << lg_chunk); - } else { - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks\n"); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64"%13zu%13zu\n", - chunks_total, chunks_high, chunks_current); - } + malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " + "highchunks curchunks\n"); + malloc_cprintf(write_cb, cbopaque, " %13"PRIu64"%13zu%13zu\n", + chunks_total, chunks_high, chunks_current); /* Print huge stats. */ CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t); @@ -736,11 +490,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.narenas", &narenas, unsigned); { - bool initialized[narenas]; + VARIABLE_ARRAY(bool, initialized, narenas); size_t isz; unsigned i, ninitialized; - isz = sizeof(initialized); + isz = sizeof(bool) * narenas; xmallctl("arenas.initialized", initialized, &isz, NULL, 0); for (i = ninitialized = 0; i < narenas; i++) { @@ -753,7 +507,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "\nMerged arenas stats:\n"); stats_arena_print(write_cb, cbopaque, - narenas); + narenas, bins, large); } } } @@ -765,11 +519,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.narenas", &narenas, unsigned); { - bool initialized[narenas]; + VARIABLE_ARRAY(bool, initialized, narenas); size_t isz; unsigned i; - isz = sizeof(initialized); + isz = sizeof(bool) * narenas; xmallctl("arenas.initialized", initialized, &isz, NULL, 0); @@ -779,12 +533,11 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, cbopaque, "\narenas[%u]:\n", i); stats_arena_print(write_cb, - cbopaque, i); + cbopaque, i, bins, large); } } } } } -#endif /* #ifdef JEMALLOC_STATS */ - write_cb(cbopaque, "--- End jemalloc statistics ---\n"); + malloc_cprintf(write_cb, cbopaque, "--- End jemalloc statistics ---\n"); } diff --git a/deps/jemalloc/src/tcache.c b/deps/jemalloc/src/tcache.c index 31c329e1613..60244c45f8b 100644 --- a/deps/jemalloc/src/tcache.c +++ b/deps/jemalloc/src/tcache.c @@ -1,70 +1,92 @@ #define JEMALLOC_TCACHE_C_ #include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_TCACHE + /******************************************************************************/ /* Data. */ +malloc_tsd_data(, tcache, tcache_t *, NULL) +malloc_tsd_data(, tcache_enabled, tcache_enabled_t, tcache_enabled_default) + bool opt_tcache = true; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; -ssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; tcache_bin_info_t *tcache_bin_info; static unsigned stack_nelms; /* Total stack elms per tcache. */ -/* Map of thread-specific caches. */ -#ifndef NO_TLS -__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); -#endif +size_t nhbins; +size_t tcache_maxclass; -/* - * Same contents as tcache, but initialized such that the TSD destructor is - * called when a thread exits, so that the cache can be cleaned up. - */ -pthread_key_t tcache_tsd; +/******************************************************************************/ -size_t nhbins; -size_t tcache_maxclass; -unsigned tcache_gc_incr; +size_t tcache_salloc(const void *ptr) +{ -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ + return (arena_salloc(ptr, false)); +} -static void tcache_thread_cleanup(void *arg); +void +tcache_event_hard(tcache_t *tcache) +{ + size_t binind = tcache->next_gc_bin; + tcache_bin_t *tbin = &tcache->tbins[binind]; + tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; -/******************************************************************************/ + if (tbin->low_water > 0) { + /* + * Flush (ceiling) 3/4 of the objects below the low water mark. + */ + if (binind < NBINS) { + tcache_bin_flush_small(tbin, binind, tbin->ncached - + tbin->low_water + (tbin->low_water >> 2), tcache); + } else { + tcache_bin_flush_large(tbin, binind, tbin->ncached - + tbin->low_water + (tbin->low_water >> 2), tcache); + } + /* + * Reduce fill count by 2X. Limit lg_fill_div such that the + * fill count is always at least 1. + */ + if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) >= 1) + tbin->lg_fill_div++; + } else if (tbin->low_water < 0) { + /* + * Increase fill count by 2X. Make sure lg_fill_div stays + * greater than 0. + */ + if (tbin->lg_fill_div > 1) + tbin->lg_fill_div--; + } + tbin->low_water = tbin->ncached; + + tcache->next_gc_bin++; + if (tcache->next_gc_bin == nhbins) + tcache->next_gc_bin = 0; + tcache->ev_cnt = 0; +} void * tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) { void *ret; - arena_tcache_fill_small(tcache->arena, tbin, binind -#ifdef JEMALLOC_PROF - , tcache->prof_accumbytes -#endif - ); -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes = 0; -#endif + arena_tcache_fill_small(tcache->arena, tbin, binind, + config_prof ? tcache->prof_accumbytes : 0); + if (config_prof) + tcache->prof_accumbytes = 0; ret = tcache_alloc_easy(tbin); return (ret); } void -tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ) +tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache) { void *ptr; unsigned i, nflush, ndeferred; -#ifdef JEMALLOC_STATS bool merged_stats = false; -#endif - assert(binind < nbins); + assert(binind < NBINS); assert(rem <= tbin->ncached); for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { @@ -74,25 +96,21 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem arena_t *arena = chunk->arena; arena_bin_t *bin = &arena->bins[binind]; -#ifdef JEMALLOC_PROF - if (arena == tcache->arena) { + if (config_prof && arena == tcache->arena) { malloc_mutex_lock(&arena->lock); arena_prof_accum(arena, tcache->prof_accumbytes); malloc_mutex_unlock(&arena->lock); tcache->prof_accumbytes = 0; } -#endif malloc_mutex_lock(&bin->lock); -#ifdef JEMALLOC_STATS - if (arena == tcache->arena) { + if (config_stats && arena == tcache->arena) { assert(merged_stats == false); merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; } -#endif ndeferred = 0; for (i = 0; i < nflush; i++) { ptr = tbin->avail[i]; @@ -100,10 +118,15 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk->arena == arena) { size_t pageind = ((uintptr_t)ptr - - (uintptr_t)chunk) >> PAGE_SHIFT; + (uintptr_t)chunk) >> LG_PAGE; arena_chunk_map_t *mapelm = - &chunk->map[pageind-map_bias]; - arena_dalloc_bin(arena, chunk, ptr, mapelm); + arena_mapp_get(chunk, pageind); + if (config_fill && opt_junk) { + arena_alloc_junk_small(ptr, + &arena_bin_info[binind], true); + } + arena_dalloc_bin_locked(arena, chunk, ptr, + mapelm); } else { /* * This object was allocated via a different @@ -117,8 +140,7 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem } malloc_mutex_unlock(&bin->lock); } -#ifdef JEMALLOC_STATS - if (merged_stats == false) { + if (config_stats && merged_stats == false) { /* * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. @@ -130,7 +152,6 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem tbin->tstats.nrequests = 0; malloc_mutex_unlock(&bin->lock); } -#endif memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], rem * sizeof(void *)); @@ -140,17 +161,12 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem } void -tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ) +tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, + tcache_t *tcache) { void *ptr; unsigned i, nflush, ndeferred; -#ifdef JEMALLOC_STATS bool merged_stats = false; -#endif assert(binind < nhbins); assert(rem <= tbin->ncached); @@ -162,30 +178,28 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem arena_t *arena = chunk->arena; malloc_mutex_lock(&arena->lock); -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - if (arena == tcache->arena) { -#endif -#ifdef JEMALLOC_PROF - arena_prof_accum(arena, tcache->prof_accumbytes); - tcache->prof_accumbytes = 0; -#endif -#ifdef JEMALLOC_STATS - merged_stats = true; - arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[binind - nbins].nrequests += - tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; -#endif -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) + if ((config_prof || config_stats) && arena == tcache->arena) { + if (config_prof) { + arena_prof_accum(arena, + tcache->prof_accumbytes); + tcache->prof_accumbytes = 0; + } + if (config_stats) { + merged_stats = true; + arena->stats.nrequests_large += + tbin->tstats.nrequests; + arena->stats.lstats[binind - NBINS].nrequests += + tbin->tstats.nrequests; + tbin->tstats.nrequests = 0; + } } -#endif ndeferred = 0; for (i = 0; i < nflush; i++) { ptr = tbin->avail[i]; assert(ptr != NULL); chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk->arena == arena) - arena_dalloc_large(arena, chunk, ptr); + arena_dalloc_large_locked(arena, chunk, ptr); else { /* * This object was allocated via a different @@ -199,8 +213,7 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem } malloc_mutex_unlock(&arena->lock); } -#ifdef JEMALLOC_STATS - if (merged_stats == false) { + if (config_stats && merged_stats == false) { /* * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. @@ -208,12 +221,11 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem arena_t *arena = tcache->arena; malloc_mutex_lock(&arena->lock); arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[binind - nbins].nrequests += + arena->stats.lstats[binind - NBINS].nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; malloc_mutex_unlock(&arena->lock); } -#endif memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], rem * sizeof(void *)); @@ -222,6 +234,33 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem tbin->low_water = tbin->ncached; } +void +tcache_arena_associate(tcache_t *tcache, arena_t *arena) +{ + + if (config_stats) { + /* Link into list of extant tcaches. */ + malloc_mutex_lock(&arena->lock); + ql_elm_new(tcache, link); + ql_tail_insert(&arena->tcache_ql, tcache, link); + malloc_mutex_unlock(&arena->lock); + } + tcache->arena = arena; +} + +void +tcache_arena_dissociate(tcache_t *tcache) +{ + + if (config_stats) { + /* Unlink from list of extant tcaches. */ + malloc_mutex_lock(&tcache->arena->lock); + ql_remove(&tcache->arena->tcache_ql, tcache, link); + malloc_mutex_unlock(&tcache->arena->lock); + tcache_stats_merge(tcache, tcache->arena); + } +} + tcache_t * tcache_create(arena_t *arena) { @@ -244,7 +283,7 @@ tcache_create(arena_t *arena) */ size = (size + CACHELINE_MASK) & (-CACHELINE); - if (size <= small_maxclass) + if (size <= SMALL_MAXCLASS) tcache = (tcache_t *)arena_malloc_small(arena, size, true); else if (size <= tcache_maxclass) tcache = (tcache_t *)arena_malloc_large(arena, size, true); @@ -254,15 +293,8 @@ tcache_create(arena_t *arena) if (tcache == NULL) return (NULL); -#ifdef JEMALLOC_STATS - /* Link into list of extant tcaches. */ - malloc_mutex_lock(&arena->lock); - ql_elm_new(tcache, link); - ql_tail_insert(&arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&arena->lock); -#endif + tcache_arena_associate(tcache, arena); - tcache->arena = arena; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); for (i = 0; i < nhbins; i++) { tcache->tbins[i].lg_fill_div = 1; @@ -271,7 +303,7 @@ tcache_create(arena_t *arena) stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); } - TCACHE_SET(tcache); + tcache_tsd_set(&tcache); return (tcache); } @@ -282,121 +314,96 @@ tcache_destroy(tcache_t *tcache) unsigned i; size_t tcache_size; -#ifdef JEMALLOC_STATS - /* Unlink from list of extant tcaches. */ - malloc_mutex_lock(&tcache->arena->lock); - ql_remove(&tcache->arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&tcache->arena->lock); - tcache_stats_merge(tcache, tcache->arena); -#endif + tcache_arena_dissociate(tcache); - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_small(tbin, i, 0 -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - -#ifdef JEMALLOC_STATS - if (tbin->tstats.nrequests != 0) { + tcache_bin_flush_small(tbin, i, 0, tcache); + + if (config_stats && tbin->tstats.nrequests != 0) { arena_t *arena = tcache->arena; arena_bin_t *bin = &arena->bins[i]; malloc_mutex_lock(&bin->lock); bin->stats.nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(&bin->lock); } -#endif } for (; i < nhbins; i++) { tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_large(tbin, i, 0 -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - -#ifdef JEMALLOC_STATS - if (tbin->tstats.nrequests != 0) { + tcache_bin_flush_large(tbin, i, 0, tcache); + + if (config_stats && tbin->tstats.nrequests != 0) { arena_t *arena = tcache->arena; malloc_mutex_lock(&arena->lock); arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[i - nbins].nrequests += + arena->stats.lstats[i - NBINS].nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(&arena->lock); } -#endif } -#ifdef JEMALLOC_PROF - if (tcache->prof_accumbytes > 0) { + if (config_prof && tcache->prof_accumbytes > 0) { malloc_mutex_lock(&tcache->arena->lock); arena_prof_accum(tcache->arena, tcache->prof_accumbytes); malloc_mutex_unlock(&tcache->arena->lock); } -#endif - tcache_size = arena_salloc(tcache); - if (tcache_size <= small_maxclass) { + tcache_size = arena_salloc(tcache, false); + if (tcache_size <= SMALL_MAXCLASS) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_t *arena = chunk->arena; size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> - PAGE_SHIFT; - arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - arena_bin_t *bin = run->bin; + LG_PAGE; + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); - malloc_mutex_lock(&bin->lock); - arena_dalloc_bin(arena, chunk, tcache, mapelm); - malloc_mutex_unlock(&bin->lock); + arena_dalloc_bin(arena, chunk, tcache, pageind, mapelm); } else if (tcache_size <= tcache_maxclass) { arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); arena_t *arena = chunk->arena; - malloc_mutex_lock(&arena->lock); arena_dalloc_large(arena, chunk, tcache); - malloc_mutex_unlock(&arena->lock); } else idalloc(tcache); } -static void +void tcache_thread_cleanup(void *arg) { - tcache_t *tcache = (tcache_t *)arg; + tcache_t *tcache = *(tcache_t **)arg; - if (tcache == (void *)(uintptr_t)1) { + if (tcache == TCACHE_STATE_DISABLED) { + /* Do nothing. */ + } else if (tcache == TCACHE_STATE_REINCARNATED) { /* - * The previous time this destructor was called, we set the key - * to 1 so that other destructors wouldn't cause re-creation of - * the tcache. This time, do nothing, so that the destructor - * will not be called again. + * Another destructor called an allocator function after this + * destructor was called. Reset tcache to + * TCACHE_STATE_PURGATORY in order to receive another callback. */ - } else if (tcache == (void *)(uintptr_t)2) { + tcache = TCACHE_STATE_PURGATORY; + tcache_tsd_set(&tcache); + } else if (tcache == TCACHE_STATE_PURGATORY) { /* - * Another destructor called an allocator function after this - * destructor was called. Reset tcache to 1 in order to - * receive another callback. + * The previous time this destructor was called, we set the key + * to TCACHE_STATE_PURGATORY so that other destructors wouldn't + * cause re-creation of the tcache. This time, do nothing, so + * that the destructor will not be called again. */ - TCACHE_SET((uintptr_t)1); } else if (tcache != NULL) { - assert(tcache != (void *)(uintptr_t)1); + assert(tcache != TCACHE_STATE_PURGATORY); tcache_destroy(tcache); - TCACHE_SET((uintptr_t)1); + tcache = TCACHE_STATE_PURGATORY; + tcache_tsd_set(&tcache); } } -#ifdef JEMALLOC_STATS void tcache_stats_merge(tcache_t *tcache, arena_t *arena) { unsigned i; /* Merge and reset tcache stats. */ - for (i = 0; i < nbins; i++) { + for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; tcache_bin_t *tbin = &tcache->tbins[i]; malloc_mutex_lock(&bin->lock); @@ -406,75 +413,62 @@ tcache_stats_merge(tcache_t *tcache, arena_t *arena) } for (; i < nhbins; i++) { - malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; + malloc_large_stats_t *lstats = &arena->stats.lstats[i - NBINS]; tcache_bin_t *tbin = &tcache->tbins[i]; arena->stats.nrequests_large += tbin->tstats.nrequests; lstats->nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; } } -#endif bool -tcache_boot(void) +tcache_boot0(void) { + unsigned i; - if (opt_tcache) { - unsigned i; - - /* - * If necessary, clamp opt_lg_tcache_max, now that - * small_maxclass and arena_maxclass are known. - */ - if (opt_lg_tcache_max < 0 || (1U << - opt_lg_tcache_max) < small_maxclass) - tcache_maxclass = small_maxclass; - else if ((1U << opt_lg_tcache_max) > arena_maxclass) - tcache_maxclass = arena_maxclass; - else - tcache_maxclass = (1U << opt_lg_tcache_max); - - nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); - - /* Initialize tcache_bin_info. */ - tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * - sizeof(tcache_bin_info_t)); - if (tcache_bin_info == NULL) - return (true); - stack_nelms = 0; - for (i = 0; i < nbins; i++) { - if ((arena_bin_info[i].nregs << 1) <= - TCACHE_NSLOTS_SMALL_MAX) { - tcache_bin_info[i].ncached_max = - (arena_bin_info[i].nregs << 1); - } else { - tcache_bin_info[i].ncached_max = - TCACHE_NSLOTS_SMALL_MAX; - } - stack_nelms += tcache_bin_info[i].ncached_max; - } - for (; i < nhbins; i++) { - tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; - stack_nelms += tcache_bin_info[i].ncached_max; - } - - /* Compute incremental GC event threshold. */ - if (opt_lg_tcache_gc_sweep >= 0) { - tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / - nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == - 0) ? 0 : 1); - } else - tcache_gc_incr = 0; - - if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != - 0) { - malloc_write( - ": Error in pthread_key_create()\n"); - abort(); + /* + * If necessary, clamp opt_lg_tcache_max, now that arena_maxclass is + * known. + */ + if (opt_lg_tcache_max < 0 || (1U << opt_lg_tcache_max) < SMALL_MAXCLASS) + tcache_maxclass = SMALL_MAXCLASS; + else if ((1U << opt_lg_tcache_max) > arena_maxclass) + tcache_maxclass = arena_maxclass; + else + tcache_maxclass = (1U << opt_lg_tcache_max); + + nhbins = NBINS + (tcache_maxclass >> LG_PAGE); + + /* Initialize tcache_bin_info. */ + tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * + sizeof(tcache_bin_info_t)); + if (tcache_bin_info == NULL) + return (true); + stack_nelms = 0; + for (i = 0; i < NBINS; i++) { + if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) { + tcache_bin_info[i].ncached_max = + (arena_bin_info[i].nregs << 1); + } else { + tcache_bin_info[i].ncached_max = + TCACHE_NSLOTS_SMALL_MAX; } + stack_nelms += tcache_bin_info[i].ncached_max; + } + for (; i < nhbins; i++) { + tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; + stack_nelms += tcache_bin_info[i].ncached_max; } return (false); } -/******************************************************************************/ -#endif /* JEMALLOC_TCACHE */ + +bool +tcache_boot1(void) +{ + + if (tcache_tsd_boot() || tcache_enabled_tsd_boot()) + return (true); + + return (false); +} diff --git a/deps/jemalloc/src/tsd.c b/deps/jemalloc/src/tsd.c new file mode 100644 index 00000000000..961a546329c --- /dev/null +++ b/deps/jemalloc/src/tsd.c @@ -0,0 +1,107 @@ +#define JEMALLOC_TSD_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Data. */ + +static unsigned ncleanups; +static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; + +/******************************************************************************/ + +void * +malloc_tsd_malloc(size_t size) +{ + + /* Avoid choose_arena() in order to dodge bootstrapping issues. */ + return (arena_malloc(arenas[0], size, false, false)); +} + +void +malloc_tsd_dalloc(void *wrapper) +{ + + idalloc(wrapper); +} + +void +malloc_tsd_no_cleanup(void *arg) +{ + + not_reached(); +} + +#if defined(JEMALLOC_MALLOC_THREAD_CLEANUP) || defined(_WIN32) +#ifndef _WIN32 +JEMALLOC_EXPORT +#endif +void +_malloc_thread_cleanup(void) +{ + bool pending[MALLOC_TSD_CLEANUPS_MAX], again; + unsigned i; + + for (i = 0; i < ncleanups; i++) + pending[i] = true; + + do { + again = false; + for (i = 0; i < ncleanups; i++) { + if (pending[i]) { + pending[i] = cleanups[i](); + if (pending[i]) + again = true; + } + } + } while (again); +} +#endif + +void +malloc_tsd_cleanup_register(bool (*f)(void)) +{ + + assert(ncleanups < MALLOC_TSD_CLEANUPS_MAX); + cleanups[ncleanups] = f; + ncleanups++; +} + +void +malloc_tsd_boot(void) +{ + + ncleanups = 0; +} + +#ifdef _WIN32 +static BOOL WINAPI +_tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) +{ + + switch (fdwReason) { +#ifdef JEMALLOC_LAZY_LOCK + case DLL_THREAD_ATTACH: + isthreaded = true; + break; +#endif + case DLL_THREAD_DETACH: + _malloc_thread_cleanup(); + break; + default: + break; + } + return (true); +} + +#ifdef _MSC_VER +# ifdef _M_IX86 +# pragma comment(linker, "/INCLUDE:__tls_used") +# else +# pragma comment(linker, "/INCLUDE:_tls_used") +# endif +# pragma section(".CRT$XLY",long,read) +#endif +JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) +static const BOOL (WINAPI *tls_callback)(HINSTANCE hinstDLL, + DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; +#endif diff --git a/deps/jemalloc/src/util.c b/deps/jemalloc/src/util.c new file mode 100644 index 00000000000..9b73c3ec045 --- /dev/null +++ b/deps/jemalloc/src/util.c @@ -0,0 +1,646 @@ +#define assert(e) do { \ + if (config_debug && !(e)) { \ + malloc_write(": Failed assertion\n"); \ + abort(); \ + } \ +} while (0) + +#define not_reached() do { \ + if (config_debug) { \ + malloc_write(": Unreachable code reached\n"); \ + abort(); \ + } \ +} while (0) + +#define not_implemented() do { \ + if (config_debug) { \ + malloc_write(": Not implemented\n"); \ + abort(); \ + } \ +} while (0) + +#define JEMALLOC_UTIL_C_ +#include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* Function prototypes for non-inline static functions. */ + +static void wrtmessage(void *cbopaque, const char *s); +#define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) +static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, + size_t *slen_p); +#define D2S_BUFSIZE (1 + U2S_BUFSIZE) +static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); +#define O2S_BUFSIZE (1 + U2S_BUFSIZE) +static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); +#define X2S_BUFSIZE (2 + U2S_BUFSIZE) +static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, + size_t *slen_p); + +/******************************************************************************/ + +/* malloc_message() setup. */ +static void +wrtmessage(void *cbopaque, const char *s) +{ + +#ifdef SYS_write + /* + * Use syscall(2) rather than write(2) when possible in order to avoid + * the possibility of memory allocation within libc. This is necessary + * on FreeBSD; most operating systems do not have this problem though. + */ + UNUSED int result = syscall(SYS_write, STDERR_FILENO, s, strlen(s)); +#else + UNUSED int result = write(STDERR_FILENO, s, strlen(s)); +#endif +} + +JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); + +/* + * Wrapper around malloc_message() that avoids the need for + * je_malloc_message(...) throughout the code. + */ +void +malloc_write(const char *s) +{ + + if (je_malloc_message != NULL) + je_malloc_message(NULL, s); + else + wrtmessage(NULL, s); +} + +/* + * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so + * provide a wrapper. + */ +int +buferror(char *buf, size_t buflen) +{ + +#ifdef _WIN32 + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(), 0, + (LPSTR)buf, buflen, NULL); + return (0); +#elif defined(_GNU_SOURCE) + char *b = strerror_r(errno, buf, buflen); + if (b != buf) { + strncpy(buf, b, buflen); + buf[buflen-1] = '\0'; + } + return (0); +#else + return (strerror_r(errno, buf, buflen)); +#endif +} + +uintmax_t +malloc_strtoumax(const char *nptr, char **endptr, int base) +{ + uintmax_t ret, digit; + int b; + bool neg; + const char *p, *ns; + + if (base < 0 || base == 1 || base > 36) { + set_errno(EINVAL); + return (UINTMAX_MAX); + } + b = base; + + /* Swallow leading whitespace and get sign, if any. */ + neg = false; + p = nptr; + while (true) { + switch (*p) { + case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': + p++; + break; + case '-': + neg = true; + /* Fall through. */ + case '+': + p++; + /* Fall through. */ + default: + goto label_prefix; + } + } + + /* Get prefix, if any. */ + label_prefix: + /* + * Note where the first non-whitespace/sign character is so that it is + * possible to tell whether any digits are consumed (e.g., " 0" vs. + * " -x"). + */ + ns = p; + if (*p == '0') { + switch (p[1]) { + case '0': case '1': case '2': case '3': case '4': case '5': + case '6': case '7': + if (b == 0) + b = 8; + if (b == 8) + p++; + break; + case 'x': + switch (p[2]) { + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + case 'A': case 'B': case 'C': case 'D': case 'E': + case 'F': + case 'a': case 'b': case 'c': case 'd': case 'e': + case 'f': + if (b == 0) + b = 16; + if (b == 16) + p += 2; + break; + default: + break; + } + break; + default: + break; + } + } + if (b == 0) + b = 10; + + /* Convert. */ + ret = 0; + while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) + || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) + || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { + uintmax_t pret = ret; + ret *= b; + ret += digit; + if (ret < pret) { + /* Overflow. */ + set_errno(ERANGE); + return (UINTMAX_MAX); + } + p++; + } + if (neg) + ret = -ret; + + if (endptr != NULL) { + if (p == ns) { + /* No characters were converted. */ + *endptr = (char *)nptr; + } else + *endptr = (char *)p; + } + + return (ret); +} + +static char * +u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) +{ + unsigned i; + + i = U2S_BUFSIZE - 1; + s[i] = '\0'; + switch (base) { + case 10: + do { + i--; + s[i] = "0123456789"[x % (uint64_t)10]; + x /= (uint64_t)10; + } while (x > 0); + break; + case 16: { + const char *digits = (uppercase) + ? "0123456789ABCDEF" + : "0123456789abcdef"; + + do { + i--; + s[i] = digits[x & 0xf]; + x >>= 4; + } while (x > 0); + break; + } default: { + const char *digits = (uppercase) + ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + : "0123456789abcdefghijklmnopqrstuvwxyz"; + + assert(base >= 2 && base <= 36); + do { + i--; + s[i] = digits[x % (uint64_t)base]; + x /= (uint64_t)base; + } while (x > 0); + }} + + *slen_p = U2S_BUFSIZE - 1 - i; + return (&s[i]); +} + +static char * +d2s(intmax_t x, char sign, char *s, size_t *slen_p) +{ + bool neg; + + if ((neg = (x < 0))) + x = -x; + s = u2s(x, 10, false, s, slen_p); + if (neg) + sign = '-'; + switch (sign) { + case '-': + if (neg == false) + break; + /* Fall through. */ + case ' ': + case '+': + s--; + (*slen_p)++; + *s = sign; + break; + default: not_reached(); + } + return (s); +} + +static char * +o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) +{ + + s = u2s(x, 8, false, s, slen_p); + if (alt_form && *s != '0') { + s--; + (*slen_p)++; + *s = '0'; + } + return (s); +} + +static char * +x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) +{ + + s = u2s(x, 16, uppercase, s, slen_p); + if (alt_form) { + s -= 2; + (*slen_p) += 2; + memcpy(s, uppercase ? "0X" : "0x", 2); + } + return (s); +} + +int +malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) +{ + int ret; + size_t i; + const char *f; + +#define APPEND_C(c) do { \ + if (i < size) \ + str[i] = (c); \ + i++; \ +} while (0) +#define APPEND_S(s, slen) do { \ + if (i < size) { \ + size_t cpylen = (slen <= size - i) ? slen : size - i; \ + memcpy(&str[i], s, cpylen); \ + } \ + i += slen; \ +} while (0) +#define APPEND_PADDED_S(s, slen, width, left_justify) do { \ + /* Left padding. */ \ + size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ + (size_t)width - slen : 0); \ + if (left_justify == false && pad_len != 0) { \ + size_t j; \ + for (j = 0; j < pad_len; j++) \ + APPEND_C(' '); \ + } \ + /* Value. */ \ + APPEND_S(s, slen); \ + /* Right padding. */ \ + if (left_justify && pad_len != 0) { \ + size_t j; \ + for (j = 0; j < pad_len; j++) \ + APPEND_C(' '); \ + } \ +} while (0) +#define GET_ARG_NUMERIC(val, len) do { \ + switch (len) { \ + case '?': \ + val = va_arg(ap, int); \ + break; \ + case '?' | 0x80: \ + val = va_arg(ap, unsigned int); \ + break; \ + case 'l': \ + val = va_arg(ap, long); \ + break; \ + case 'l' | 0x80: \ + val = va_arg(ap, unsigned long); \ + break; \ + case 'q': \ + val = va_arg(ap, long long); \ + break; \ + case 'q' | 0x80: \ + val = va_arg(ap, unsigned long long); \ + break; \ + case 'j': \ + val = va_arg(ap, intmax_t); \ + break; \ + case 't': \ + val = va_arg(ap, ptrdiff_t); \ + break; \ + case 'z': \ + val = va_arg(ap, ssize_t); \ + break; \ + case 'z' | 0x80: \ + val = va_arg(ap, size_t); \ + break; \ + case 'p': /* Synthetic; used for %p. */ \ + val = va_arg(ap, uintptr_t); \ + break; \ + default: not_reached(); \ + } \ +} while (0) + + i = 0; + f = format; + while (true) { + switch (*f) { + case '\0': goto label_out; + case '%': { + bool alt_form = false; + bool zero_pad = false; + bool left_justify = false; + bool plus_space = false; + bool plus_plus = false; + int prec = -1; + int width = -1; + unsigned char len = '?'; + + f++; + if (*f == '%') { + /* %% */ + APPEND_C(*f); + break; + } + /* Flags. */ + while (true) { + switch (*f) { + case '#': + assert(alt_form == false); + alt_form = true; + break; + case '0': + assert(zero_pad == false); + zero_pad = true; + break; + case '-': + assert(left_justify == false); + left_justify = true; + break; + case ' ': + assert(plus_space == false); + plus_space = true; + break; + case '+': + assert(plus_plus == false); + plus_plus = true; + break; + default: goto label_width; + } + f++; + } + /* Width. */ + label_width: + switch (*f) { + case '*': + width = va_arg(ap, int); + f++; + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + uintmax_t uwidth; + set_errno(0); + uwidth = malloc_strtoumax(f, (char **)&f, 10); + assert(uwidth != UINTMAX_MAX || get_errno() != + ERANGE); + width = (int)uwidth; + if (*f == '.') { + f++; + goto label_precision; + } else + goto label_length; + break; + } case '.': + f++; + goto label_precision; + default: goto label_length; + } + /* Precision. */ + label_precision: + switch (*f) { + case '*': + prec = va_arg(ap, int); + f++; + break; + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': { + uintmax_t uprec; + set_errno(0); + uprec = malloc_strtoumax(f, (char **)&f, 10); + assert(uprec != UINTMAX_MAX || get_errno() != + ERANGE); + prec = (int)uprec; + break; + } + default: break; + } + /* Length. */ + label_length: + switch (*f) { + case 'l': + f++; + if (*f == 'l') { + len = 'q'; + f++; + } else + len = 'l'; + break; + case 'j': + len = 'j'; + f++; + break; + case 't': + len = 't'; + f++; + break; + case 'z': + len = 'z'; + f++; + break; + default: break; + } + /* Conversion specifier. */ + switch (*f) { + char *s; + size_t slen; + case 'd': case 'i': { + intmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[D2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len); + s = d2s(val, (plus_plus ? '+' : (plus_space ? + ' ' : '-')), buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'o': { + uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[O2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len | 0x80); + s = o2s(val, alt_form, buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'u': { + uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[U2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len | 0x80); + s = u2s(val, 10, false, buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'x': case 'X': { + uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); + char buf[X2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, len | 0x80); + s = x2s(val, alt_form, *f == 'X', buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } case 'c': { + unsigned char val; + char buf[2]; + + assert(len == '?' || len == 'l'); + assert_not_implemented(len != 'l'); + val = va_arg(ap, int); + buf[0] = val; + buf[1] = '\0'; + APPEND_PADDED_S(buf, 1, width, left_justify); + f++; + break; + } case 's': + assert(len == '?' || len == 'l'); + assert_not_implemented(len != 'l'); + s = va_arg(ap, char *); + slen = (prec == -1) ? strlen(s) : prec; + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + case 'p': { + uintmax_t val; + char buf[X2S_BUFSIZE]; + + GET_ARG_NUMERIC(val, 'p'); + s = x2s(val, true, false, buf, &slen); + APPEND_PADDED_S(s, slen, width, left_justify); + f++; + break; + } + default: not_implemented(); + } + break; + } default: { + APPEND_C(*f); + f++; + break; + }} + } + label_out: + if (i < size) + str[i] = '\0'; + else + str[size - 1] = '\0'; + ret = i; + +#undef APPEND_C +#undef APPEND_S +#undef APPEND_PADDED_S +#undef GET_ARG_NUMERIC + return (ret); +} + +JEMALLOC_ATTR(format(printf, 3, 4)) +int +malloc_snprintf(char *str, size_t size, const char *format, ...) +{ + int ret; + va_list ap; + + va_start(ap, format); + ret = malloc_vsnprintf(str, size, format, ap); + va_end(ap); + + return (ret); +} + +void +malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, va_list ap) +{ + char buf[MALLOC_PRINTF_BUFSIZE]; + + if (write_cb == NULL) { + /* + * The caller did not provide an alternate write_cb callback + * function, so use the default one. malloc_write() is an + * inline function, so use malloc_message() directly here. + */ + write_cb = (je_malloc_message != NULL) ? je_malloc_message : + wrtmessage; + cbopaque = NULL; + } + + malloc_vsnprintf(buf, sizeof(buf), format, ap); + write_cb(cbopaque, buf); +} + +/* + * Print to a callback function in such a way as to (hopefully) avoid memory + * allocation. + */ +JEMALLOC_ATTR(format(printf, 3, 4)) +void +malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, + const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(write_cb, cbopaque, format, ap); + va_end(ap); +} + +/* Print to stderr in such a way as to avoid memory allocation. */ +JEMALLOC_ATTR(format(printf, 1, 2)) +void +malloc_printf(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); +} diff --git a/deps/jemalloc/src/zone.c b/deps/jemalloc/src/zone.c index 2c1b2318ef4..cde5d49a460 100644 --- a/deps/jemalloc/src/zone.c +++ b/deps/jemalloc/src/zone.c @@ -3,11 +3,18 @@ # error "This source file is for zones on Darwin (OS X)." #endif +/* + * The malloc_default_purgeable_zone function is only available on >= 10.6. + * We need to check whether it is present at runtime, thus the weak_import. + */ +extern malloc_zone_t *malloc_default_purgeable_zone(void) +JEMALLOC_ATTR(weak_import); + /******************************************************************************/ /* Data. */ -static malloc_zone_t zone, szone; -static struct malloc_introspection_t zone_introspect, ozone_introspect; +static malloc_zone_t zone; +static struct malloc_introspection_t zone_introspect; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -18,8 +25,10 @@ static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); static void *zone_valloc(malloc_zone_t *zone, size_t size); static void zone_free(malloc_zone_t *zone, void *ptr); static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -#if (JEMALLOC_ZONE_VERSION >= 6) +#if (JEMALLOC_ZONE_VERSION >= 5) static void *zone_memalign(malloc_zone_t *zone, size_t alignment, +#endif +#if (JEMALLOC_ZONE_VERSION >= 6) size_t size); static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size); @@ -28,19 +37,6 @@ static void *zone_destroy(malloc_zone_t *zone); static size_t zone_good_size(malloc_zone_t *zone, size_t size); static void zone_force_lock(malloc_zone_t *zone); static void zone_force_unlock(malloc_zone_t *zone); -static size_t ozone_size(malloc_zone_t *zone, void *ptr); -static void ozone_free(malloc_zone_t *zone, void *ptr); -static void *ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -static unsigned ozone_batch_malloc(malloc_zone_t *zone, size_t size, - void **results, unsigned num_requested); -static void ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, - unsigned num); -#if (JEMALLOC_ZONE_VERSION >= 6) -static void ozone_free_definite_size(malloc_zone_t *zone, void *ptr, - size_t size); -#endif -static void ozone_force_lock(malloc_zone_t *zone); -static void ozone_force_unlock(malloc_zone_t *zone); /******************************************************************************/ /* @@ -60,21 +56,21 @@ zone_size(malloc_zone_t *zone, void *ptr) * not work in practice, we must check all pointers to assure that they * reside within a mapped chunk before determining size. */ - return (ivsalloc(ptr)); + return (ivsalloc(ptr, config_prof)); } static void * zone_malloc(malloc_zone_t *zone, size_t size) { - return (JEMALLOC_P(malloc)(size)); + return (je_malloc(size)); } static void * zone_calloc(malloc_zone_t *zone, size_t num, size_t size) { - return (JEMALLOC_P(calloc)(num, size)); + return (je_calloc(num, size)); } static void * @@ -82,7 +78,7 @@ zone_valloc(malloc_zone_t *zone, size_t size) { void *ret = NULL; /* Assignment avoids useless compiler warning. */ - JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size); + je_posix_memalign(&ret, PAGE, size); return (ret); } @@ -91,33 +87,48 @@ static void zone_free(malloc_zone_t *zone, void *ptr) { - JEMALLOC_P(free)(ptr); + if (ivsalloc(ptr, config_prof) != 0) { + je_free(ptr); + return; + } + + free(ptr); } static void * zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) { - return (JEMALLOC_P(realloc)(ptr, size)); + if (ivsalloc(ptr, config_prof) != 0) + return (je_realloc(ptr, size)); + + return (realloc(ptr, size)); } -#if (JEMALLOC_ZONE_VERSION >= 6) +#if (JEMALLOC_ZONE_VERSION >= 5) static void * zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) { void *ret = NULL; /* Assignment avoids useless compiler warning. */ - JEMALLOC_P(posix_memalign)(&ret, alignment, size); + je_posix_memalign(&ret, alignment, size); return (ret); } +#endif +#if (JEMALLOC_ZONE_VERSION >= 6) static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) { - assert(ivsalloc(ptr) == size); - JEMALLOC_P(free)(ptr); + if (ivsalloc(ptr, config_prof) != 0) { + assert(ivsalloc(ptr, config_prof) == size); + je_free(ptr); + return; + } + + free(ptr); } #endif @@ -133,22 +144,10 @@ zone_destroy(malloc_zone_t *zone) static size_t zone_good_size(malloc_zone_t *zone, size_t size) { - size_t ret; - void *p; - /* - * Actually create an object of the appropriate size, then find out - * how large it could have been without moving up to the next size - * class. - */ - p = JEMALLOC_P(malloc)(size); - if (p != NULL) { - ret = isalloc(p); - JEMALLOC_P(free)(p); - } else - ret = size; - - return (ret); + if (size == 0) + size = 1; + return (s2u(size)); } static void @@ -164,11 +163,12 @@ zone_force_unlock(malloc_zone_t *zone) { if (isthreaded) - jemalloc_postfork(); + jemalloc_postfork_parent(); } -malloc_zone_t * -create_zone(void) +JEMALLOC_ATTR(constructor) +void +register_zone(void) { zone.size = (void *)zone_size; @@ -183,10 +183,15 @@ create_zone(void) zone.batch_free = NULL; zone.introspect = &zone_introspect; zone.version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 6) +#if (JEMALLOC_ZONE_VERSION >= 5) zone.memalign = zone_memalign; +#endif +#if (JEMALLOC_ZONE_VERSION >= 6) zone.free_definite_size = zone_free_definite_size; #endif +#if (JEMALLOC_ZONE_VERSION >= 8) + zone.pressure_relief = NULL; +#endif zone_introspect.enumerator = NULL; zone_introspect.good_size = (void *)zone_good_size; @@ -199,156 +204,45 @@ create_zone(void) #if (JEMALLOC_ZONE_VERSION >= 6) zone_introspect.zone_locked = NULL; #endif - - return (&zone); -} - -static size_t -ozone_size(malloc_zone_t *zone, void *ptr) -{ - size_t ret; - - ret = ivsalloc(ptr); - if (ret == 0) - ret = szone.size(zone, ptr); - - return (ret); -} - -static void -ozone_free(malloc_zone_t *zone, void *ptr) -{ - - if (ivsalloc(ptr) != 0) - JEMALLOC_P(free)(ptr); - else { - size_t size = szone.size(zone, ptr); - if (size != 0) - (szone.free)(zone, ptr); - } -} - -static void * -ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size) -{ - size_t oldsize; - - if (ptr == NULL) - return (JEMALLOC_P(malloc)(size)); - - oldsize = ivsalloc(ptr); - if (oldsize != 0) - return (JEMALLOC_P(realloc)(ptr, size)); - else { - oldsize = szone.size(zone, ptr); - if (oldsize == 0) - return (JEMALLOC_P(malloc)(size)); - else { - void *ret = JEMALLOC_P(malloc)(size); - if (ret != NULL) { - memcpy(ret, ptr, (oldsize < size) ? oldsize : - size); - (szone.free)(zone, ptr); - } - return (ret); - } - } -} - -static unsigned -ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, - unsigned num_requested) -{ - - /* Don't bother implementing this interface, since it isn't required. */ - return (0); -} - -static void -ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) -{ - unsigned i; - - for (i = 0; i < num; i++) - ozone_free(zone, to_be_freed[i]); -} - -#if (JEMALLOC_ZONE_VERSION >= 6) -static void -ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) -{ - - if (ivsalloc(ptr) != 0) { - assert(ivsalloc(ptr) == size); - JEMALLOC_P(free)(ptr); - } else { - assert(size == szone.size(zone, ptr)); - szone.free_definite_size(zone, ptr, size); - } -} +#if (JEMALLOC_ZONE_VERSION >= 7) + zone_introspect.enable_discharge_checking = NULL; + zone_introspect.disable_discharge_checking = NULL; + zone_introspect.discharge = NULL; +#ifdef __BLOCKS__ + zone_introspect.enumerate_discharged_pointers = NULL; +#else + zone_introspect.enumerate_unavailable_without_blocks = NULL; +#endif #endif -static void -ozone_force_lock(malloc_zone_t *zone) -{ - - /* jemalloc locking is taken care of by the normal jemalloc zone. */ - szone.introspect->force_lock(zone); -} - -static void -ozone_force_unlock(malloc_zone_t *zone) -{ - - /* jemalloc locking is taken care of by the normal jemalloc zone. */ - szone.introspect->force_unlock(zone); -} + /* + * The default purgeable zone is created lazily by OSX's libc. It uses + * the default zone when it is created for "small" allocations + * (< 15 KiB), but assumes the default zone is a scalable_zone. This + * obviously fails when the default zone is the jemalloc zone, so + * malloc_default_purgeable_zone is called beforehand so that the + * default purgeable zone is created when the default zone is still + * a scalable_zone. As purgeable zones only exist on >= 10.6, we need + * to check for the existence of malloc_default_purgeable_zone() at + * run time. + */ + if (malloc_default_purgeable_zone != NULL) + malloc_default_purgeable_zone(); -/* - * Overlay the default scalable zone (szone) such that existing allocations are - * drained, and further allocations come from jemalloc. This is necessary - * because Core Foundation directly accesses and uses the szone before the - * jemalloc library is even loaded. - */ -void -szone2ozone(malloc_zone_t *zone) -{ + /* Register the custom zone. At this point it won't be the default. */ + malloc_zone_register(&zone); /* - * Stash a copy of the original szone so that we can call its - * functions as needed. Note that the internally, the szone stores its - * bookkeeping data structures immediately following the malloc_zone_t - * header, so when calling szone functions, we need to pass a pointer - * to the original zone structure. + * Unregister and reregister the default zone. On OSX >= 10.6, + * unregistering takes the last registered zone and places it at the + * location of the specified zone. Unregistering the default zone thus + * makes the last registered one the default. On OSX < 10.6, + * unregistering shifts all registered zones. The first registered zone + * then becomes the default. */ - memcpy(&szone, zone, sizeof(malloc_zone_t)); - - zone->size = (void *)ozone_size; - zone->malloc = (void *)zone_malloc; - zone->calloc = (void *)zone_calloc; - zone->valloc = (void *)zone_valloc; - zone->free = (void *)ozone_free; - zone->realloc = (void *)ozone_realloc; - zone->destroy = (void *)zone_destroy; - zone->zone_name = "jemalloc_ozone"; - zone->batch_malloc = ozone_batch_malloc; - zone->batch_free = ozone_batch_free; - zone->introspect = &ozone_introspect; - zone->version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 6) - zone->memalign = zone_memalign; - zone->free_definite_size = ozone_free_definite_size; -#endif - - ozone_introspect.enumerator = NULL; - ozone_introspect.good_size = (void *)zone_good_size; - ozone_introspect.check = NULL; - ozone_introspect.print = NULL; - ozone_introspect.log = NULL; - ozone_introspect.force_lock = (void *)ozone_force_lock; - ozone_introspect.force_unlock = (void *)ozone_force_unlock; - ozone_introspect.statistics = NULL; -#if (JEMALLOC_ZONE_VERSION >= 6) - ozone_introspect.zone_locked = NULL; -#endif + do { + malloc_zone_t *default_zone = malloc_default_zone(); + malloc_zone_unregister(default_zone); + malloc_zone_register(default_zone); + } while (malloc_default_zone() != &zone); } diff --git a/deps/jemalloc/test/aligned_alloc.c b/deps/jemalloc/test/aligned_alloc.c new file mode 100644 index 00000000000..5a9b0caea78 --- /dev/null +++ b/deps/jemalloc/test/aligned_alloc.c @@ -0,0 +1,119 @@ +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +#define CHUNK 0x400000 +/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ +#define MAXALIGN ((size_t)0x2000000LU) +#define NITER 4 + +int +main(void) +{ + size_t alignment, size, total; + unsigned i; + void *p, *ps[NITER]; + + malloc_printf("Test begin\n"); + + /* Test error conditions. */ + alignment = 0; + set_errno(0); + p = aligned_alloc(alignment, 1); + if (p != NULL || get_errno() != EINVAL) { + malloc_printf( + "Expected error for invalid alignment %zu\n", alignment); + } + + for (alignment = sizeof(size_t); alignment < MAXALIGN; + alignment <<= 1) { + set_errno(0); + p = aligned_alloc(alignment + 1, 1); + if (p != NULL || get_errno() != EINVAL) { + malloc_printf( + "Expected error for invalid alignment %zu\n", + alignment + 1); + } + } + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x8000000000000000); + size = UINT64_C(0x8000000000000000); +#else + alignment = 0x80000000LU; + size = 0x80000000LU; +#endif + set_errno(0); + p = aligned_alloc(alignment, size); + if (p != NULL || get_errno() != ENOMEM) { + malloc_printf( + "Expected error for aligned_alloc(%zu, %zu)\n", + alignment, size); + } + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x4000000000000000); + size = UINT64_C(0x8400000000000001); +#else + alignment = 0x40000000LU; + size = 0x84000001LU; +#endif + set_errno(0); + p = aligned_alloc(alignment, size); + if (p != NULL || get_errno() != ENOMEM) { + malloc_printf( + "Expected error for aligned_alloc(%zu, %zu)\n", + alignment, size); + } + + alignment = 0x10LU; +#if LG_SIZEOF_PTR == 3 + size = UINT64_C(0xfffffffffffffff0); +#else + size = 0xfffffff0LU; +#endif + set_errno(0); + p = aligned_alloc(alignment, size); + if (p != NULL || get_errno() != ENOMEM) { + malloc_printf( + "Expected error for aligned_alloc(&p, %zu, %zu)\n", + alignment, size); + } + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + malloc_printf("Alignment: %zu\n", alignment); + for (size = 1; + size < 3 * alignment && size < (1U << 31); + size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + ps[i] = aligned_alloc(alignment, size); + if (ps[i] == NULL) { + char buf[BUFERROR_BUF]; + + buferror(buf, sizeof(buf)); + malloc_printf( + "Error for size %zu (%#zx): %s\n", + size, size, buf); + exit(1); + } + total += malloc_usable_size(ps[i]); + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + free(ps[i]); + ps[i] = NULL; + } + } + } + } + + malloc_printf("Test end\n"); + return (0); +} diff --git a/deps/jemalloc/test/aligned_alloc.exp b/deps/jemalloc/test/aligned_alloc.exp new file mode 100644 index 00000000000..b5061c7277e --- /dev/null +++ b/deps/jemalloc/test/aligned_alloc.exp @@ -0,0 +1,25 @@ +Test begin +Alignment: 8 +Alignment: 16 +Alignment: 32 +Alignment: 64 +Alignment: 128 +Alignment: 256 +Alignment: 512 +Alignment: 1024 +Alignment: 2048 +Alignment: 4096 +Alignment: 8192 +Alignment: 16384 +Alignment: 32768 +Alignment: 65536 +Alignment: 131072 +Alignment: 262144 +Alignment: 524288 +Alignment: 1048576 +Alignment: 2097152 +Alignment: 4194304 +Alignment: 8388608 +Alignment: 16777216 +Alignment: 33554432 +Test end diff --git a/deps/jemalloc/test/allocated.c b/deps/jemalloc/test/allocated.c index b1e40e4720e..9884905d810 100644 --- a/deps/jemalloc/test/allocated.c +++ b/deps/jemalloc/test/allocated.c @@ -1,17 +1,8 @@ -#include -#include -#include -#include -#include -#include -#include -#include - #define JEMALLOC_MANGLE #include "jemalloc_test.h" void * -thread_start(void *arg) +je_thread_start(void *arg) { int err; void *p; @@ -20,89 +11,85 @@ thread_start(void *arg) size_t sz, usize; sz = sizeof(a0); - if ((err = JEMALLOC_P(mallctl)("thread.allocated", &a0, &sz, NULL, - 0))) { + if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) { if (err == ENOENT) { #ifdef JEMALLOC_STATS assert(false); #endif - goto RETURN; + goto label_return; } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); exit(1); } sz = sizeof(ap0); - if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL, - 0))) { + if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) { if (err == ENOENT) { #ifdef JEMALLOC_STATS assert(false); #endif - goto RETURN; + goto label_return; } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); exit(1); } assert(*ap0 == a0); sz = sizeof(d0); - if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL, - 0))) { + if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) { if (err == ENOENT) { #ifdef JEMALLOC_STATS assert(false); #endif - goto RETURN; + goto label_return; } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); exit(1); } sz = sizeof(dp0); - if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL, - 0))) { + if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) { if (err == ENOENT) { #ifdef JEMALLOC_STATS assert(false); #endif - goto RETURN; + goto label_return; } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); exit(1); } assert(*dp0 == d0); - p = JEMALLOC_P(malloc)(1); + p = malloc(1); if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); + malloc_printf("%s(): Error in malloc()\n", __func__); exit(1); } sz = sizeof(a1); - JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0); + mallctl("thread.allocated", &a1, &sz, NULL, 0); sz = sizeof(ap1); - JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0); + mallctl("thread.allocatedp", &ap1, &sz, NULL, 0); assert(*ap1 == a1); assert(ap0 == ap1); - usize = JEMALLOC_P(malloc_usable_size)(p); + usize = malloc_usable_size(p); assert(a0 + usize <= a1); - JEMALLOC_P(free)(p); + free(p); sz = sizeof(d1); - JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0); + mallctl("thread.deallocated", &d1, &sz, NULL, 0); sz = sizeof(dp1); - JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0); + mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0); assert(*dp1 == d1); assert(dp0 == dp1); assert(d0 + usize <= d1); -RETURN: +label_return: return (NULL); } @@ -110,33 +97,22 @@ int main(void) { int ret = 0; - pthread_t thread; + je_thread_t thread; - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); - thread_start(NULL); + je_thread_start(NULL); - if (pthread_create(&thread, NULL, thread_start, NULL) - != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); - ret = 1; - goto RETURN; - } - pthread_join(thread, (void *)&ret); + je_thread_create(&thread, je_thread_start, NULL); + je_thread_join(thread, (void *)&ret); - thread_start(NULL); + je_thread_start(NULL); - if (pthread_create(&thread, NULL, thread_start, NULL) - != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); - ret = 1; - goto RETURN; - } - pthread_join(thread, (void *)&ret); + je_thread_create(&thread, je_thread_start, NULL); + je_thread_join(thread, (void *)&ret); - thread_start(NULL); + je_thread_start(NULL); -RETURN: - fprintf(stderr, "Test end\n"); + malloc_printf("Test end\n"); return (ret); } diff --git a/deps/jemalloc/test/allocm.c b/deps/jemalloc/test/allocm.c index 59d0002e7be..80be673b8fd 100644 --- a/deps/jemalloc/test/allocm.c +++ b/deps/jemalloc/test/allocm.c @@ -1,13 +1,9 @@ -#include -#include -#include - #define JEMALLOC_MANGLE #include "jemalloc_test.h" #define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)0x80000000000LLU) */ -#define MAXALIGN ((size_t)0x2000000LLU) +/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ +#define MAXALIGN ((size_t)0x2000000LU) #define NITER 4 int @@ -15,79 +11,122 @@ main(void) { int r; void *p; - size_t sz, alignment, total, tsz; + size_t nsz, rsz, sz, alignment, total; unsigned i; void *ps[NITER]; - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); - sz = 0; - r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); + sz = 42; + nsz = 0; + r = nallocm(&nsz, sz, 0); if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); + malloc_printf("Unexpected nallocm() error\n"); abort(); } - if (sz < 42) - fprintf(stderr, "Real size smaller than expected\n"); - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); + rsz = 0; + r = allocm(&p, &rsz, sz, 0); + if (r != ALLOCM_SUCCESS) { + malloc_printf("Unexpected allocm() error\n"); + abort(); + } + if (rsz < sz) + malloc_printf("Real size smaller than expected\n"); + if (nsz != rsz) + malloc_printf("nallocm()/allocm() rsize mismatch\n"); + if (dallocm(p, 0) != ALLOCM_SUCCESS) + malloc_printf("Unexpected dallocm() error\n"); - r = JEMALLOC_P(allocm)(&p, NULL, 42, 0); + r = allocm(&p, NULL, sz, 0); if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); + malloc_printf("Unexpected allocm() error\n"); abort(); } - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); + if (dallocm(p, 0) != ALLOCM_SUCCESS) + malloc_printf("Unexpected dallocm() error\n"); - r = JEMALLOC_P(allocm)(&p, NULL, 42, ALLOCM_ZERO); + nsz = 0; + r = nallocm(&nsz, sz, ALLOCM_ZERO); + if (r != ALLOCM_SUCCESS) { + malloc_printf("Unexpected nallocm() error\n"); + abort(); + } + rsz = 0; + r = allocm(&p, &rsz, sz, ALLOCM_ZERO); if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); + malloc_printf("Unexpected allocm() error\n"); abort(); } - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); + if (nsz != rsz) + malloc_printf("nallocm()/allocm() rsize mismatch\n"); + if (dallocm(p, 0) != ALLOCM_SUCCESS) + malloc_printf("Unexpected dallocm() error\n"); #if LG_SIZEOF_PTR == 3 - alignment = 0x8000000000000000LLU; - sz = 0x8000000000000000LLU; + alignment = UINT64_C(0x8000000000000000); + sz = UINT64_C(0x8000000000000000); #else alignment = 0x80000000LU; sz = 0x80000000LU; #endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); + nsz = 0; + r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)); + if (r == ALLOCM_SUCCESS) { + malloc_printf( + "Expected error for nallocm(&nsz, %zu, %#x)\n", + sz, ALLOCM_ALIGN(alignment)); + } + rsz = 0; + r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)); if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", + malloc_printf( + "Expected error for allocm(&p, %zu, %#x)\n", sz, ALLOCM_ALIGN(alignment)); } + if (nsz != rsz) + malloc_printf("nallocm()/allocm() rsize mismatch\n"); #if LG_SIZEOF_PTR == 3 - alignment = 0x4000000000000000LLU; - sz = 0x8400000000000001LLU; + alignment = UINT64_C(0x4000000000000000); + sz = UINT64_C(0x8400000000000001); #else alignment = 0x40000000LU; sz = 0x84000001LU; #endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); + nsz = 0; + r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)); + if (r != ALLOCM_SUCCESS) + malloc_printf("Unexpected nallocm() error\n"); + rsz = 0; + r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)); if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", + malloc_printf( + "Expected error for allocm(&p, %zu, %#x)\n", sz, ALLOCM_ALIGN(alignment)); } - alignment = 0x10LLU; + alignment = 0x10LU; #if LG_SIZEOF_PTR == 3 - sz = 0xfffffffffffffff0LLU; + sz = UINT64_C(0xfffffffffffffff0); #else - sz = 0xfffffff0LU; + sz = 0xfffffff0LU; #endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); + nsz = 0; + r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)); if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", + malloc_printf( + "Expected error for nallocm(&nsz, %zu, %#x)\n", sz, ALLOCM_ALIGN(alignment)); } + rsz = 0; + r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)); + if (r == ALLOCM_SUCCESS) { + malloc_printf( + "Expected error for allocm(&p, %zu, %#x)\n", + sz, ALLOCM_ALIGN(alignment)); + } + if (nsz != rsz) + malloc_printf("nallocm()/allocm() rsize mismatch\n"); for (i = 0; i < NITER; i++) ps[i] = NULL; @@ -96,38 +135,60 @@ main(void) alignment <= MAXALIGN; alignment <<= 1) { total = 0; - fprintf(stderr, "Alignment: %zu\n", alignment); + malloc_printf("Alignment: %zu\n", alignment); for (sz = 1; sz < 3 * alignment && sz < (1U << 31); sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { for (i = 0; i < NITER; i++) { - r = JEMALLOC_P(allocm)(&ps[i], NULL, sz, + nsz = 0; + r = nallocm(&nsz, sz, + ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); + if (r != ALLOCM_SUCCESS) { + malloc_printf( + "nallocm() error for size %zu" + " (%#zx): %d\n", + sz, sz, r); + exit(1); + } + rsz = 0; + r = allocm(&ps[i], &rsz, sz, ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); if (r != ALLOCM_SUCCESS) { - fprintf(stderr, - "Error for size %zu (0x%zx): %d\n", + malloc_printf( + "allocm() error for size %zu" + " (%#zx): %d\n", sz, sz, r); exit(1); } + if (rsz < sz) { + malloc_printf( + "Real size smaller than" + " expected\n"); + } + if (nsz != rsz) { + malloc_printf( + "nallocm()/allocm() rsize" + " mismatch\n"); + } if ((uintptr_t)p & (alignment-1)) { - fprintf(stderr, + malloc_printf( "%p inadequately aligned for" " alignment: %zu\n", p, alignment); } - JEMALLOC_P(sallocm)(ps[i], &tsz, 0); - total += tsz; + sallocm(ps[i], &rsz, 0); + total += rsz; if (total >= (MAXALIGN << 1)) break; } for (i = 0; i < NITER; i++) { if (ps[i] != NULL) { - JEMALLOC_P(dallocm)(ps[i], 0); + dallocm(ps[i], 0); ps[i] = NULL; } } } } - fprintf(stderr, "Test end\n"); + malloc_printf("Test end\n"); return (0); } diff --git a/deps/jemalloc/test/bitmap.c b/deps/jemalloc/test/bitmap.c index adfaacfe52b..b2cb63004bc 100644 --- a/deps/jemalloc/test/bitmap.c +++ b/deps/jemalloc/test/bitmap.c @@ -1,18 +1,6 @@ #define JEMALLOC_MANGLE #include "jemalloc_test.h" -/* - * Avoid using the assert() from jemalloc_internal.h, since it requires - * internal libjemalloc functionality. - * */ -#include - -/* - * Directly include the bitmap code, since it isn't exposed outside - * libjemalloc. - */ -#include "../src/bitmap.c" - #if (LG_BITMAP_MAXBITS > 12) # define MAXBITS 4500 #else @@ -42,11 +30,13 @@ test_bitmap_init(void) bitmap_info_init(&binfo, i); { size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); bitmap_init(bitmap, &binfo); for (j = 0; j < i; j++) assert(bitmap_get(bitmap, &binfo, j) == false); + free(bitmap); } } @@ -62,12 +52,14 @@ test_bitmap_set(void) bitmap_info_init(&binfo, i); { size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); bitmap_init(bitmap, &binfo); for (j = 0; j < i; j++) bitmap_set(bitmap, &binfo, j); assert(bitmap_full(bitmap, &binfo)); + free(bitmap); } } } @@ -82,7 +74,8 @@ test_bitmap_unset(void) bitmap_info_init(&binfo, i); { size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); bitmap_init(bitmap, &binfo); for (j = 0; j < i; j++) @@ -93,6 +86,7 @@ test_bitmap_unset(void) for (j = 0; j < i; j++) bitmap_set(bitmap, &binfo, j); assert(bitmap_full(bitmap, &binfo)); + free(bitmap); } } } @@ -107,7 +101,8 @@ test_bitmap_sfu(void) bitmap_info_init(&binfo, i); { ssize_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; + bitmap_t *bitmap = malloc(sizeof(bitmap_t) * + bitmap_info_ngroups(&binfo)); bitmap_init(bitmap, &binfo); /* Iteratively set bits starting at the beginning. */ @@ -137,6 +132,7 @@ test_bitmap_sfu(void) } assert(bitmap_sfu(bitmap, &binfo) == i - 1); assert(bitmap_full(bitmap, &binfo)); + free(bitmap); } } } @@ -144,7 +140,7 @@ test_bitmap_sfu(void) int main(void) { - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); test_bitmap_size(); test_bitmap_init(); @@ -152,6 +148,6 @@ main(void) test_bitmap_unset(); test_bitmap_sfu(); - fprintf(stderr, "Test end\n"); + malloc_printf("Test end\n"); return (0); } diff --git a/deps/jemalloc/test/jemalloc_test.h.in b/deps/jemalloc/test/jemalloc_test.h.in index 0c48895e724..e38b48efa41 100644 --- a/deps/jemalloc/test/jemalloc_test.h.in +++ b/deps/jemalloc/test/jemalloc_test.h.in @@ -4,3 +4,50 @@ * have a different name. */ #include "jemalloc/jemalloc@install_suffix@.h" +#include "jemalloc/internal/jemalloc_internal.h" + +/* Abstraction layer for threading in tests */ +#ifdef _WIN32 +#include + +typedef HANDLE je_thread_t; + +void +je_thread_create(je_thread_t *thread, void *(*proc)(void *), void *arg) +{ + LPTHREAD_START_ROUTINE routine = (LPTHREAD_START_ROUTINE)proc; + *thread = CreateThread(NULL, 0, routine, arg, 0, NULL); + if (*thread == NULL) { + malloc_printf("Error in CreateThread()\n"); + exit(1); + } +} + +void +je_thread_join(je_thread_t thread, void **ret) +{ + WaitForSingleObject(thread, INFINITE); +} + +#else +#include + +typedef pthread_t je_thread_t; + +void +je_thread_create(je_thread_t *thread, void *(*proc)(void *), void *arg) +{ + + if (pthread_create(thread, NULL, proc, arg) != 0) { + malloc_printf("Error in pthread_create()\n"); + exit(1); + } +} + +void +je_thread_join(je_thread_t thread, void **ret) +{ + + pthread_join(thread, ret); +} +#endif diff --git a/deps/jemalloc/test/mremap.c b/deps/jemalloc/test/mremap.c index 146c66f4212..47efa7c415b 100644 --- a/deps/jemalloc/test/mremap.c +++ b/deps/jemalloc/test/mremap.c @@ -1,9 +1,3 @@ -#include -#include -#include -#include -#include - #define JEMALLOC_MANGLE #include "jemalloc_test.h" @@ -14,33 +8,32 @@ main(void) size_t sz, lg_chunk, chunksize, i; char *p, *q; - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); sz = sizeof(lg_chunk); - if ((err = JEMALLOC_P(mallctl)("opt.lg_chunk", &lg_chunk, &sz, NULL, - 0))) { + if ((err = mallctl("opt.lg_chunk", &lg_chunk, &sz, NULL, 0))) { assert(err != ENOENT); - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); ret = 1; - goto RETURN; + goto label_return; } chunksize = ((size_t)1U) << lg_chunk; p = (char *)malloc(chunksize); if (p == NULL) { - fprintf(stderr, "malloc(%zu) --> %p\n", chunksize, p); + malloc_printf("malloc(%zu) --> %p\n", chunksize, p); ret = 1; - goto RETURN; + goto label_return; } memset(p, 'a', chunksize); q = (char *)realloc(p, chunksize * 2); if (q == NULL) { - fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize * 2, + malloc_printf("realloc(%p, %zu) --> %p\n", p, chunksize * 2, q); ret = 1; - goto RETURN; + goto label_return; } for (i = 0; i < chunksize; i++) { assert(q[i] == 'a'); @@ -50,9 +43,9 @@ main(void) q = (char *)realloc(p, chunksize); if (q == NULL) { - fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize, q); + malloc_printf("realloc(%p, %zu) --> %p\n", p, chunksize, q); ret = 1; - goto RETURN; + goto label_return; } for (i = 0; i < chunksize; i++) { assert(q[i] == 'a'); @@ -61,7 +54,7 @@ main(void) free(q); ret = 0; -RETURN: - fprintf(stderr, "Test end\n"); +label_return: + malloc_printf("Test end\n"); return (ret); } diff --git a/deps/jemalloc/test/posix_memalign.c b/deps/jemalloc/test/posix_memalign.c index 3e306c01dd2..2185bcf762a 100644 --- a/deps/jemalloc/test/posix_memalign.c +++ b/deps/jemalloc/test/posix_memalign.c @@ -1,15 +1,9 @@ -#include -#include -#include -#include -#include - #define JEMALLOC_MANGLE #include "jemalloc_test.h" #define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)0x80000000000LLU) */ -#define MAXALIGN ((size_t)0x2000000LLU) +/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ +#define MAXALIGN ((size_t)0x2000000LU) #define NITER 4 int @@ -20,13 +14,13 @@ main(void) int err; void *p, *ps[NITER]; - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); /* Test error conditions. */ for (alignment = 0; alignment < sizeof(void *); alignment++) { - err = JEMALLOC_P(posix_memalign)(&p, alignment, 1); + err = posix_memalign(&p, alignment, 1); if (err != EINVAL) { - fprintf(stderr, + malloc_printf( "Expected error for invalid alignment %zu\n", alignment); } @@ -34,51 +28,51 @@ main(void) for (alignment = sizeof(size_t); alignment < MAXALIGN; alignment <<= 1) { - err = JEMALLOC_P(posix_memalign)(&p, alignment + 1, 1); + err = posix_memalign(&p, alignment + 1, 1); if (err == 0) { - fprintf(stderr, + malloc_printf( "Expected error for invalid alignment %zu\n", alignment + 1); } } #if LG_SIZEOF_PTR == 3 - alignment = 0x8000000000000000LLU; - size = 0x8000000000000000LLU; + alignment = UINT64_C(0x8000000000000000); + size = UINT64_C(0x8000000000000000); #else alignment = 0x80000000LU; size = 0x80000000LU; #endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); + err = posix_memalign(&p, alignment, size); if (err == 0) { - fprintf(stderr, + malloc_printf( "Expected error for posix_memalign(&p, %zu, %zu)\n", alignment, size); } #if LG_SIZEOF_PTR == 3 - alignment = 0x4000000000000000LLU; - size = 0x8400000000000001LLU; + alignment = UINT64_C(0x4000000000000000); + size = UINT64_C(0x8400000000000001); #else alignment = 0x40000000LU; size = 0x84000001LU; #endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); + err = posix_memalign(&p, alignment, size); if (err == 0) { - fprintf(stderr, + malloc_printf( "Expected error for posix_memalign(&p, %zu, %zu)\n", alignment, size); } - alignment = 0x10LLU; + alignment = 0x10LU; #if LG_SIZEOF_PTR == 3 - size = 0xfffffffffffffff0LLU; + size = UINT64_C(0xfffffffffffffff0); #else size = 0xfffffff0LU; #endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); + err = posix_memalign(&p, alignment, size); if (err == 0) { - fprintf(stderr, + malloc_printf( "Expected error for posix_memalign(&p, %zu, %zu)\n", alignment, size); } @@ -90,32 +84,32 @@ main(void) alignment <= MAXALIGN; alignment <<= 1) { total = 0; - fprintf(stderr, "Alignment: %zu\n", alignment); + malloc_printf("Alignment: %zu\n", alignment); for (size = 1; size < 3 * alignment && size < (1U << 31); size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { for (i = 0; i < NITER; i++) { - err = JEMALLOC_P(posix_memalign)(&ps[i], + err = posix_memalign(&ps[i], alignment, size); if (err) { - fprintf(stderr, - "Error for size %zu (0x%zx): %s\n", + malloc_printf( + "Error for size %zu (%#zx): %s\n", size, size, strerror(err)); exit(1); } - total += JEMALLOC_P(malloc_usable_size)(ps[i]); + total += malloc_usable_size(ps[i]); if (total >= (MAXALIGN << 1)) break; } for (i = 0; i < NITER; i++) { if (ps[i] != NULL) { - JEMALLOC_P(free)(ps[i]); + free(ps[i]); ps[i] = NULL; } } } } - fprintf(stderr, "Test end\n"); + malloc_printf("Test end\n"); return (0); } diff --git a/deps/jemalloc/test/rallocm.c b/deps/jemalloc/test/rallocm.c index ccf326bb523..c5dedf48d7b 100644 --- a/deps/jemalloc/test/rallocm.c +++ b/deps/jemalloc/test/rallocm.c @@ -1,9 +1,3 @@ -#include -#include -#include -#include -#include - #define JEMALLOC_MANGLE #include "jemalloc_test.h" @@ -15,113 +9,119 @@ main(void) size_t sz, tsz; int r; - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); /* Get page size. */ { +#ifdef _WIN32 + SYSTEM_INFO si; + GetSystemInfo(&si); + pagesize = (size_t)si.dwPageSize; +#else long result = sysconf(_SC_PAGESIZE); assert(result != -1); pagesize = (size_t)result; +#endif } - r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); + r = allocm(&p, &sz, 42, 0); if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); + malloc_printf("Unexpected allocm() error\n"); abort(); } q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 0, ALLOCM_NO_MOVE); + r = rallocm(&q, &tsz, sz, 0, ALLOCM_NO_MOVE); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (q != p) - fprintf(stderr, "Unexpected object move\n"); + malloc_printf("Unexpected object move\n"); if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", + malloc_printf("Unexpected size change: %zu --> %zu\n", sz, tsz); } q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 5, ALLOCM_NO_MOVE); + r = rallocm(&q, &tsz, sz, 5, ALLOCM_NO_MOVE); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (q != p) - fprintf(stderr, "Unexpected object move\n"); + malloc_printf("Unexpected object move\n"); if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", + malloc_printf("Unexpected size change: %zu --> %zu\n", sz, tsz); } q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE); + r = rallocm(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE); if (r != ALLOCM_ERR_NOT_MOVED) - fprintf(stderr, "Unexpected rallocm() result\n"); + malloc_printf("Unexpected rallocm() result\n"); if (q != p) - fprintf(stderr, "Unexpected object move\n"); + malloc_printf("Unexpected object move\n"); if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", + malloc_printf("Unexpected size change: %zu --> %zu\n", sz, tsz); } q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, 0); + r = rallocm(&q, &tsz, sz + 5, 0, 0); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (q == p) - fprintf(stderr, "Expected object move\n"); + malloc_printf("Expected object move\n"); if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", + malloc_printf("Expected size change: %zu --> %zu\n", sz, tsz); } p = q; sz = tsz; - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, 0); + r = rallocm(&q, &tsz, pagesize*2, 0, 0); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (q == p) - fprintf(stderr, "Expected object move\n"); + malloc_printf("Expected object move\n"); if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", + malloc_printf("Expected size change: %zu --> %zu\n", sz, tsz); } p = q; sz = tsz; - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, 0); + r = rallocm(&q, &tsz, pagesize*4, 0, 0); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", + malloc_printf("Expected size change: %zu --> %zu\n", sz, tsz); } p = q; sz = tsz; - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, ALLOCM_NO_MOVE); + r = rallocm(&q, &tsz, pagesize*2, 0, ALLOCM_NO_MOVE); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (q != p) - fprintf(stderr, "Unexpected object move\n"); + malloc_printf("Unexpected object move\n"); if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", + malloc_printf("Expected size change: %zu --> %zu\n", sz, tsz); } sz = tsz; - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, ALLOCM_NO_MOVE); + r = rallocm(&q, &tsz, pagesize*4, 0, ALLOCM_NO_MOVE); if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); + malloc_printf("Unexpected rallocm() error\n"); if (q != p) - fprintf(stderr, "Unexpected object move\n"); + malloc_printf("Unexpected object move\n"); if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", + malloc_printf("Expected size change: %zu --> %zu\n", sz, tsz); } sz = tsz; - JEMALLOC_P(dallocm)(p, 0); + dallocm(p, 0); - fprintf(stderr, "Test end\n"); + malloc_printf("Test end\n"); return (0); } diff --git a/deps/jemalloc/test/thread_arena.c b/deps/jemalloc/test/thread_arena.c index ef8d6817500..2020d99480a 100644 --- a/deps/jemalloc/test/thread_arena.c +++ b/deps/jemalloc/test/thread_arena.c @@ -1,16 +1,10 @@ -#include -#include -#include -#include -#include - #define JEMALLOC_MANGLE #include "jemalloc_test.h" #define NTHREADS 10 void * -thread_start(void *arg) +je_thread_start(void *arg) { unsigned main_arena_ind = *(unsigned *)arg; void *p; @@ -18,24 +12,24 @@ thread_start(void *arg) size_t size; int err; - p = JEMALLOC_P(malloc)(1); + p = malloc(1); if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); + malloc_printf("%s(): Error in malloc()\n", __func__); return (void *)1; } size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, - &main_arena_ind, sizeof(main_arena_ind)))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind, + sizeof(main_arena_ind)))) { + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); return (void *)1; } size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, + if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); return (void *)1; } @@ -52,41 +46,33 @@ main(void) unsigned arena_ind; size_t size; int err; - pthread_t threads[NTHREADS]; + je_thread_t threads[NTHREADS]; unsigned i; - fprintf(stderr, "Test begin\n"); + malloc_printf("Test begin\n"); - p = JEMALLOC_P(malloc)(1); + p = malloc(1); if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); + malloc_printf("%s(): Error in malloc()\n", __func__); ret = 1; - goto RETURN; + goto label_return; } size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, - 0))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, + if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { + malloc_printf("%s(): Error in mallctl(): %s\n", __func__, strerror(err)); ret = 1; - goto RETURN; + goto label_return; } - for (i = 0; i < NTHREADS; i++) { - if (pthread_create(&threads[i], NULL, thread_start, - (void *)&arena_ind) != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", - __func__); - ret = 1; - goto RETURN; - } - } + for (i = 0; i < NTHREADS; i++) + je_thread_create(&threads[i], je_thread_start, (void *)&arena_ind); for (i = 0; i < NTHREADS; i++) - pthread_join(threads[i], (void *)&ret); + je_thread_join(threads[i], (void *)&ret); -RETURN: - fprintf(stderr, "Test end\n"); +label_return: + malloc_printf("Test end\n"); return (ret); } diff --git a/deps/jemalloc/test/thread_tcache_enabled.c b/deps/jemalloc/test/thread_tcache_enabled.c new file mode 100644 index 00000000000..2061b7bbaff --- /dev/null +++ b/deps/jemalloc/test/thread_tcache_enabled.c @@ -0,0 +1,91 @@ +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +void * +je_thread_start(void *arg) +{ + int err; + size_t sz; + bool e0, e1; + + sz = sizeof(bool); + if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) { + if (err == ENOENT) { +#ifdef JEMALLOC_TCACHE + assert(false); +#endif + } + goto label_return; + } + + if (e0) { + e1 = false; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) + == 0); + assert(e0); + } + + e1 = true; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0 == false); + + e1 = true; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0); + + e1 = false; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0); + + e1 = false; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0 == false); + + free(malloc(1)); + e1 = true; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0 == false); + + free(malloc(1)); + e1 = true; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0); + + free(malloc(1)); + e1 = false; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0); + + free(malloc(1)); + e1 = false; + assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); + assert(e0 == false); + + free(malloc(1)); +label_return: + return (NULL); +} + +int +main(void) +{ + int ret = 0; + je_thread_t thread; + + malloc_printf("Test begin\n"); + + je_thread_start(NULL); + + je_thread_create(&thread, je_thread_start, NULL); + je_thread_join(thread, (void *)&ret); + + je_thread_start(NULL); + + je_thread_create(&thread, je_thread_start, NULL); + je_thread_join(thread, (void *)&ret); + + je_thread_start(NULL); + + malloc_printf("Test end\n"); + return (ret); +} diff --git a/deps/jemalloc/test/thread_tcache_enabled.exp b/deps/jemalloc/test/thread_tcache_enabled.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc/test/thread_tcache_enabled.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/src/zmalloc.h b/src/zmalloc.h index ff555619e5f..89f5b6ee046 100644 --- a/src/zmalloc.h +++ b/src/zmalloc.h @@ -47,11 +47,10 @@ #elif defined(USE_JEMALLOC) #define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX)) -#define JEMALLOC_MANGLE #include #if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2) #define HAVE_MALLOC_SIZE 1 -#define zmalloc_size(p) JEMALLOC_P(malloc_usable_size)(p) +#define zmalloc_size(p) je_malloc_usable_size(p) #else #error "Newer version of jemalloc required" #endif From afc6bd1b48aede03d88dfc24cf1d832bc25f9a43 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 16 May 2012 12:22:29 +0200 Subject: [PATCH 0241/1752] Deleted jemalloc.orig from /deps. In the commit upgrading jemalloc to version 3.0.0 I added the old version of Jemalloc in the 'jemalloc.orig' directory for an error. This commit removes the not useful version of jemalloc. --- deps/jemalloc.orig/.gitignore | 23 - deps/jemalloc.orig/COPYING | 51 - deps/jemalloc.orig/ChangeLog | 250 - deps/jemalloc.orig/INSTALL | 257 - deps/jemalloc.orig/Makefile.in | 259 - deps/jemalloc.orig/README | 16 - deps/jemalloc.orig/autogen.sh | 17 - deps/jemalloc.orig/bin/pprof | 4893 ----------------- deps/jemalloc.orig/config.guess | 1456 ----- deps/jemalloc.orig/config.stamp.in | 0 deps/jemalloc.orig/config.sub | 1549 ------ deps/jemalloc.orig/configure.ac | 938 ---- deps/jemalloc.orig/doc/html.xsl.in | 4 - deps/jemalloc.orig/doc/jemalloc.xml.in | 2280 -------- deps/jemalloc.orig/doc/manpages.xsl.in | 4 - deps/jemalloc.orig/doc/stylesheet.xsl | 7 - .../include/jemalloc/internal/arena.h | 743 --- .../include/jemalloc/internal/atomic.h | 169 - .../include/jemalloc/internal/base.h | 24 - .../include/jemalloc/internal/bitmap.h | 184 - .../include/jemalloc/internal/chunk.h | 65 - .../include/jemalloc/internal/chunk_dss.h | 30 - .../include/jemalloc/internal/chunk_mmap.h | 23 - .../include/jemalloc/internal/chunk_swap.h | 34 - .../include/jemalloc/internal/ckh.h | 95 - .../include/jemalloc/internal/ctl.h | 118 - .../include/jemalloc/internal/extent.h | 49 - .../include/jemalloc/internal/hash.h | 70 - .../include/jemalloc/internal/huge.h | 41 - .../jemalloc/internal/jemalloc_internal.h.in | 788 --- .../include/jemalloc/internal/mb.h | 108 - .../include/jemalloc/internal/mutex.h | 86 - .../jemalloc/internal/private_namespace.h | 195 - .../include/jemalloc/internal/prn.h | 60 - .../include/jemalloc/internal/prof.h | 547 -- .../include/jemalloc/internal/ql.h | 83 - .../include/jemalloc/internal/qr.h | 67 - .../include/jemalloc/internal/rb.h | 973 ---- .../include/jemalloc/internal/rtree.h | 161 - .../include/jemalloc/internal/stats.h | 207 - .../include/jemalloc/internal/tcache.h | 431 -- .../include/jemalloc/internal/zone.h | 23 - .../include/jemalloc/jemalloc.h.in | 66 - .../include/jemalloc/jemalloc_defs.h.in | 167 - deps/jemalloc.orig/install-sh | 250 - deps/jemalloc.orig/src/arena.c | 2704 --------- deps/jemalloc.orig/src/atomic.c | 2 - deps/jemalloc.orig/src/base.c | 106 - deps/jemalloc.orig/src/bitmap.c | 90 - deps/jemalloc.orig/src/chunk.c | 173 - deps/jemalloc.orig/src/chunk_dss.c | 284 - deps/jemalloc.orig/src/chunk_mmap.c | 239 - deps/jemalloc.orig/src/chunk_swap.c | 402 -- deps/jemalloc.orig/src/ckh.c | 619 --- deps/jemalloc.orig/src/ctl.c | 1670 ------ deps/jemalloc.orig/src/extent.c | 41 - deps/jemalloc.orig/src/hash.c | 2 - deps/jemalloc.orig/src/huge.c | 386 -- deps/jemalloc.orig/src/jemalloc.c | 1881 ------- deps/jemalloc.orig/src/mb.c | 2 - deps/jemalloc.orig/src/mutex.c | 90 - deps/jemalloc.orig/src/prof.c | 1244 ----- deps/jemalloc.orig/src/rtree.c | 46 - deps/jemalloc.orig/src/stats.c | 790 --- deps/jemalloc.orig/src/tcache.c | 480 -- deps/jemalloc.orig/src/zone.c | 354 -- deps/jemalloc.orig/test/allocated.c | 142 - deps/jemalloc.orig/test/allocated.exp | 2 - deps/jemalloc.orig/test/allocm.c | 133 - deps/jemalloc.orig/test/allocm.exp | 25 - deps/jemalloc.orig/test/bitmap.c | 157 - deps/jemalloc.orig/test/bitmap.exp | 2 - deps/jemalloc.orig/test/mremap.c | 67 - deps/jemalloc.orig/test/mremap.exp | 2 - deps/jemalloc.orig/test/posix_memalign.c | 121 - deps/jemalloc.orig/test/posix_memalign.exp | 25 - deps/jemalloc.orig/test/rallocm.c | 127 - deps/jemalloc.orig/test/rallocm.exp | 2 - deps/jemalloc.orig/test/thread_arena.c | 92 - deps/jemalloc.orig/test/thread_arena.exp | 2 - 80 files changed, 30365 deletions(-) delete mode 100644 deps/jemalloc.orig/.gitignore delete mode 100644 deps/jemalloc.orig/COPYING delete mode 100644 deps/jemalloc.orig/ChangeLog delete mode 100644 deps/jemalloc.orig/INSTALL delete mode 100644 deps/jemalloc.orig/Makefile.in delete mode 100644 deps/jemalloc.orig/README delete mode 100755 deps/jemalloc.orig/autogen.sh delete mode 100755 deps/jemalloc.orig/bin/pprof delete mode 100755 deps/jemalloc.orig/config.guess delete mode 100644 deps/jemalloc.orig/config.stamp.in delete mode 100755 deps/jemalloc.orig/config.sub delete mode 100644 deps/jemalloc.orig/configure.ac delete mode 100644 deps/jemalloc.orig/doc/html.xsl.in delete mode 100644 deps/jemalloc.orig/doc/jemalloc.xml.in delete mode 100644 deps/jemalloc.orig/doc/manpages.xsl.in delete mode 100644 deps/jemalloc.orig/doc/stylesheet.xsl delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/arena.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/atomic.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/base.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/bitmap.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/chunk_swap.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/ckh.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/ctl.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/extent.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/hash.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/huge.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/mb.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/mutex.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/prn.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/prof.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/ql.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/qr.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/rb.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/rtree.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/stats.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/tcache.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/internal/zone.h delete mode 100644 deps/jemalloc.orig/include/jemalloc/jemalloc.h.in delete mode 100644 deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in delete mode 100755 deps/jemalloc.orig/install-sh delete mode 100644 deps/jemalloc.orig/src/arena.c delete mode 100644 deps/jemalloc.orig/src/atomic.c delete mode 100644 deps/jemalloc.orig/src/base.c delete mode 100644 deps/jemalloc.orig/src/bitmap.c delete mode 100644 deps/jemalloc.orig/src/chunk.c delete mode 100644 deps/jemalloc.orig/src/chunk_dss.c delete mode 100644 deps/jemalloc.orig/src/chunk_mmap.c delete mode 100644 deps/jemalloc.orig/src/chunk_swap.c delete mode 100644 deps/jemalloc.orig/src/ckh.c delete mode 100644 deps/jemalloc.orig/src/ctl.c delete mode 100644 deps/jemalloc.orig/src/extent.c delete mode 100644 deps/jemalloc.orig/src/hash.c delete mode 100644 deps/jemalloc.orig/src/huge.c delete mode 100644 deps/jemalloc.orig/src/jemalloc.c delete mode 100644 deps/jemalloc.orig/src/mb.c delete mode 100644 deps/jemalloc.orig/src/mutex.c delete mode 100644 deps/jemalloc.orig/src/prof.c delete mode 100644 deps/jemalloc.orig/src/rtree.c delete mode 100644 deps/jemalloc.orig/src/stats.c delete mode 100644 deps/jemalloc.orig/src/tcache.c delete mode 100644 deps/jemalloc.orig/src/zone.c delete mode 100644 deps/jemalloc.orig/test/allocated.c delete mode 100644 deps/jemalloc.orig/test/allocated.exp delete mode 100644 deps/jemalloc.orig/test/allocm.c delete mode 100644 deps/jemalloc.orig/test/allocm.exp delete mode 100644 deps/jemalloc.orig/test/bitmap.c delete mode 100644 deps/jemalloc.orig/test/bitmap.exp delete mode 100644 deps/jemalloc.orig/test/mremap.c delete mode 100644 deps/jemalloc.orig/test/mremap.exp delete mode 100644 deps/jemalloc.orig/test/posix_memalign.c delete mode 100644 deps/jemalloc.orig/test/posix_memalign.exp delete mode 100644 deps/jemalloc.orig/test/rallocm.c delete mode 100644 deps/jemalloc.orig/test/rallocm.exp delete mode 100644 deps/jemalloc.orig/test/thread_arena.c delete mode 100644 deps/jemalloc.orig/test/thread_arena.exp diff --git a/deps/jemalloc.orig/.gitignore b/deps/jemalloc.orig/.gitignore deleted file mode 100644 index 32b4c424bde..00000000000 --- a/deps/jemalloc.orig/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -/autom4te.cache/ -/config.stamp -/config.log -/config.status -/configure -/doc/html.xsl -/doc/manpages.xsl -/doc/jemalloc.xml -/doc/jemalloc.html -/doc/jemalloc.3 -/lib/ -/Makefile -/include/jemalloc/internal/jemalloc_internal\.h -/include/jemalloc/jemalloc\.h -/include/jemalloc/jemalloc_defs\.h -/test/jemalloc_test\.h -/src/*.[od] -/test/*.[od] -/test/*.out -/test/[a-z]* -!test/*.c -!test/*.exp -/VERSION diff --git a/deps/jemalloc.orig/COPYING b/deps/jemalloc.orig/COPYING deleted file mode 100644 index 10ade120049..00000000000 --- a/deps/jemalloc.orig/COPYING +++ /dev/null @@ -1,51 +0,0 @@ -Unless otherwise specified, files in the jemalloc source distribution are -subject to the following licenses: --------------------------------------------------------------------------------- -Copyright (C) 2002-2010 Jason Evans . -All rights reserved. -Copyright (C) 2007-2010 Mozilla Foundation. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: -1. Redistributions of source code must retain the above copyright notice(s), - this list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice(s), - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER(S) ``AS IS'' AND ANY EXPRESS -OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO -EVENT SHALL THE COPYRIGHT HOLDER(S) BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE -OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF -ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- -Copyright (C) 2009-2010 Facebook, Inc. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Facebook, Inc. nor the names of its contributors may be - used to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --------------------------------------------------------------------------------- diff --git a/deps/jemalloc.orig/ChangeLog b/deps/jemalloc.orig/ChangeLog deleted file mode 100644 index 326ee7a97e1..00000000000 --- a/deps/jemalloc.orig/ChangeLog +++ /dev/null @@ -1,250 +0,0 @@ -Following are change highlights associated with official releases. Important -bug fixes are all mentioned, but internal enhancements are omitted here for -brevity (even though they are more fun to write about). Much more detail can be -found in the git revision history: - - http://www.canonware.com/cgi-bin/gitweb.cgi?p=jemalloc.git - git://canonware.com/jemalloc.git - -* 2.2.5 (November 14, 2011) - - Bug fixes: - - Fix huge_ralloc() race when using mremap(2). This is a serious bug that - could cause memory corruption and/or crashes. - - Fix huge_ralloc() to maintain chunk statistics. - - Fix malloc_stats_print(..., "a") output. - -* 2.2.4 (November 5, 2011) - - Bug fixes: - - Initialize arenas_tsd before using it. This bug existed for 2.2.[0-3], as - well as for --disable-tls builds in earlier releases. - - Do not assume a 4 KiB page size in test/rallocm.c. - -* 2.2.3 (August 31, 2011) - - This version fixes numerous bugs related to heap profiling. - - Bug fixes: - - Fix a prof-related race condition. This bug could cause memory corruption, - but only occurred in non-default configurations (prof_accum:false). - - Fix off-by-one backtracing issues (make sure that prof_alloc_prep() is - excluded from backtraces). - - Fix a prof-related bug in realloc() (only triggered by OOM errors). - - Fix prof-related bugs in allocm() and rallocm(). - - Fix prof_tdata_cleanup() for --disable-tls builds. - - Fix a relative include path, to fix objdir builds. - -* 2.2.2 (July 30, 2011) - - Bug fixes: - - Fix a build error for --disable-tcache. - - Fix assertions in arena_purge() (for real this time). - - Add the --with-private-namespace option. This is a workaround for symbol - conflicts that can inadvertently arise when using static libraries. - -* 2.2.1 (March 30, 2011) - - Bug fixes: - - Implement atomic operations for x86/x64. This fixes compilation failures - for versions of gcc that are still in wide use. - - Fix an assertion in arena_purge(). - -* 2.2.0 (March 22, 2011) - - This version incorporates several improvements to algorithms and data - structures that tend to reduce fragmentation and increase speed. - - New features: - - Add the "stats.cactive" mallctl. - - Update pprof (from google-perftools 1.7). - - Improve backtracing-related configuration logic, and add the - --disable-prof-libgcc option. - - Bug fixes: - - Change default symbol visibility from "internal", to "hidden", which - decreases the overhead of library-internal function calls. - - Fix symbol visibility so that it is also set on OS X. - - Fix a build dependency regression caused by the introduction of the .pic.o - suffix for PIC object files. - - Add missing checks for mutex initialization failures. - - Don't use libgcc-based backtracing except on x64, where it is known to work. - - Fix deadlocks on OS X that were due to memory allocation in - pthread_mutex_lock(). - - Heap profiling-specific fixes: - + Fix memory corruption due to integer overflow in small region index - computation, when using a small enough sample interval that profiling - context pointers are stored in small run headers. - + Fix a bootstrap ordering bug that only occurred with TLS disabled. - + Fix a rallocm() rsize bug. - + Fix error detection bugs for aligned memory allocation. - -* 2.1.3 (March 14, 2011) - - Bug fixes: - - Fix a cpp logic regression (due to the "thread.{de,}allocatedp" mallctl fix - for OS X in 2.1.2). - - Fix a "thread.arena" mallctl bug. - - Fix a thread cache stats merging bug. - -* 2.1.2 (March 2, 2011) - - Bug fixes: - - Fix "thread.{de,}allocatedp" mallctl for OS X. - - Add missing jemalloc.a to build system. - -* 2.1.1 (January 31, 2011) - - Bug fixes: - - Fix aligned huge reallocation (affected allocm()). - - Fix the ALLOCM_LG_ALIGN macro definition. - - Fix a heap dumping deadlock. - - Fix a "thread.arena" mallctl bug. - -* 2.1.0 (December 3, 2010) - - This version incorporates some optimizations that can't quite be considered - bug fixes. - - New features: - - Use Linux's mremap(2) for huge object reallocation when possible. - - Avoid locking in mallctl*() when possible. - - Add the "thread.[de]allocatedp" mallctl's. - - Convert the manual page source from roff to DocBook, and generate both roff - and HTML manuals. - - Bug fixes: - - Fix a crash due to incorrect bootstrap ordering. This only impacted - --enable-debug --enable-dss configurations. - - Fix a minor statistics bug for mallctl("swap.avail", ...). - -* 2.0.1 (October 29, 2010) - - Bug fixes: - - Fix a race condition in heap profiling that could cause undefined behavior - if "opt.prof_accum" were disabled. - - Add missing mutex unlocks for some OOM error paths in the heap profiling - code. - - Fix a compilation error for non-C99 builds. - -* 2.0.0 (October 24, 2010) - - This version focuses on the experimental *allocm() API, and on improved - run-time configuration/introspection. Nonetheless, numerous performance - improvements are also included. - - New features: - - Implement the experimental {,r,s,d}allocm() API, which provides a superset - of the functionality available via malloc(), calloc(), posix_memalign(), - realloc(), malloc_usable_size(), and free(). These functions can be used to - allocate/reallocate aligned zeroed memory, ask for optional extra memory - during reallocation, prevent object movement during reallocation, etc. - - Replace JEMALLOC_OPTIONS/JEMALLOC_PROF_PREFIX with MALLOC_CONF, which is - more human-readable, and more flexible. For example: - JEMALLOC_OPTIONS=AJP - is now: - MALLOC_CONF=abort:true,fill:true,stats_print:true - - Port to Apple OS X. Sponsored by Mozilla. - - Make it possible for the application to control thread-->arena mappings via - the "thread.arena" mallctl. - - Add compile-time support for all TLS-related functionality via pthreads TSD. - This is mainly of interest for OS X, which does not support TLS, but has a - TSD implementation with similar performance. - - Override memalign() and valloc() if they are provided by the system. - - Add the "arenas.purge" mallctl, which can be used to synchronously purge all - dirty unused pages. - - Make cumulative heap profiling data optional, so that it is possible to - limit the amount of memory consumed by heap profiling data structures. - - Add per thread allocation counters that can be accessed via the - "thread.allocated" and "thread.deallocated" mallctls. - - Incompatible changes: - - Remove JEMALLOC_OPTIONS and malloc_options (see MALLOC_CONF above). - - Increase default backtrace depth from 4 to 128 for heap profiling. - - Disable interval-based profile dumps by default. - - Bug fixes: - - Remove bad assertions in fork handler functions. These assertions could - cause aborts for some combinations of configure settings. - - Fix strerror_r() usage to deal with non-standard semantics in GNU libc. - - Fix leak context reporting. This bug tended to cause the number of contexts - to be underreported (though the reported number of objects and bytes were - correct). - - Fix a realloc() bug for large in-place growing reallocation. This bug could - cause memory corruption, but it was hard to trigger. - - Fix an allocation bug for small allocations that could be triggered if - multiple threads raced to create a new run of backing pages. - - Enhance the heap profiler to trigger samples based on usable size, rather - than request size. - - Fix a heap profiling bug due to sometimes losing track of requested object - size for sampled objects. - -* 1.0.3 (August 12, 2010) - - Bug fixes: - - Fix the libunwind-based implementation of stack backtracing (used for heap - profiling). This bug could cause zero-length backtraces to be reported. - - Add a missing mutex unlock in library initialization code. If multiple - threads raced to initialize malloc, some of them could end up permanently - blocked. - -* 1.0.2 (May 11, 2010) - - Bug fixes: - - Fix junk filling of large objects, which could cause memory corruption. - - Add MAP_NORESERVE support for chunk mapping, because otherwise virtual - memory limits could cause swap file configuration to fail. Contributed by - Jordan DeLong. - -* 1.0.1 (April 14, 2010) - - Bug fixes: - - Fix compilation when --enable-fill is specified. - - Fix threads-related profiling bugs that affected accuracy and caused memory - to be leaked during thread exit. - - Fix dirty page purging race conditions that could cause crashes. - - Fix crash in tcache flushing code during thread destruction. - -* 1.0.0 (April 11, 2010) - - This release focuses on speed and run-time introspection. Numerous - algorithmic improvements make this release substantially faster than its - predecessors. - - New features: - - Implement autoconf-based configuration system. - - Add mallctl*(), for the purposes of introspection and run-time - configuration. - - Make it possible for the application to manually flush a thread's cache, via - the "tcache.flush" mallctl. - - Base maximum dirty page count on proportion of active memory. - - Compute various addtional run-time statistics, including per size class - statistics for large objects. - - Expose malloc_stats_print(), which can be called repeatedly by the - application. - - Simplify the malloc_message() signature to only take one string argument, - and incorporate an opaque data pointer argument for use by the application - in combination with malloc_stats_print(). - - Add support for allocation backed by one or more swap files, and allow the - application to disable over-commit if swap files are in use. - - Implement allocation profiling and leak checking. - - Removed features: - - Remove the dynamic arena rebalancing code, since thread-specific caching - reduces its utility. - - Bug fixes: - - Modify chunk allocation to work when address space layout randomization - (ASLR) is in use. - - Fix thread cleanup bugs related to TLS destruction. - - Handle 0-size allocation requests in posix_memalign(). - - Fix a chunk leak. The leaked chunks were never touched, so this impacted - virtual memory usage, but not physical memory usage. - -* linux_2008082[78]a (August 27/28, 2008) - - These snapshot releases are the simple result of incorporating Linux-specific - support into the FreeBSD malloc sources. - --------------------------------------------------------------------------------- -vim:filetype=text:textwidth=80 diff --git a/deps/jemalloc.orig/INSTALL b/deps/jemalloc.orig/INSTALL deleted file mode 100644 index 2a1e469cb8b..00000000000 --- a/deps/jemalloc.orig/INSTALL +++ /dev/null @@ -1,257 +0,0 @@ -Building and installing jemalloc can be as simple as typing the following while -in the root directory of the source tree: - - ./configure - make - make install - -=== Advanced configuration ===================================================== - -The 'configure' script supports numerous options that allow control of which -functionality is enabled, where jemalloc is installed, etc. Optionally, pass -any of the following arguments (not a definitive list) to 'configure': - ---help - Print a definitive list of options. - ---prefix= - Set the base directory in which to install. For example: - - ./configure --prefix=/usr/local - - will cause files to be installed into /usr/local/include, /usr/local/lib, - and /usr/local/man. - ---with-rpath= - Embed one or more library paths, so that libjemalloc can find the libraries - it is linked to. This works only on ELF-based systems. - ---with-jemalloc-prefix= - Prefix all public APIs with . For example, if is - "prefix_", API changes like the following occur: - - malloc() --> prefix_malloc() - malloc_conf --> prefix_malloc_conf - /etc/malloc.conf --> /etc/prefix_malloc.conf - MALLOC_CONF --> PREFIX_MALLOC_CONF - - This makes it possible to use jemalloc at the same time as the system - allocator, or even to use multiple copies of jemalloc simultaneously. - - By default, the prefix is "", except on OS X, where it is "je_". On OS X, - jemalloc overlays the default malloc zone, but makes no attempt to actually - replace the "malloc", "calloc", etc. symbols. - ---with-private-namespace= - Prefix all library-private APIs with . For shared libraries, - symbol visibility mechanisms prevent these symbols from being exported, but - for static libraries, naming collisions are a real possibility. By - default, the prefix is "" (empty string). - ---with-install-suffix= - Append to the base name of all installed files, such that multiple - versions of jemalloc can coexist in the same installation directory. For - example, libjemalloc.so.0 becomes libjemalloc.so.0. - ---enable-cc-silence - Enable code that silences non-useful compiler warnings. This is helpful - when trying to tell serious warnings from those due to compiler - limitations, but it potentially incurs a performance penalty. - ---enable-debug - Enable assertions and validation code. This incurs a substantial - performance hit, but is very useful during application development. - ---enable-stats - Enable statistics gathering functionality. See the "opt.stats_print" - option documentation for usage details. - ---enable-prof - Enable heap profiling and leak detection functionality. See the "opt.prof" - option documentation for usage details. When enabled, there are several - approaches to backtracing, and the configure script chooses the first one - in the following list that appears to function correctly: - - + libunwind (requires --enable-prof-libunwind) - + libgcc (unless --disable-prof-libgcc) - + gcc intrinsics (unless --disable-prof-gcc) - ---enable-prof-libunwind - Use the libunwind library (http://www.nongnu.org/libunwind/) for stack - backtracing. - ---disable-prof-libgcc - Disable the use of libgcc's backtracing functionality. - ---disable-prof-gcc - Disable the use of gcc intrinsics for backtracing. - ---with-static-libunwind= - Statically link against the specified libunwind.a rather than dynamically - linking with -lunwind. - ---disable-tiny - Disable tiny (sub-quantum-sized) object support. Technically it is not - legal for a malloc implementation to allocate objects with less than - quantum alignment (8 or 16 bytes, depending on architecture), but in - practice it never causes any problems if, for example, 4-byte allocations - are 4-byte-aligned. - ---disable-tcache - Disable thread-specific caches for small objects. Objects are cached and - released in bulk, thus reducing the total number of mutex operations. See - the "opt.tcache" option for usage details. - ---enable-swap - Enable mmap()ed swap file support. When this feature is built in, it is - possible to specify one or more files that act as backing store. This - effectively allows for per application swap files. - ---enable-dss - Enable support for page allocation/deallocation via sbrk(2), in addition to - mmap(2). - ---enable-fill - Enable support for junk/zero filling of memory. See the "opt.junk"/ - "opt.zero" option documentation for usage details. - ---enable-xmalloc - Enable support for optional immediate termination due to out-of-memory - errors, as is commonly implemented by "xmalloc" wrapper function for malloc. - See the "opt.xmalloc" option documentation for usage details. - ---enable-sysv - Enable support for System V semantics, wherein malloc(0) returns NULL - rather than a minimal allocation. See the "opt.sysv" option documentation - for usage details. - ---enable-dynamic-page-shift - Under most conditions, the system page size never changes (usually 4KiB or - 8KiB, depending on architecture and configuration), and unless this option - is enabled, jemalloc assumes that page size can safely be determined during - configuration and hard-coded. Enabling dynamic page size determination has - a measurable impact on performance, since the compiler is forced to load - the page size from memory rather than embedding immediate values. - ---disable-lazy-lock - Disable code that wraps pthread_create() to detect when an application - switches from single-threaded to multi-threaded mode, so that it can avoid - mutex locking/unlocking operations while in single-threaded mode. In - practice, this feature usually has little impact on performance unless - thread-specific caching is disabled. - ---disable-tls - Disable thread-local storage (TLS), which allows for fast access to - thread-local variables via the __thread keyword. If TLS is available, - jemalloc uses it for several purposes. - ---with-xslroot= - Specify where to find DocBook XSL stylesheets when building the - documentation. - -The following environment variables (not a definitive list) impact configure's -behavior: - -CFLAGS="?" - Pass these flags to the compiler. You probably shouldn't define this unless - you know what you are doing. (Use EXTRA_CFLAGS instead.) - -EXTRA_CFLAGS="?" - Append these flags to CFLAGS. This makes it possible to add flags such as - -Werror, while allowing the configure script to determine what other flags - are appropriate for the specified configuration. - - The configure script specifically checks whether an optimization flag (-O*) - is specified in EXTRA_CFLAGS, and refrains from specifying an optimization - level if it finds that one has already been specified. - -CPPFLAGS="?" - Pass these flags to the C preprocessor. Note that CFLAGS is not passed to - 'cpp' when 'configure' is looking for include files, so you must use - CPPFLAGS instead if you need to help 'configure' find header files. - -LD_LIBRARY_PATH="?" - 'ld' uses this colon-separated list to find libraries. - -LDFLAGS="?" - Pass these flags when linking. - -PATH="?" - 'configure' uses this to find programs. - -=== Advanced compilation ======================================================= - -To install only parts of jemalloc, use the following targets: - - install_bin - install_include - install_lib - install_doc - -To clean up build results to varying degrees, use the following make targets: - - clean - distclean - relclean - -=== Advanced installation ====================================================== - -Optionally, define make variables when invoking make, including (not -exclusively): - -INCLUDEDIR="?" - Use this as the installation prefix for header files. - -LIBDIR="?" - Use this as the installation prefix for libraries. - -MANDIR="?" - Use this as the installation prefix for man pages. - -DESTDIR="?" - Prepend DESTDIR to INCLUDEDIR, LIBDIR, DATADIR, and MANDIR. This is useful - when installing to a different path than was specified via --prefix. - -CC="?" - Use this to invoke the C compiler. - -CFLAGS="?" - Pass these flags to the compiler. - -CPPFLAGS="?" - Pass these flags to the C preprocessor. - -LDFLAGS="?" - Pass these flags when linking. - -PATH="?" - Use this to search for programs used during configuration and building. - -=== Development ================================================================ - -If you intend to make non-trivial changes to jemalloc, use the 'autogen.sh' -script rather than 'configure'. This re-generates 'configure', enables -configuration dependency rules, and enables re-generation of automatically -generated source files. - -The build system supports using an object directory separate from the source -tree. For example, you can create an 'obj' directory, and from within that -directory, issue configuration and build commands: - - autoconf - mkdir obj - cd obj - ../configure --enable-autogen - make - -=== Documentation ============================================================== - -The manual page is generated in both html and roff formats. Any web browser -can be used to view the html manual. The roff manual page can be formatted -prior to installation via any of the following commands: - - nroff -man -t doc/jemalloc.3 - - groff -man -t -Tps doc/jemalloc.3 | ps2pdf - doc/jemalloc.3.pdf - - (cd doc; groff -man -man-ext -t -Thtml jemalloc.3 > jemalloc.3.html) diff --git a/deps/jemalloc.orig/Makefile.in b/deps/jemalloc.orig/Makefile.in deleted file mode 100644 index de7492f951a..00000000000 --- a/deps/jemalloc.orig/Makefile.in +++ /dev/null @@ -1,259 +0,0 @@ -# Clear out all vpaths, then set just one (default vpath) for the main build -# directory. -vpath -vpath % . - -# Clear the default suffixes, so that built-in rules are not used. -.SUFFIXES : - -SHELL := /bin/sh - -CC := @CC@ - -# Configuration parameters. -DESTDIR = -BINDIR := $(DESTDIR)@BINDIR@ -INCLUDEDIR := $(DESTDIR)@INCLUDEDIR@ -LIBDIR := $(DESTDIR)@LIBDIR@ -DATADIR := $(DESTDIR)@DATADIR@ -MANDIR := $(DESTDIR)@MANDIR@ - -# Build parameters. -CPPFLAGS := @CPPFLAGS@ -I@srcroot@include -I@objroot@include -CFLAGS := @CFLAGS@ -ifeq (macho, @abi@) -CFLAGS += -dynamic -endif -LDFLAGS := @LDFLAGS@ -LIBS := @LIBS@ -RPATH_EXTRA := @RPATH_EXTRA@ -ifeq (macho, @abi@) -SO := dylib -WL_SONAME := dylib_install_name -else -SO := so -WL_SONAME := soname -endif -REV := 1 -ifeq (macho, @abi@) -TEST_LIBRARY_PATH := DYLD_FALLBACK_LIBRARY_PATH=@objroot@lib -else -TEST_LIBRARY_PATH := -endif - -# Lists of files. -BINS := @srcroot@bin/pprof -CHDRS := @objroot@include/jemalloc/jemalloc@install_suffix@.h \ - @objroot@include/jemalloc/jemalloc_defs@install_suffix@.h -CSRCS := @srcroot@src/jemalloc.c @srcroot@src/arena.c @srcroot@src/atomic.c \ - @srcroot@src/base.c @srcroot@src/bitmap.c @srcroot@src/chunk.c \ - @srcroot@src/chunk_dss.c @srcroot@src/chunk_mmap.c \ - @srcroot@src/chunk_swap.c @srcroot@src/ckh.c @srcroot@src/ctl.c \ - @srcroot@src/extent.c @srcroot@src/hash.c @srcroot@src/huge.c \ - @srcroot@src/mb.c @srcroot@src/mutex.c @srcroot@src/prof.c \ - @srcroot@src/rtree.c @srcroot@src/stats.c @srcroot@src/tcache.c -ifeq (macho, @abi@) -CSRCS += @srcroot@src/zone.c -endif -STATIC_LIBS := @objroot@lib/libjemalloc@install_suffix@.a -DSOS := @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) \ - @objroot@lib/libjemalloc@install_suffix@.$(SO) \ - @objroot@lib/libjemalloc@install_suffix@_pic.a -MAN3 := @objroot@doc/jemalloc@install_suffix@.3 -DOCS_XML := @objroot@doc/jemalloc@install_suffix@.xml -DOCS_HTML := $(DOCS_XML:@objroot@%.xml=@srcroot@%.html) -DOCS_MAN3 := $(DOCS_XML:@objroot@%.xml=@srcroot@%.3) -DOCS := $(DOCS_HTML) $(DOCS_MAN3) -CTESTS := @srcroot@test/allocated.c @srcroot@test/allocm.c \ - @srcroot@test/bitmap.c @srcroot@test/mremap.c \ - @srcroot@test/posix_memalign.c @srcroot@test/rallocm.c \ - @srcroot@test/thread_arena.c - -.PHONY: all dist doc_html doc_man doc -.PHONY: install_bin install_include install_lib -.PHONY: install_html install_man install_doc install -.PHONY: tests check clean distclean relclean - -.SECONDARY : $(CTESTS:@srcroot@%.c=@objroot@%.o) - -# Default target. -all: $(DSOS) $(STATIC_LIBS) - -dist: doc - -@srcroot@doc/%.html : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/html.xsl - @XSLTPROC@ -o $@ @objroot@doc/html.xsl $< - -@srcroot@doc/%.3 : @objroot@doc/%.xml @srcroot@doc/stylesheet.xsl @objroot@doc/manpages.xsl - @XSLTPROC@ -o $@ @objroot@doc/manpages.xsl $< - -doc_html: $(DOCS_HTML) -doc_man: $(DOCS_MAN3) -doc: $(DOCS) - -# -# Include generated dependency files. -# --include $(CSRCS:@srcroot@%.c=@objroot@%.d) --include $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) --include $(CTESTS:@srcroot@%.c=@objroot@%.d) - -@objroot@src/%.o: @srcroot@src/%.c - @mkdir -p $(@D) - $(CC) $(CFLAGS) -c $(CPPFLAGS) -o $@ $< - @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $@)))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.o \2/g\" > $(@:%.o=%.d)" - -@objroot@src/%.pic.o: @srcroot@src/%.c - @mkdir -p $(@D) - $(CC) $(CFLAGS) -fPIC -DPIC -c $(CPPFLAGS) -o $@ $< - @$(SHELL) -ec "$(CC) -MM $(CPPFLAGS) $< | sed \"s/\($(subst /,\/,$(notdir $(basename $(basename $@))))\)\.o\([ :]*\)/$(subst /,\/,$(strip $(dir $@)))\1.pic.o \2/g\" > $(@:%.o=%.d)" - -%.$(SO) : %.$(SO).$(REV) - @mkdir -p $(@D) - ln -sf $( $(@:%.o=%.d)" - -# Automatic dependency generation misses #include "*.c". -@objroot@test/bitmap.o : @objroot@src/bitmap.o - -@objroot@test/%: @objroot@test/%.o \ - @objroot@lib/libjemalloc@install_suffix@.$(SO) - @mkdir -p $(@D) -ifneq (@RPATH@, ) - $(CC) -o $@ $< @RPATH@@objroot@lib -L@objroot@lib -ljemalloc@install_suffix@ -lpthread -else - $(CC) -o $@ $< -L@objroot@lib -ljemalloc@install_suffix@ -lpthread -endif - -install_bin: - install -d $(BINDIR) - @for b in $(BINS); do \ - echo "install -m 755 $$b $(BINDIR)"; \ - install -m 755 $$b $(BINDIR); \ -done - -install_include: - install -d $(INCLUDEDIR)/jemalloc - @for h in $(CHDRS); do \ - echo "install -m 644 $$h $(INCLUDEDIR)/jemalloc"; \ - install -m 644 $$h $(INCLUDEDIR)/jemalloc; \ -done - -install_lib: $(DSOS) $(STATIC_LIBS) - install -d $(LIBDIR) - install -m 755 @objroot@lib/libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR) - ln -sf libjemalloc@install_suffix@.$(SO).$(REV) $(LIBDIR)/libjemalloc@install_suffix@.$(SO) - install -m 755 @objroot@lib/libjemalloc@install_suffix@_pic.a $(LIBDIR) - install -m 755 @objroot@lib/libjemalloc@install_suffix@.a $(LIBDIR) - -install_html: - install -d $(DATADIR)/doc/jemalloc@install_suffix@ - @for d in $(DOCS_HTML); do \ - echo "install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@"; \ - install -m 644 $$d $(DATADIR)/doc/jemalloc@install_suffix@; \ -done - -install_man: - install -d $(MANDIR)/man3 - @for d in $(DOCS_MAN3); do \ - echo "install -m 644 $$d $(MANDIR)/man3"; \ - install -m 644 $$d $(MANDIR)/man3; \ -done - -install_doc: install_html install_man - -install: install_bin install_include install_lib install_doc - -tests: $(CTESTS:@srcroot@%.c=@objroot@%) - -check: tests - @mkdir -p @objroot@test - @$(SHELL) -c 'total=0; \ - failures=0; \ - echo "========================================="; \ - for t in $(CTESTS:@srcroot@%.c=@objroot@%); do \ - total=`expr $$total + 1`; \ - /bin/echo -n "$${t} ... "; \ - $(TEST_LIBRARY_PATH) $${t} @abs_srcroot@ @abs_objroot@ \ - > @objroot@$${t}.out 2>&1; \ - if test -e "@srcroot@$${t}.exp"; then \ - diff -u @srcroot@$${t}.exp \ - @objroot@$${t}.out >/dev/null 2>&1; \ - fail=$$?; \ - if test "$${fail}" -eq "1" ; then \ - failures=`expr $${failures} + 1`; \ - echo "*** FAIL ***"; \ - else \ - echo "pass"; \ - fi; \ - else \ - echo "*** FAIL *** (.exp file is missing)"; \ - failures=`expr $${failures} + 1`; \ - fi; \ - done; \ - echo "========================================="; \ - echo "Failures: $${failures}/$${total}"' - -clean: - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.o) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.o) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.d) - rm -f $(CSRCS:@srcroot@%.c=@objroot@%.pic.d) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.o) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.d) - rm -f $(CTESTS:@srcroot@%.c=@objroot@%.out) - rm -f $(DSOS) $(STATIC_LIBS) - -distclean: clean - rm -rf @objroot@autom4te.cache - rm -f @objroot@config.log - rm -f @objroot@config.status - rm -f @objroot@config.stamp - rm -f @cfghdrs_out@ - rm -f @cfgoutputs_out@ - -relclean: distclean - rm -f @objroot@configure - rm -f @srcroot@VERSION - rm -f $(DOCS_HTML) - rm -f $(DOCS_MAN3) - -#=============================================================================== -# Re-configuration rules. - -ifeq (@enable_autogen@, 1) -@srcroot@configure : @srcroot@configure.ac - cd ./@srcroot@ && @AUTOCONF@ - -@objroot@config.status : @srcroot@configure - ./@objroot@config.status --recheck - -@srcroot@config.stamp.in : @srcroot@configure.ac - echo stamp > @srcroot@config.stamp.in - -@objroot@config.stamp : @cfgoutputs_in@ @cfghdrs_in@ @srcroot@configure - ./@objroot@config.status - @touch $@ - -# There must be some action in order for make to re-read Makefile when it is -# out of date. -@cfgoutputs_out@ @cfghdrs_out@ : @objroot@config.stamp - @true -endif diff --git a/deps/jemalloc.orig/README b/deps/jemalloc.orig/README deleted file mode 100644 index 4d7b552bf3d..00000000000 --- a/deps/jemalloc.orig/README +++ /dev/null @@ -1,16 +0,0 @@ -jemalloc is a general-purpose scalable concurrent malloc(3) implementation. -This distribution is a stand-alone "portable" implementation that currently -targets Linux and Apple OS X. jemalloc is included as the default allocator in -the FreeBSD and NetBSD operating systems, and it is used by the Mozilla Firefox -web browser on Microsoft Windows-related platforms. Depending on your needs, -one of the other divergent versions may suit your needs better than this -distribution. - -The COPYING file contains copyright and licensing information. - -The INSTALL file contains information on how to configure, build, and install -jemalloc. - -The ChangeLog file contains a brief summary of changes for each release. - -URL: http://www.canonware.com/jemalloc/ diff --git a/deps/jemalloc.orig/autogen.sh b/deps/jemalloc.orig/autogen.sh deleted file mode 100755 index 75f32da6873..00000000000 --- a/deps/jemalloc.orig/autogen.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh - -for i in autoconf; do - echo "$i" - $i - if [ $? -ne 0 ]; then - echo "Error $? in $i" - exit 1 - fi -done - -echo "./configure --enable-autogen $@" -./configure --enable-autogen $@ -if [ $? -ne 0 ]; then - echo "Error $? in ./configure" - exit 1 -fi diff --git a/deps/jemalloc.orig/bin/pprof b/deps/jemalloc.orig/bin/pprof deleted file mode 100755 index 280ddcc8329..00000000000 --- a/deps/jemalloc.orig/bin/pprof +++ /dev/null @@ -1,4893 +0,0 @@ -#! /usr/bin/env perl - -# Copyright (c) 1998-2007, Google Inc. -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# --- -# Program for printing the profile generated by common/profiler.cc, -# or by the heap profiler (common/debugallocation.cc) -# -# The profile contains a sequence of entries of the form: -# -# This program parses the profile, and generates user-readable -# output. -# -# Examples: -# -# % tools/pprof "program" "profile" -# Enters "interactive" mode -# -# % tools/pprof --text "program" "profile" -# Generates one line per procedure -# -# % tools/pprof --gv "program" "profile" -# Generates annotated call-graph and displays via "gv" -# -# % tools/pprof --gv --focus=Mutex "program" "profile" -# Restrict to code paths that involve an entry that matches "Mutex" -# -# % tools/pprof --gv --focus=Mutex --ignore=string "program" "profile" -# Restrict to code paths that involve an entry that matches "Mutex" -# and does not match "string" -# -# % tools/pprof --list=IBF_CheckDocid "program" "profile" -# Generates disassembly listing of all routines with at least one -# sample that match the --list= pattern. The listing is -# annotated with the flat and cumulative sample counts at each line. -# -# % tools/pprof --disasm=IBF_CheckDocid "program" "profile" -# Generates disassembly listing of all routines with at least one -# sample that match the --disasm= pattern. The listing is -# annotated with the flat and cumulative sample counts at each PC value. -# -# TODO: Use color to indicate files? - -use strict; -use warnings; -use Getopt::Long; - -my $PPROF_VERSION = "1.7"; - -# These are the object tools we use which can come from a -# user-specified location using --tools, from the PPROF_TOOLS -# environment variable, or from the environment. -my %obj_tool_map = ( - "objdump" => "objdump", - "nm" => "nm", - "addr2line" => "addr2line", - "c++filt" => "c++filt", - ## ConfigureObjTools may add architecture-specific entries: - #"nm_pdb" => "nm-pdb", # for reading windows (PDB-format) executables - #"addr2line_pdb" => "addr2line-pdb", # ditto - #"otool" => "otool", # equivalent of objdump on OS X -); -my $DOT = "dot"; # leave non-absolute, since it may be in /usr/local -my $GV = "gv"; -my $EVINCE = "evince"; # could also be xpdf or perhaps acroread -my $KCACHEGRIND = "kcachegrind"; -my $PS2PDF = "ps2pdf"; -# These are used for dynamic profiles -my $URL_FETCHER = "curl -s"; - -# These are the web pages that servers need to support for dynamic profiles -my $HEAP_PAGE = "/pprof/heap"; -my $PROFILE_PAGE = "/pprof/profile"; # must support cgi-param "?seconds=#" -my $PMUPROFILE_PAGE = "/pprof/pmuprofile(?:\\?.*)?"; # must support cgi-param - # ?seconds=#&event=x&period=n -my $GROWTH_PAGE = "/pprof/growth"; -my $CONTENTION_PAGE = "/pprof/contention"; -my $WALL_PAGE = "/pprof/wall(?:\\?.*)?"; # accepts options like namefilter -my $FILTEREDPROFILE_PAGE = "/pprof/filteredprofile(?:\\?.*)?"; -my $CENSUSPROFILE_PAGE = "/pprof/censusprofile"; # must support "?seconds=#" -my $SYMBOL_PAGE = "/pprof/symbol"; # must support symbol lookup via POST -my $PROGRAM_NAME_PAGE = "/pprof/cmdline"; - -# These are the web pages that can be named on the command line. -# All the alternatives must begin with /. -my $PROFILES = "($HEAP_PAGE|$PROFILE_PAGE|$PMUPROFILE_PAGE|" . - "$GROWTH_PAGE|$CONTENTION_PAGE|$WALL_PAGE|" . - "$FILTEREDPROFILE_PAGE|$CENSUSPROFILE_PAGE)"; - -# default binary name -my $UNKNOWN_BINARY = "(unknown)"; - -# There is a pervasive dependency on the length (in hex characters, -# i.e., nibbles) of an address, distinguishing between 32-bit and -# 64-bit profiles. To err on the safe size, default to 64-bit here: -my $address_length = 16; - -# A list of paths to search for shared object files -my @prefix_list = (); - -# Special routine name that should not have any symbols. -# Used as separator to parse "addr2line -i" output. -my $sep_symbol = '_fini'; -my $sep_address = undef; - -##### Argument parsing ##### - -sub usage_string { - return < - is a space separated list of profile names. -pprof [options] - is a list of profile files where each file contains - the necessary symbol mappings as well as profile data (likely generated - with --raw). -pprof [options] - is a remote form. Symbols are obtained from host:port$SYMBOL_PAGE - - Each name can be: - /path/to/profile - a path to a profile file - host:port[/] - a location of a service to get profile from - - The / can be $HEAP_PAGE, $PROFILE_PAGE, /pprof/pmuprofile, - $GROWTH_PAGE, $CONTENTION_PAGE, /pprof/wall, - $CENSUSPROFILE_PAGE, or /pprof/filteredprofile. - For instance: "pprof http://myserver.com:80$HEAP_PAGE". - If / is omitted, the service defaults to $PROFILE_PAGE (cpu profiling). -pprof --symbols - Maps addresses to symbol names. In this mode, stdin should be a - list of library mappings, in the same format as is found in the heap- - and cpu-profile files (this loosely matches that of /proc/self/maps - on linux), followed by a list of hex addresses to map, one per line. - - For more help with querying remote servers, including how to add the - necessary server-side support code, see this filename (or one like it): - - /usr/doc/google-perftools-$PPROF_VERSION/pprof_remote_servers.html - -Options: - --cum Sort by cumulative data - --base= Subtract from before display - --interactive Run in interactive mode (interactive "help" gives help) [default] - --seconds= Length of time for dynamic profiles [default=30 secs] - --add_lib= Read additional symbols and line info from the given library - --lib_prefix= Comma separated list of library path prefixes - -Reporting Granularity: - --addresses Report at address level - --lines Report at source line level - --functions Report at function level [default] - --files Report at source file level - -Output type: - --text Generate text report - --callgrind Generate callgrind format to stdout - --gv Generate Postscript and display - --evince Generate PDF and display - --web Generate SVG and display - --list= Generate source listing of matching routines - --disasm= Generate disassembly of matching routines - --symbols Print demangled symbol names found at given addresses - --dot Generate DOT file to stdout - --ps Generate Postcript to stdout - --pdf Generate PDF to stdout - --svg Generate SVG to stdout - --gif Generate GIF to stdout - --raw Generate symbolized pprof data (useful with remote fetch) - -Heap-Profile Options: - --inuse_space Display in-use (mega)bytes [default] - --inuse_objects Display in-use objects - --alloc_space Display allocated (mega)bytes - --alloc_objects Display allocated objects - --show_bytes Display space in bytes - --drop_negative Ignore negative differences - -Contention-profile options: - --total_delay Display total delay at each region [default] - --contentions Display number of delays at each region - --mean_delay Display mean delay at each region - -Call-graph Options: - --nodecount= Show at most so many nodes [default=80] - --nodefraction= Hide nodes below *total [default=.005] - --edgefraction= Hide edges below *total [default=.001] - --maxdegree= Max incoming/outgoing edges per node [default=8] - --focus= Focus on nodes matching - --ignore= Ignore nodes matching - --scale= Set GV scaling [default=0] - --heapcheck Make nodes with non-0 object counts - (i.e. direct leak generators) more visible - -Miscellaneous: - --tools=[,...] \$PATH for object tool pathnames - --test Run unit tests - --help This message - --version Version information - -Environment Variables: - PPROF_TMPDIR Profiles directory. Defaults to \$HOME/pprof - PPROF_TOOLS Prefix for object tools pathnames - -Examples: - -pprof /bin/ls ls.prof - Enters "interactive" mode -pprof --text /bin/ls ls.prof - Outputs one line per procedure -pprof --web /bin/ls ls.prof - Displays annotated call-graph in web browser -pprof --gv /bin/ls ls.prof - Displays annotated call-graph via 'gv' -pprof --gv --focus=Mutex /bin/ls ls.prof - Restricts to code paths including a .*Mutex.* entry -pprof --gv --focus=Mutex --ignore=string /bin/ls ls.prof - Code paths including Mutex but not string -pprof --list=getdir /bin/ls ls.prof - (Per-line) annotated source listing for getdir() -pprof --disasm=getdir /bin/ls ls.prof - (Per-PC) annotated disassembly for getdir() - -pprof http://localhost:1234/ - Enters "interactive" mode -pprof --text localhost:1234 - Outputs one line per procedure for localhost:1234 -pprof --raw localhost:1234 > ./local.raw -pprof --text ./local.raw - Fetches a remote profile for later analysis and then - analyzes it in text mode. -EOF -} - -sub version_string { - return < \$main::opt_help, - "version!" => \$main::opt_version, - "cum!" => \$main::opt_cum, - "base=s" => \$main::opt_base, - "seconds=i" => \$main::opt_seconds, - "add_lib=s" => \$main::opt_lib, - "lib_prefix=s" => \$main::opt_lib_prefix, - "functions!" => \$main::opt_functions, - "lines!" => \$main::opt_lines, - "addresses!" => \$main::opt_addresses, - "files!" => \$main::opt_files, - "text!" => \$main::opt_text, - "callgrind!" => \$main::opt_callgrind, - "list=s" => \$main::opt_list, - "disasm=s" => \$main::opt_disasm, - "symbols!" => \$main::opt_symbols, - "gv!" => \$main::opt_gv, - "evince!" => \$main::opt_evince, - "web!" => \$main::opt_web, - "dot!" => \$main::opt_dot, - "ps!" => \$main::opt_ps, - "pdf!" => \$main::opt_pdf, - "svg!" => \$main::opt_svg, - "gif!" => \$main::opt_gif, - "raw!" => \$main::opt_raw, - "interactive!" => \$main::opt_interactive, - "nodecount=i" => \$main::opt_nodecount, - "nodefraction=f" => \$main::opt_nodefraction, - "edgefraction=f" => \$main::opt_edgefraction, - "maxdegree=i" => \$main::opt_maxdegree, - "focus=s" => \$main::opt_focus, - "ignore=s" => \$main::opt_ignore, - "scale=i" => \$main::opt_scale, - "heapcheck" => \$main::opt_heapcheck, - "inuse_space!" => \$main::opt_inuse_space, - "inuse_objects!" => \$main::opt_inuse_objects, - "alloc_space!" => \$main::opt_alloc_space, - "alloc_objects!" => \$main::opt_alloc_objects, - "show_bytes!" => \$main::opt_show_bytes, - "drop_negative!" => \$main::opt_drop_negative, - "total_delay!" => \$main::opt_total_delay, - "contentions!" => \$main::opt_contentions, - "mean_delay!" => \$main::opt_mean_delay, - "tools=s" => \$main::opt_tools, - "test!" => \$main::opt_test, - "debug!" => \$main::opt_debug, - # Undocumented flags used only by unittests: - "test_stride=i" => \$main::opt_test_stride, - ) || usage("Invalid option(s)"); - - # Deal with the standard --help and --version - if ($main::opt_help) { - print usage_string(); - exit(0); - } - - if ($main::opt_version) { - print version_string(); - exit(0); - } - - # Disassembly/listing/symbols mode requires address-level info - if ($main::opt_disasm || $main::opt_list || $main::opt_symbols) { - $main::opt_functions = 0; - $main::opt_lines = 0; - $main::opt_addresses = 1; - $main::opt_files = 0; - } - - # Check heap-profiling flags - if ($main::opt_inuse_space + - $main::opt_inuse_objects + - $main::opt_alloc_space + - $main::opt_alloc_objects > 1) { - usage("Specify at most on of --inuse/--alloc options"); - } - - # Check output granularities - my $grains = - $main::opt_functions + - $main::opt_lines + - $main::opt_addresses + - $main::opt_files + - 0; - if ($grains > 1) { - usage("Only specify one output granularity option"); - } - if ($grains == 0) { - $main::opt_functions = 1; - } - - # Check output modes - my $modes = - $main::opt_text + - $main::opt_callgrind + - ($main::opt_list eq '' ? 0 : 1) + - ($main::opt_disasm eq '' ? 0 : 1) + - ($main::opt_symbols == 0 ? 0 : 1) + - $main::opt_gv + - $main::opt_evince + - $main::opt_web + - $main::opt_dot + - $main::opt_ps + - $main::opt_pdf + - $main::opt_svg + - $main::opt_gif + - $main::opt_raw + - $main::opt_interactive + - 0; - if ($modes > 1) { - usage("Only specify one output mode"); - } - if ($modes == 0) { - if (-t STDOUT) { # If STDOUT is a tty, activate interactive mode - $main::opt_interactive = 1; - } else { - $main::opt_text = 1; - } - } - - if ($main::opt_test) { - RunUnitTests(); - # Should not return - exit(1); - } - - # Binary name and profile arguments list - $main::prog = ""; - @main::pfile_args = (); - - # Remote profiling without a binary (using $SYMBOL_PAGE instead) - if (IsProfileURL($ARGV[0])) { - $main::use_symbol_page = 1; - } elsif (IsSymbolizedProfileFile($ARGV[0])) { - $main::use_symbolized_profile = 1; - $main::prog = $UNKNOWN_BINARY; # will be set later from the profile file - } - - if ($main::use_symbol_page || $main::use_symbolized_profile) { - # We don't need a binary! - my %disabled = ('--lines' => $main::opt_lines, - '--disasm' => $main::opt_disasm); - for my $option (keys %disabled) { - usage("$option cannot be used without a binary") if $disabled{$option}; - } - # Set $main::prog later... - scalar(@ARGV) || usage("Did not specify profile file"); - } elsif ($main::opt_symbols) { - # --symbols needs a binary-name (to run nm on, etc) but not profiles - $main::prog = shift(@ARGV) || usage("Did not specify program"); - } else { - $main::prog = shift(@ARGV) || usage("Did not specify program"); - scalar(@ARGV) || usage("Did not specify profile file"); - } - - # Parse profile file/location arguments - foreach my $farg (@ARGV) { - if ($farg =~ m/(.*)\@([0-9]+)(|\/.*)$/ ) { - my $machine = $1; - my $num_machines = $2; - my $path = $3; - for (my $i = 0; $i < $num_machines; $i++) { - unshift(@main::pfile_args, "$i.$machine$path"); - } - } else { - unshift(@main::pfile_args, $farg); - } - } - - if ($main::use_symbol_page) { - unless (IsProfileURL($main::pfile_args[0])) { - error("The first profile should be a remote form to use $SYMBOL_PAGE\n"); - } - CheckSymbolPage(); - $main::prog = FetchProgramName(); - } elsif (!$main::use_symbolized_profile) { # may not need objtools! - ConfigureObjTools($main::prog) - } - - # Break the opt_list_prefix into the prefix_list array - @prefix_list = split (',', $main::opt_lib_prefix); - - # Remove trailing / from the prefixes, in the list to prevent - # searching things like /my/path//lib/mylib.so - foreach (@prefix_list) { - s|/+$||; - } -} - -sub Main() { - Init(); - $main::collected_profile = undef; - @main::profile_files = (); - $main::op_time = time(); - - # Printing symbols is special and requires a lot less info that most. - if ($main::opt_symbols) { - PrintSymbols(*STDIN); # Get /proc/maps and symbols output from stdin - return; - } - - # Fetch all profile data - FetchDynamicProfiles(); - - # this will hold symbols that we read from the profile files - my $symbol_map = {}; - - # Read one profile, pick the last item on the list - my $data = ReadProfile($main::prog, pop(@main::profile_files)); - my $profile = $data->{profile}; - my $pcs = $data->{pcs}; - my $libs = $data->{libs}; # Info about main program and shared libraries - $symbol_map = MergeSymbols($symbol_map, $data->{symbols}); - - # Add additional profiles, if available. - if (scalar(@main::profile_files) > 0) { - foreach my $pname (@main::profile_files) { - my $data2 = ReadProfile($main::prog, $pname); - $profile = AddProfile($profile, $data2->{profile}); - $pcs = AddPcs($pcs, $data2->{pcs}); - $symbol_map = MergeSymbols($symbol_map, $data2->{symbols}); - } - } - - # Subtract base from profile, if specified - if ($main::opt_base ne '') { - my $base = ReadProfile($main::prog, $main::opt_base); - $profile = SubtractProfile($profile, $base->{profile}); - $pcs = AddPcs($pcs, $base->{pcs}); - $symbol_map = MergeSymbols($symbol_map, $base->{symbols}); - } - - # Get total data in profile - my $total = TotalProfile($profile); - - # Collect symbols - my $symbols; - if ($main::use_symbolized_profile) { - $symbols = FetchSymbols($pcs, $symbol_map); - } elsif ($main::use_symbol_page) { - $symbols = FetchSymbols($pcs); - } else { - # TODO(csilvers): $libs uses the /proc/self/maps data from profile1, - # which may differ from the data from subsequent profiles, especially - # if they were run on different machines. Use appropriate libs for - # each pc somehow. - $symbols = ExtractSymbols($libs, $pcs); - } - - # Remove uniniteresting stack items - $profile = RemoveUninterestingFrames($symbols, $profile); - - # Focus? - if ($main::opt_focus ne '') { - $profile = FocusProfile($symbols, $profile, $main::opt_focus); - } - - # Ignore? - if ($main::opt_ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $main::opt_ignore); - } - - my $calls = ExtractCalls($symbols, $profile); - - # Reduce profiles to required output granularity, and also clean - # each stack trace so a given entry exists at most once. - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - # Print - if (!$main::opt_interactive) { - if ($main::opt_disasm) { - PrintDisassembly($libs, $flat, $cumulative, $main::opt_disasm, $total); - } elsif ($main::opt_list) { - PrintListing($libs, $flat, $cumulative, $main::opt_list); - } elsif ($main::opt_text) { - # Make sure the output is empty when have nothing to report - # (only matters when --heapcheck is given but we must be - # compatible with old branches that did not pass --heapcheck always): - if ($total != 0) { - printf("Total: %s %s\n", Unparse($total), Units()); - } - PrintText($symbols, $flat, $cumulative, $total, -1); - } elsif ($main::opt_raw) { - PrintSymbolizedProfile($symbols, $profile, $main::prog); - } elsif ($main::opt_callgrind) { - PrintCallgrind($calls); - } else { - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), ""); - } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), ""); - } elsif ($main::opt_web) { - my $tmp = TempName($main::next_tmpfile, "svg"); - RunWeb($tmp); - # The command we run might hand the file name off - # to an already running browser instance and then exit. - # Normally, we'd remove $tmp on exit (right now), - # but fork a child to remove $tmp a little later, so that the - # browser has time to load it first. - delete $main::tempnames{$tmp}; - if (fork() == 0) { - sleep 5; - unlink($tmp); - exit(0); - } - } - } else { - cleanup(); - exit(1); - } - } - } else { - InteractiveMode($profile, $symbols, $libs, $total); - } - - cleanup(); - exit(0); -} - -##### Entry Point ##### - -Main(); - -# Temporary code to detect if we're running on a Goobuntu system. -# These systems don't have the right stuff installed for the special -# Readline libraries to work, so as a temporary workaround, we default -# to using the normal stdio code, rather than the fancier readline-based -# code -sub ReadlineMightFail { - if (-e '/lib/libtermcap.so.2') { - return 0; # libtermcap exists, so readline should be okay - } else { - return 1; - } -} - -sub RunGV { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - if (!system("$GV --version >/dev/null 2>&1")) { - # Options using double dash are supported by this gv version. - # Also, turn on noantialias to better handle bug in gv for - # postscript files with large dimensions. - # TODO: Maybe we should not pass the --noantialias flag - # if the gv version is known to work properly without the flag. - system("$GV --scale=$main::opt_scale --noantialias " . $fname . $bg); - } else { - # Old gv version - only supports options that use single dash. - print STDERR "$GV -scale $main::opt_scale\n"; - system("$GV -scale $main::opt_scale " . $fname . $bg); - } -} - -sub RunEvince { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - system("$EVINCE " . $fname . $bg); -} - -sub RunWeb { - my $fname = shift; - print STDERR "Loading web page file:///$fname\n"; - - if (`uname` =~ /Darwin/) { - # OS X: open will use standard preference for SVG files. - system("/usr/bin/open", $fname); - return; - } - - # Some kind of Unix; try generic symlinks, then specific browsers. - # (Stop once we find one.) - # Works best if the browser is already running. - my @alt = ( - "/etc/alternatives/gnome-www-browser", - "/etc/alternatives/x-www-browser", - "google-chrome", - "firefox", - ); - foreach my $b (@alt) { - if (system($b, $fname) == 0) { - return; - } - } - - print STDERR "Could not load web browser.\n"; -} - -sub RunKcachegrind { - my $fname = shift; - my $bg = shift; # "" or " &" if we should run in background - print STDERR "Starting '$KCACHEGRIND " . $fname . $bg . "'\n"; - system("$KCACHEGRIND " . $fname . $bg); -} - - -##### Interactive helper routines ##### - -sub InteractiveMode { - $| = 1; # Make output unbuffered for interactive mode - my ($orig_profile, $symbols, $libs, $total) = @_; - - print STDERR "Welcome to pprof! For help, type 'help'.\n"; - - # Use ReadLine if it's installed and input comes from a console. - if ( -t STDIN && - !ReadlineMightFail() && - defined(eval {require Term::ReadLine}) ) { - my $term = new Term::ReadLine 'pprof'; - while ( defined ($_ = $term->readline('(pprof) '))) { - $term->addhistory($_) if /\S/; - if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { - last; # exit when we get an interactive command to quit - } - } - } else { # don't have readline - while (1) { - print STDERR "(pprof) "; - $_ = ; - last if ! defined $_ ; - s/\r//g; # turn windows-looking lines into unix-looking lines - - # Save some flags that might be reset by InteractiveCommand() - my $save_opt_lines = $main::opt_lines; - - if (!InteractiveCommand($orig_profile, $symbols, $libs, $total, $_)) { - last; # exit when we get an interactive command to quit - } - - # Restore flags - $main::opt_lines = $save_opt_lines; - } - } -} - -# Takes two args: orig profile, and command to run. -# Returns 1 if we should keep going, or 0 if we were asked to quit -sub InteractiveCommand { - my($orig_profile, $symbols, $libs, $total, $command) = @_; - $_ = $command; # just to make future m//'s easier - if (!defined($_)) { - print STDERR "\n"; - return 0; - } - if (m/^\s*quit/) { - return 0; - } - if (m/^\s*help/) { - InteractiveHelpMessage(); - return 1; - } - # Clear all the mode options -- mode is controlled by "$command" - $main::opt_text = 0; - $main::opt_callgrind = 0; - $main::opt_disasm = 0; - $main::opt_list = 0; - $main::opt_gv = 0; - $main::opt_evince = 0; - $main::opt_cum = 0; - - if (m/^\s*(text|top)(\d*)\s*(.*)/) { - $main::opt_text = 1; - - my $line_limit = ($2 ne "") ? int($2) : 10; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($3); - - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintText($symbols, $flat, $cumulative, $total, $line_limit); - return 1; - } - if (m/^\s*callgrind\s*([^ \n]*)/) { - $main::opt_callgrind = 1; - - # Get derived profiles - my $calls = ExtractCalls($symbols, $orig_profile); - my $filename = $1; - if ( $1 eq '' ) { - $filename = TempName($main::next_tmpfile, "callgrind"); - } - PrintCallgrind($calls, $filename); - if ( $1 eq '' ) { - RunKcachegrind($filename, " & "); - $main::next_tmpfile++; - } - - return 1; - } - if (m/^\s*list\s*(.+)/) { - $main::opt_list = 1; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); - - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintListing($libs, $flat, $cumulative, $routine); - return 1; - } - if (m/^\s*disasm\s*(.+)/) { - $main::opt_disasm = 1; - - my $routine; - my $ignore; - ($routine, $ignore) = ParseInteractiveArgs($1); - - # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, "", $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - PrintDisassembly($libs, $flat, $cumulative, $routine, $total); - return 1; - } - if (m/^\s*(gv|web|evince)\s*(.*)/) { - $main::opt_gv = 0; - $main::opt_evince = 0; - $main::opt_web = 0; - if ($1 eq "gv") { - $main::opt_gv = 1; - } elsif ($1 eq "evince") { - $main::opt_evince = 1; - } elsif ($1 eq "web") { - $main::opt_web = 1; - } - - my $focus; - my $ignore; - ($focus, $ignore) = ParseInteractiveArgs($2); - - # Process current profile to account for various settings - my $profile = ProcessProfile($orig_profile, $symbols, $focus, $ignore); - my $reduced = ReduceProfile($symbols, $profile); - - # Get derived profiles - my $flat = FlatProfile($reduced); - my $cumulative = CumulativeProfile($reduced); - - if (PrintDot($main::prog, $symbols, $profile, $flat, $cumulative, $total)) { - if ($main::opt_gv) { - RunGV(TempName($main::next_tmpfile, "ps"), " &"); - } elsif ($main::opt_evince) { - RunEvince(TempName($main::next_tmpfile, "pdf"), " &"); - } elsif ($main::opt_web) { - RunWeb(TempName($main::next_tmpfile, "svg")); - } - $main::next_tmpfile++; - } - return 1; - } - if (m/^\s*$/) { - return 1; - } - print STDERR "Unknown command: try 'help'.\n"; - return 1; -} - - -sub ProcessProfile { - my $orig_profile = shift; - my $symbols = shift; - my $focus = shift; - my $ignore = shift; - - # Process current profile to account for various settings - my $profile = $orig_profile; - my $total_count = TotalProfile($profile); - printf("Total: %s %s\n", Unparse($total_count), Units()); - if ($focus ne '') { - $profile = FocusProfile($symbols, $profile, $focus); - my $focus_count = TotalProfile($profile); - printf("After focusing on '%s': %s %s of %s (%0.1f%%)\n", - $focus, - Unparse($focus_count), Units(), - Unparse($total_count), ($focus_count*100.0) / $total_count); - } - if ($ignore ne '') { - $profile = IgnoreProfile($symbols, $profile, $ignore); - my $ignore_count = TotalProfile($profile); - printf("After ignoring '%s': %s %s of %s (%0.1f%%)\n", - $ignore, - Unparse($ignore_count), Units(), - Unparse($total_count), - ($ignore_count*100.0) / $total_count); - } - - return $profile; -} - -sub InteractiveHelpMessage { - print STDERR <{$k}; - my @addrs = split(/\n/, $k); - if ($#addrs >= 0) { - my $depth = $#addrs + 1; - # int(foo / 2**32) is the only reliable way to get rid of bottom - # 32 bits on both 32- and 64-bit systems. - print pack('L*', $count & 0xFFFFFFFF, int($count / 2**32)); - print pack('L*', $depth & 0xFFFFFFFF, int($depth / 2**32)); - - foreach my $full_addr (@addrs) { - my $addr = $full_addr; - $addr =~ s/0x0*//; # strip off leading 0x, zeroes - if (length($addr) > 16) { - print STDERR "Invalid address in profile: $full_addr\n"; - next; - } - my $low_addr = substr($addr, -8); # get last 8 hex chars - my $high_addr = substr($addr, -16, 8); # get up to 8 more hex chars - print pack('L*', hex('0x' . $low_addr), hex('0x' . $high_addr)); - } - } - } -} - -# Print symbols and profile data -sub PrintSymbolizedProfile { - my $symbols = shift; - my $profile = shift; - my $prog = shift; - - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - - print '--- ', $symbol_marker, "\n"; - if (defined($prog)) { - print 'binary=', $prog, "\n"; - } - while (my ($pc, $name) = each(%{$symbols})) { - my $sep = ' '; - print '0x', $pc; - # We have a list of function names, which include the inlined - # calls. They are separated (and terminated) by --, which is - # illegal in function names. - for (my $j = 2; $j <= $#{$name}; $j += 3) { - print $sep, $name->[$j]; - $sep = '--'; - } - print "\n"; - } - print '---', "\n"; - - $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $profile_marker = $&; - print '--- ', $profile_marker, "\n"; - if (defined($main::collected_profile)) { - # if used with remote fetch, simply dump the collected profile to output. - open(SRC, "<$main::collected_profile"); - while () { - print $_; - } - close(SRC); - } else { - # dump a cpu-format profile to standard out - PrintProfileData($profile); - } -} - -# Print text output -sub PrintText { - my $symbols = shift; - my $flat = shift; - my $cumulative = shift; - my $total = shift; - my $line_limit = shift; - - # Which profile to sort by? - my $s = $main::opt_cum ? $cumulative : $flat; - - my $running_sum = 0; - my $lines = 0; - foreach my $k (sort { GetEntry($s, $b) <=> GetEntry($s, $a) || $a cmp $b } - keys(%{$cumulative})) { - my $f = GetEntry($flat, $k); - my $c = GetEntry($cumulative, $k); - $running_sum += $f; - - my $sym = $k; - if (exists($symbols->{$k})) { - $sym = $symbols->{$k}->[0] . " " . $symbols->{$k}->[1]; - if ($main::opt_addresses) { - $sym = $k . " " . $sym; - } - } - - if ($f != 0 || $c != 0) { - printf("%8s %6s %6s %8s %6s %s\n", - Unparse($f), - Percent($f, $total), - Percent($running_sum, $total), - Unparse($c), - Percent($c, $total), - $sym); - } - $lines++; - last if ($line_limit >= 0 && $lines > $line_limit); - } -} - -# Print the call graph in a way that's suiteable for callgrind. -sub PrintCallgrind { - my $calls = shift; - my $filename; - if ($main::opt_interactive) { - $filename = shift; - print STDERR "Writing callgrind file to '$filename'.\n" - } else { - $filename = "&STDOUT"; - } - open(CG, ">".$filename ); - printf CG ("events: Hits\n\n"); - foreach my $call ( map { $_->[0] } - sort { $a->[1] cmp $b ->[1] || - $a->[2] <=> $b->[2] } - map { /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; - [$_, $1, $2] } - keys %$calls ) { - my $count = int($calls->{$call}); - $call =~ /([^:]+):(\d+):([^ ]+)( -> ([^:]+):(\d+):(.+))?/; - my ( $caller_file, $caller_line, $caller_function, - $callee_file, $callee_line, $callee_function ) = - ( $1, $2, $3, $5, $6, $7 ); - - - printf CG ("fl=$caller_file\nfn=$caller_function\n"); - if (defined $6) { - printf CG ("cfl=$callee_file\n"); - printf CG ("cfn=$callee_function\n"); - printf CG ("calls=$count $callee_line\n"); - } - printf CG ("$caller_line $count\n\n"); - } -} - -# Print disassembly for all all routines that match $main::opt_disasm -sub PrintDisassembly { - my $libs = shift; - my $flat = shift; - my $cumulative = shift; - my $disasm_opts = shift; - my $total = shift; - - foreach my $lib (@{$libs}) { - my $symbol_table = GetProcedureBoundaries($lib->[0], $disasm_opts); - my $offset = AddressSub($lib->[1], $lib->[3]); - foreach my $routine (sort ByName keys(%{$symbol_table})) { - my $start_addr = $symbol_table->{$routine}->[0]; - my $end_addr = $symbol_table->{$routine}->[1]; - # See if there are any samples in this routine - my $length = hex(AddressSub($end_addr, $start_addr)); - my $addr = AddressAdd($start_addr, $offset); - for (my $i = 0; $i < $length; $i++) { - if (defined($cumulative->{$addr})) { - PrintDisassembledFunction($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr, $total); - last; - } - $addr = AddressInc($addr); - } - } - } -} - -# Return reference to array of tuples of the form: -# [start_address, filename, linenumber, instruction, limit_address] -# E.g., -# ["0x806c43d", "/foo/bar.cc", 131, "ret", "0x806c440"] -sub Disassemble { - my $prog = shift; - my $offset = shift; - my $start_addr = shift; - my $end_addr = shift; - - my $objdump = $obj_tool_map{"objdump"}; - my $cmd = sprintf("$objdump -C -d -l --no-show-raw-insn " . - "--start-address=0x$start_addr " . - "--stop-address=0x$end_addr $prog"); - open(OBJDUMP, "$cmd |") || error("$objdump: $!\n"); - my @result = (); - my $filename = ""; - my $linenumber = -1; - my $last = ["", "", "", ""]; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - chop; - if (m|\s*([^:\s]+):(\d+)\s*$|) { - # Location line of the form: - # : - $filename = $1; - $linenumber = $2; - } elsif (m/^ +([0-9a-f]+):\s*(.*)/) { - # Disassembly line -- zero-extend address to full length - my $addr = HexExtend($1); - my $k = AddressAdd($addr, $offset); - $last->[4] = $k; # Store ending address for previous instruction - $last = [$k, $filename, $linenumber, $2, $end_addr]; - push(@result, $last); - } - } - close(OBJDUMP); - return @result; -} - -# The input file should contain lines of the form /proc/maps-like -# output (same format as expected from the profiles) or that looks -# like hex addresses (like "0xDEADBEEF"). We will parse all -# /proc/maps output, and for all the hex addresses, we will output -# "short" symbol names, one per line, in the same order as the input. -sub PrintSymbols { - my $maps_and_symbols_file = shift; - - # ParseLibraries expects pcs to be in a set. Fine by us... - my @pclist = (); # pcs in sorted order - my $pcs = {}; - my $map = ""; - foreach my $line (<$maps_and_symbols_file>) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ($line =~ /\b(0x[0-9a-f]+)\b/i) { - push(@pclist, HexExtend($1)); - $pcs->{$pclist[-1]} = 1; - } else { - $map .= $line; - } - } - - my $libs = ParseLibraries($main::prog, $map, $pcs); - my $symbols = ExtractSymbols($libs, $pcs); - - foreach my $pc (@pclist) { - # ->[0] is the shortname, ->[2] is the full name - print(($symbols->{$pc}->[0] || "??") . "\n"); - } -} - - -# For sorting functions by name -sub ByName { - return ShortFunctionName($a) cmp ShortFunctionName($b); -} - -# Print source-listing for all all routines that match $main::opt_list -sub PrintListing { - my $libs = shift; - my $flat = shift; - my $cumulative = shift; - my $list_opts = shift; - - foreach my $lib (@{$libs}) { - my $symbol_table = GetProcedureBoundaries($lib->[0], $list_opts); - my $offset = AddressSub($lib->[1], $lib->[3]); - foreach my $routine (sort ByName keys(%{$symbol_table})) { - # Print if there are any samples in this routine - my $start_addr = $symbol_table->{$routine}->[0]; - my $end_addr = $symbol_table->{$routine}->[1]; - my $length = hex(AddressSub($end_addr, $start_addr)); - my $addr = AddressAdd($start_addr, $offset); - for (my $i = 0; $i < $length; $i++) { - if (defined($cumulative->{$addr})) { - PrintSource($lib->[0], $offset, - $routine, $flat, $cumulative, - $start_addr, $end_addr); - last; - } - $addr = AddressInc($addr); - } - } - } -} - -# Returns the indentation of the line, if it has any non-whitespace -# characters. Otherwise, returns -1. -sub Indentation { - my $line = shift; - if (m/^(\s*)\S/) { - return length($1); - } else { - return -1; - } -} - -# Print source-listing for one routine -sub PrintSource { - my $prog = shift; - my $offset = shift; - my $routine = shift; - my $flat = shift; - my $cumulative = shift; - my $start_addr = shift; - my $end_addr = shift; - - # Disassemble all instructions (just to get line numbers) - my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); - - # Hack 1: assume that the first source file encountered in the - # disassembly contains the routine - my $filename = undef; - for (my $i = 0; $i <= $#instructions; $i++) { - if ($instructions[$i]->[2] >= 0) { - $filename = $instructions[$i]->[1]; - last; - } - } - if (!defined($filename)) { - print STDERR "no filename found in $routine\n"; - return; - } - - # Hack 2: assume that the largest line number from $filename is the - # end of the procedure. This is typically safe since if P1 contains - # an inlined call to P2, then P2 usually occurs earlier in the - # source file. If this does not work, we might have to compute a - # density profile or just print all regions we find. - my $lastline = 0; - for (my $i = 0; $i <= $#instructions; $i++) { - my $f = $instructions[$i]->[1]; - my $l = $instructions[$i]->[2]; - if (($f eq $filename) && ($l > $lastline)) { - $lastline = $l; - } - } - - # Hack 3: assume the first source location from "filename" is the start of - # the source code. - my $firstline = 1; - for (my $i = 0; $i <= $#instructions; $i++) { - if ($instructions[$i]->[1] eq $filename) { - $firstline = $instructions[$i]->[2]; - last; - } - } - - # Hack 4: Extend last line forward until its indentation is less than - # the indentation we saw on $firstline - my $oldlastline = $lastline; - { - if (!open(FILE, "<$filename")) { - print STDERR "$filename: $!\n"; - return; - } - my $l = 0; - my $first_indentation = -1; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $l++; - my $indent = Indentation($_); - if ($l >= $firstline) { - if ($first_indentation < 0 && $indent >= 0) { - $first_indentation = $indent; - last if ($first_indentation == 0); - } - } - if ($l >= $lastline && $indent >= 0) { - if ($indent >= $first_indentation) { - $lastline = $l+1; - } else { - last; - } - } - } - close(FILE); - } - - # Assign all samples to the range $firstline,$lastline, - # Hack 4: If an instruction does not occur in the range, its samples - # are moved to the next instruction that occurs in the range. - my $samples1 = {}; - my $samples2 = {}; - my $running1 = 0; # Unassigned flat counts - my $running2 = 0; # Unassigned cumulative counts - my $total1 = 0; # Total flat counts - my $total2 = 0; # Total cumulative counts - foreach my $e (@instructions) { - # Add up counts for all address that fall inside this instruction - my $c1 = 0; - my $c2 = 0; - for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { - $c1 += GetEntry($flat, $a); - $c2 += GetEntry($cumulative, $a); - } - $running1 += $c1; - $running2 += $c2; - $total1 += $c1; - $total2 += $c2; - my $file = $e->[1]; - my $line = $e->[2]; - if (($file eq $filename) && - ($line >= $firstline) && - ($line <= $lastline)) { - # Assign all accumulated samples to this line - AddEntry($samples1, $line, $running1); - AddEntry($samples2, $line, $running2); - $running1 = 0; - $running2 = 0; - } - } - - # Assign any leftover samples to $lastline - AddEntry($samples1, $lastline, $running1); - AddEntry($samples2, $lastline, $running2); - - printf("ROUTINE ====================== %s in %s\n" . - "%6s %6s Total %s (flat / cumulative)\n", - ShortFunctionName($routine), - $filename, - Units(), - Unparse($total1), - Unparse($total2)); - if (!open(FILE, "<$filename")) { - print STDERR "$filename: $!\n"; - return; - } - my $l = 0; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $l++; - if ($l >= $firstline - 5 && - (($l <= $oldlastline + 5) || ($l <= $lastline))) { - chop; - my $text = $_; - if ($l == $firstline) { printf("---\n"); } - printf("%6s %6s %4d: %s\n", - UnparseAlt(GetEntry($samples1, $l)), - UnparseAlt(GetEntry($samples2, $l)), - $l, - $text); - if ($l == $lastline) { printf("---\n"); } - }; - } - close(FILE); -} - -# Return the source line for the specified file/linenumber. -# Returns undef if not found. -sub SourceLine { - my $file = shift; - my $line = shift; - - # Look in cache - if (!defined($main::source_cache{$file})) { - if (100 < scalar keys(%main::source_cache)) { - # Clear the cache when it gets too big - $main::source_cache = (); - } - - # Read all lines from the file - if (!open(FILE, "<$file")) { - print STDERR "$file: $!\n"; - $main::source_cache{$file} = []; # Cache the negative result - return undef; - } - my $lines = []; - push(@{$lines}, ""); # So we can use 1-based line numbers as indices - while () { - push(@{$lines}, $_); - } - close(FILE); - - # Save the lines in the cache - $main::source_cache{$file} = $lines; - } - - my $lines = $main::source_cache{$file}; - if (($line < 0) || ($line > $#{$lines})) { - return undef; - } else { - return $lines->[$line]; - } -} - -# Print disassembly for one routine with interspersed source if available -sub PrintDisassembledFunction { - my $prog = shift; - my $offset = shift; - my $routine = shift; - my $flat = shift; - my $cumulative = shift; - my $start_addr = shift; - my $end_addr = shift; - my $total = shift; - - # Disassemble all instructions - my @instructions = Disassemble($prog, $offset, $start_addr, $end_addr); - - # Make array of counts per instruction - my @flat_count = (); - my @cum_count = (); - my $flat_total = 0; - my $cum_total = 0; - foreach my $e (@instructions) { - # Add up counts for all address that fall inside this instruction - my $c1 = 0; - my $c2 = 0; - for (my $a = $e->[0]; $a lt $e->[4]; $a = AddressInc($a)) { - $c1 += GetEntry($flat, $a); - $c2 += GetEntry($cumulative, $a); - } - push(@flat_count, $c1); - push(@cum_count, $c2); - $flat_total += $c1; - $cum_total += $c2; - } - - # Print header with total counts - printf("ROUTINE ====================== %s\n" . - "%6s %6s %s (flat, cumulative) %.1f%% of total\n", - ShortFunctionName($routine), - Unparse($flat_total), - Unparse($cum_total), - Units(), - ($cum_total * 100.0) / $total); - - # Process instructions in order - my $current_file = ""; - for (my $i = 0; $i <= $#instructions; ) { - my $e = $instructions[$i]; - - # Print the new file name whenever we switch files - if ($e->[1] ne $current_file) { - $current_file = $e->[1]; - my $fname = $current_file; - $fname =~ s|^\./||; # Trim leading "./" - - # Shorten long file names - if (length($fname) >= 58) { - $fname = "..." . substr($fname, -55); - } - printf("-------------------- %s\n", $fname); - } - - # TODO: Compute range of lines to print together to deal with - # small reorderings. - my $first_line = $e->[2]; - my $last_line = $first_line; - my %flat_sum = (); - my %cum_sum = (); - for (my $l = $first_line; $l <= $last_line; $l++) { - $flat_sum{$l} = 0; - $cum_sum{$l} = 0; - } - - # Find run of instructions for this range of source lines - my $first_inst = $i; - while (($i <= $#instructions) && - ($instructions[$i]->[2] >= $first_line) && - ($instructions[$i]->[2] <= $last_line)) { - $e = $instructions[$i]; - $flat_sum{$e->[2]} += $flat_count[$i]; - $cum_sum{$e->[2]} += $cum_count[$i]; - $i++; - } - my $last_inst = $i - 1; - - # Print source lines - for (my $l = $first_line; $l <= $last_line; $l++) { - my $line = SourceLine($current_file, $l); - if (!defined($line)) { - $line = "?\n"; - next; - } else { - $line =~ s/^\s+//; - } - printf("%6s %6s %5d: %s", - UnparseAlt($flat_sum{$l}), - UnparseAlt($cum_sum{$l}), - $l, - $line); - } - - # Print disassembly - for (my $x = $first_inst; $x <= $last_inst; $x++) { - my $e = $instructions[$x]; - my $address = $e->[0]; - $address = AddressSub($address, $offset); # Make relative to section - $address =~ s/^0x//; - $address =~ s/^0*//; - - # Trim symbols - my $d = $e->[3]; - while ($d =~ s/\([^()%]*\)(\s*const)?//g) { } # Argument types, not (%rax) - while ($d =~ s/(\w+)<[^<>]*>/$1/g) { } # Remove template arguments - - printf("%6s %6s %8s: %6s\n", - UnparseAlt($flat_count[$x]), - UnparseAlt($cum_count[$x]), - $address, - $d); - } - } -} - -# Print DOT graph -sub PrintDot { - my $prog = shift; - my $symbols = shift; - my $raw = shift; - my $flat = shift; - my $cumulative = shift; - my $overall_total = shift; - - # Get total - my $local_total = TotalProfile($flat); - my $nodelimit = int($main::opt_nodefraction * $local_total); - my $edgelimit = int($main::opt_edgefraction * $local_total); - my $nodecount = $main::opt_nodecount; - - # Find nodes to include - my @list = (sort { abs(GetEntry($cumulative, $b)) <=> - abs(GetEntry($cumulative, $a)) - || $a cmp $b } - keys(%{$cumulative})); - my $last = $nodecount - 1; - if ($last > $#list) { - $last = $#list; - } - while (($last >= 0) && - (abs(GetEntry($cumulative, $list[$last])) <= $nodelimit)) { - $last--; - } - if ($last < 0) { - print STDERR "No nodes to print\n"; - return 0; - } - - if ($nodelimit > 0 || $edgelimit > 0) { - printf STDERR ("Dropping nodes with <= %s %s; edges with <= %s abs(%s)\n", - Unparse($nodelimit), Units(), - Unparse($edgelimit), Units()); - } - - # Open DOT output file - my $output; - if ($main::opt_gv) { - $output = "| $DOT -Tps2 >" . TempName($main::next_tmpfile, "ps"); - } elsif ($main::opt_evince) { - $output = "| $DOT -Tps2 | $PS2PDF - " . TempName($main::next_tmpfile, "pdf"); - } elsif ($main::opt_ps) { - $output = "| $DOT -Tps2"; - } elsif ($main::opt_pdf) { - $output = "| $DOT -Tps2 | $PS2PDF - -"; - } elsif ($main::opt_web || $main::opt_svg) { - # We need to post-process the SVG, so write to a temporary file always. - $output = "| $DOT -Tsvg >" . TempName($main::next_tmpfile, "svg"); - } elsif ($main::opt_gif) { - $output = "| $DOT -Tgif"; - } else { - $output = ">&STDOUT"; - } - open(DOT, $output) || error("$output: $!\n"); - - # Title - printf DOT ("digraph \"%s; %s %s\" {\n", - $prog, - Unparse($overall_total), - Units()); - if ($main::opt_pdf) { - # The output is more printable if we set the page size for dot. - printf DOT ("size=\"8,11\"\n"); - } - printf DOT ("node [width=0.375,height=0.25];\n"); - - # Print legend - printf DOT ("Legend [shape=box,fontsize=24,shape=plaintext," . - "label=\"%s\\l%s\\l%s\\l%s\\l%s\\l\"];\n", - $prog, - sprintf("Total %s: %s", Units(), Unparse($overall_total)), - sprintf("Focusing on: %s", Unparse($local_total)), - sprintf("Dropped nodes with <= %s abs(%s)", - Unparse($nodelimit), Units()), - sprintf("Dropped edges with <= %s %s", - Unparse($edgelimit), Units()) - ); - - # Print nodes - my %node = (); - my $nextnode = 1; - foreach my $a (@list[0..$last]) { - # Pick font size - my $f = GetEntry($flat, $a); - my $c = GetEntry($cumulative, $a); - - my $fs = 8; - if ($local_total > 0) { - $fs = 8 + (50.0 * sqrt(abs($f * 1.0 / $local_total))); - } - - $node{$a} = $nextnode++; - my $sym = $a; - $sym =~ s/\s+/\\n/g; - $sym =~ s/::/\\n/g; - - # Extra cumulative info to print for non-leaves - my $extra = ""; - if ($f != $c) { - $extra = sprintf("\\rof %s (%s)", - Unparse($c), - Percent($c, $overall_total)); - } - my $style = ""; - if ($main::opt_heapcheck) { - if ($f > 0) { - # make leak-causing nodes more visible (add a background) - $style = ",style=filled,fillcolor=gray" - } elsif ($f < 0) { - # make anti-leak-causing nodes (which almost never occur) - # stand out as well (triple border) - $style = ",peripheries=3" - } - } - - printf DOT ("N%d [label=\"%s\\n%s (%s)%s\\r" . - "\",shape=box,fontsize=%.1f%s];\n", - $node{$a}, - $sym, - Unparse($f), - Percent($f, $overall_total), - $extra, - $fs, - $style, - ); - } - - # Get edges and counts per edge - my %edge = (); - my $n; - foreach my $k (keys(%{$raw})) { - # TODO: omit low %age edges - $n = $raw->{$k}; - my @translated = TranslateStack($symbols, $k); - for (my $i = 1; $i <= $#translated; $i++) { - my $src = $translated[$i]; - my $dst = $translated[$i-1]; - #next if ($src eq $dst); # Avoid self-edges? - if (exists($node{$src}) && exists($node{$dst})) { - my $edge_label = "$src\001$dst"; - if (!exists($edge{$edge_label})) { - $edge{$edge_label} = 0; - } - $edge{$edge_label} += $n; - } - } - } - - # Print edges (process in order of decreasing counts) - my %indegree = (); # Number of incoming edges added per node so far - my %outdegree = (); # Number of outgoing edges added per node so far - foreach my $e (sort { $edge{$b} <=> $edge{$a} } keys(%edge)) { - my @x = split(/\001/, $e); - $n = $edge{$e}; - - # Initialize degree of kept incoming and outgoing edges if necessary - my $src = $x[0]; - my $dst = $x[1]; - if (!exists($outdegree{$src})) { $outdegree{$src} = 0; } - if (!exists($indegree{$dst})) { $indegree{$dst} = 0; } - - my $keep; - if ($indegree{$dst} == 0) { - # Keep edge if needed for reachability - $keep = 1; - } elsif (abs($n) <= $edgelimit) { - # Drop if we are below --edgefraction - $keep = 0; - } elsif ($outdegree{$src} >= $main::opt_maxdegree || - $indegree{$dst} >= $main::opt_maxdegree) { - # Keep limited number of in/out edges per node - $keep = 0; - } else { - $keep = 1; - } - - if ($keep) { - $outdegree{$src}++; - $indegree{$dst}++; - - # Compute line width based on edge count - my $fraction = abs($local_total ? (3 * ($n / $local_total)) : 0); - if ($fraction > 1) { $fraction = 1; } - my $w = $fraction * 2; - if ($w < 1 && ($main::opt_web || $main::opt_svg)) { - # SVG output treats line widths < 1 poorly. - $w = 1; - } - - # Dot sometimes segfaults if given edge weights that are too large, so - # we cap the weights at a large value - my $edgeweight = abs($n) ** 0.7; - if ($edgeweight > 100000) { $edgeweight = 100000; } - $edgeweight = int($edgeweight); - - my $style = sprintf("setlinewidth(%f)", $w); - if ($x[1] =~ m/\(inline\)/) { - $style .= ",dashed"; - } - - # Use a slightly squashed function of the edge count as the weight - printf DOT ("N%s -> N%s [label=%s, weight=%d, style=\"%s\"];\n", - $node{$x[0]}, - $node{$x[1]}, - Unparse($n), - $edgeweight, - $style); - } - } - - print DOT ("}\n"); - close(DOT); - - if ($main::opt_web || $main::opt_svg) { - # Rewrite SVG to be more usable inside web browser. - RewriteSvg(TempName($main::next_tmpfile, "svg")); - } - - return 1; -} - -sub RewriteSvg { - my $svgfile = shift; - - open(SVG, $svgfile) || die "open temp svg: $!"; - my @svg = ; - close(SVG); - unlink $svgfile; - my $svg = join('', @svg); - - # Dot's SVG output is - # - # - # - # ... - # - # - # - # Change it to - # - # - # $svg_javascript - # - # - # ... - # - # - # - - # Fix width, height; drop viewBox. - $svg =~ s/(?s) above first - my $svg_javascript = SvgJavascript(); - my $viewport = "\n"; - $svg =~ s/ above . - $svg =~ s/(.*)(<\/svg>)/$1<\/g>$2/; - $svg =~ s/$svgfile") || die "open $svgfile: $!"; - print SVG $svg; - close(SVG); - } -} - -sub SvgJavascript { - return <<'EOF'; - -EOF -} - -# Return a small number that identifies the argument. -# Multiple calls with the same argument will return the same number. -# Calls with different arguments will return different numbers. -sub ShortIdFor { - my $key = shift; - my $id = $main::uniqueid{$key}; - if (!defined($id)) { - $id = keys(%main::uniqueid) + 1; - $main::uniqueid{$key} = $id; - } - return $id; -} - -# Translate a stack of addresses into a stack of symbols -sub TranslateStack { - my $symbols = shift; - my $k = shift; - - my @addrs = split(/\n/, $k); - my @result = (); - for (my $i = 0; $i <= $#addrs; $i++) { - my $a = $addrs[$i]; - - # Skip large addresses since they sometimes show up as fake entries on RH9 - if (length($a) > 8 && $a gt "7fffffffffffffff") { - next; - } - - if ($main::opt_disasm || $main::opt_list) { - # We want just the address for the key - push(@result, $a); - next; - } - - my $symlist = $symbols->{$a}; - if (!defined($symlist)) { - $symlist = [$a, "", $a]; - } - - # We can have a sequence of symbols for a particular entry - # (more than one symbol in the case of inlining). Callers - # come before callees in symlist, so walk backwards since - # the translated stack should contain callees before callers. - for (my $j = $#{$symlist}; $j >= 2; $j -= 3) { - my $func = $symlist->[$j-2]; - my $fileline = $symlist->[$j-1]; - my $fullfunc = $symlist->[$j]; - if ($j > 2) { - $func = "$func (inline)"; - } - - # Do not merge nodes corresponding to Callback::Run since that - # causes confusing cycles in dot display. Instead, we synthesize - # a unique name for this frame per caller. - if ($func =~ m/Callback.*::Run$/) { - my $caller = ($i > 0) ? $addrs[$i-1] : 0; - $func = "Run#" . ShortIdFor($caller); - } - - if ($main::opt_addresses) { - push(@result, "$a $func $fileline"); - } elsif ($main::opt_lines) { - if ($func eq '??' && $fileline eq '??:0') { - push(@result, "$a"); - } else { - push(@result, "$func $fileline"); - } - } elsif ($main::opt_functions) { - if ($func eq '??') { - push(@result, "$a"); - } else { - push(@result, $func); - } - } elsif ($main::opt_files) { - if ($fileline eq '??:0' || $fileline eq '') { - push(@result, "$a"); - } else { - my $f = $fileline; - $f =~ s/:\d+$//; - push(@result, $f); - } - } else { - push(@result, $a); - last; # Do not print inlined info - } - } - } - - # print join(",", @addrs), " => ", join(",", @result), "\n"; - return @result; -} - -# Generate percent string for a number and a total -sub Percent { - my $num = shift; - my $tot = shift; - if ($tot != 0) { - return sprintf("%.1f%%", $num * 100.0 / $tot); - } else { - return ($num == 0) ? "nan" : (($num > 0) ? "+inf" : "-inf"); - } -} - -# Generate pretty-printed form of number -sub Unparse { - my $num = shift; - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - if ($main::opt_inuse_objects || $main::opt_alloc_objects) { - return sprintf("%d", $num); - } else { - if ($main::opt_show_bytes) { - return sprintf("%d", $num); - } else { - return sprintf("%.1f", $num / 1048576.0); - } - } - } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { - return sprintf("%.3f", $num / 1e9); # Convert nanoseconds to seconds - } else { - return sprintf("%d", $num); - } -} - -# Alternate pretty-printed form: 0 maps to "." -sub UnparseAlt { - my $num = shift; - if ($num == 0) { - return "."; - } else { - return Unparse($num); - } -} - -# Return output units -sub Units { - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - if ($main::opt_inuse_objects || $main::opt_alloc_objects) { - return "objects"; - } else { - if ($main::opt_show_bytes) { - return "B"; - } else { - return "MB"; - } - } - } elsif ($main::profile_type eq 'contention' && !$main::opt_contentions) { - return "seconds"; - } else { - return "samples"; - } -} - -##### Profile manipulation code ##### - -# Generate flattened profile: -# If count is charged to stack [a,b,c,d], in generated profile, -# it will be charged to [a] -sub FlatProfile { - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - if ($#addrs >= 0) { - AddEntry($result, $addrs[0], $count); - } - } - return $result; -} - -# Generate cumulative profile: -# If count is charged to stack [a,b,c,d], in generated profile, -# it will be charged to [a], [b], [c], [d] -sub CumulativeProfile { - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - foreach my $a (@addrs) { - AddEntry($result, $a, $count); - } - } - return $result; -} - -# If the second-youngest PC on the stack is always the same, returns -# that pc. Otherwise, returns undef. -sub IsSecondPcAlwaysTheSame { - my $profile = shift; - - my $second_pc = undef; - foreach my $k (keys(%{$profile})) { - my @addrs = split(/\n/, $k); - if ($#addrs < 1) { - return undef; - } - if (not defined $second_pc) { - $second_pc = $addrs[1]; - } else { - if ($second_pc ne $addrs[1]) { - return undef; - } - } - } - return $second_pc; -} - -sub ExtractSymbolLocation { - my $symbols = shift; - my $address = shift; - # 'addr2line' outputs "??:0" for unknown locations; we do the - # same to be consistent. - my $location = "??:0:unknown"; - if (exists $symbols->{$address}) { - my $file = $symbols->{$address}->[1]; - if ($file eq "?") { - $file = "??:0" - } - $location = $file . ":" . $symbols->{$address}->[0]; - } - return $location; -} - -# Extracts a graph of calls. -sub ExtractCalls { - my $symbols = shift; - my $profile = shift; - - my $calls = {}; - while( my ($stack_trace, $count) = each %$profile ) { - my @address = split(/\n/, $stack_trace); - my $destination = ExtractSymbolLocation($symbols, $address[0]); - AddEntry($calls, $destination, $count); - for (my $i = 1; $i <= $#address; $i++) { - my $source = ExtractSymbolLocation($symbols, $address[$i]); - my $call = "$source -> $destination"; - AddEntry($calls, $call, $count); - $destination = $source; - } - } - - return $calls; -} - -sub RemoveUninterestingFrames { - my $symbols = shift; - my $profile = shift; - - # List of function names to skip - my %skip = (); - my $skip_regexp = 'NOMATCH'; - if ($main::profile_type eq 'heap' || $main::profile_type eq 'growth') { - foreach my $name ('calloc', - 'cfree', - 'malloc', - 'free', - 'memalign', - 'posix_memalign', - 'pvalloc', - 'valloc', - 'realloc', - 'tc_calloc', - 'tc_cfree', - 'tc_malloc', - 'tc_free', - 'tc_memalign', - 'tc_posix_memalign', - 'tc_pvalloc', - 'tc_valloc', - 'tc_realloc', - 'tc_new', - 'tc_delete', - 'tc_newarray', - 'tc_deletearray', - 'tc_new_nothrow', - 'tc_newarray_nothrow', - 'do_malloc', - '::do_malloc', # new name -- got moved to an unnamed ns - '::do_malloc_or_cpp_alloc', - 'DoSampledAllocation', - 'simple_alloc::allocate', - '__malloc_alloc_template::allocate', - '__builtin_delete', - '__builtin_new', - '__builtin_vec_delete', - '__builtin_vec_new', - 'operator new', - 'operator new[]', - # These mark the beginning/end of our custom sections - '__start_google_malloc', - '__stop_google_malloc', - '__start_malloc_hook', - '__stop_malloc_hook') { - $skip{$name} = 1; - $skip{"_" . $name} = 1; # Mach (OS X) adds a _ prefix to everything - } - # TODO: Remove TCMalloc once everything has been - # moved into the tcmalloc:: namespace and we have flushed - # old code out of the system. - $skip_regexp = "TCMalloc|^tcmalloc::"; - } elsif ($main::profile_type eq 'contention') { - foreach my $vname ('base::RecordLockProfileData', - 'base::SubmitMutexProfileData', - 'base::SubmitSpinLockProfileData', - 'Mutex::Unlock', - 'Mutex::UnlockSlow', - 'Mutex::ReaderUnlock', - 'MutexLock::~MutexLock', - 'SpinLock::Unlock', - 'SpinLock::SlowUnlock', - 'SpinLockHolder::~SpinLockHolder') { - $skip{$vname} = 1; - } - } elsif ($main::profile_type eq 'cpu') { - # Drop signal handlers used for CPU profile collection - # TODO(dpeng): this should not be necessary; it's taken - # care of by the general 2nd-pc mechanism below. - foreach my $name ('ProfileData::Add', # historical - 'ProfileData::prof_handler', # historical - 'CpuProfiler::prof_handler', - '__FRAME_END__', - '__pthread_sighandler', - '__restore') { - $skip{$name} = 1; - } - } else { - # Nothing skipped for unknown types - } - - if ($main::profile_type eq 'cpu') { - # If all the second-youngest program counters are the same, - # this STRONGLY suggests that it is an artifact of measurement, - # i.e., stack frames pushed by the CPU profiler signal handler. - # Hence, we delete them. - # (The topmost PC is read from the signal structure, not from - # the stack, so it does not get involved.) - while (my $second_pc = IsSecondPcAlwaysTheSame($profile)) { - my $result = {}; - my $func = ''; - if (exists($symbols->{$second_pc})) { - $second_pc = $symbols->{$second_pc}->[0]; - } - print STDERR "Removing $second_pc from all stack traces.\n"; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - splice @addrs, 1, 1; - my $reduced_path = join("\n", @addrs); - AddEntry($result, $reduced_path, $count); - } - $profile = $result; - } - } - - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - my @path = (); - foreach my $a (@addrs) { - if (exists($symbols->{$a})) { - my $func = $symbols->{$a}->[0]; - if ($skip{$func} || ($func =~ m/$skip_regexp/)) { - next; - } - } - push(@path, $a); - } - my $reduced_path = join("\n", @path); - AddEntry($result, $reduced_path, $count); - } - return $result; -} - -# Reduce profile to granularity given by user -sub ReduceProfile { - my $symbols = shift; - my $profile = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @translated = TranslateStack($symbols, $k); - my @path = (); - my %seen = (); - $seen{''} = 1; # So that empty keys are skipped - foreach my $e (@translated) { - # To avoid double-counting due to recursion, skip a stack-trace - # entry if it has already been seen - if (!$seen{$e}) { - $seen{$e} = 1; - push(@path, $e); - } - } - my $reduced_path = join("\n", @path); - AddEntry($result, $reduced_path, $count); - } - return $result; -} - -# Does the specified symbol array match the regexp? -sub SymbolMatches { - my $sym = shift; - my $re = shift; - if (defined($sym)) { - for (my $i = 0; $i < $#{$sym}; $i += 3) { - if ($sym->[$i] =~ m/$re/ || $sym->[$i+1] =~ m/$re/) { - return 1; - } - } - } - return 0; -} - -# Focus only on paths involving specified regexps -sub FocusProfile { - my $symbols = shift; - my $profile = shift; - my $focus = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - foreach my $a (@addrs) { - # Reply if it matches either the address/shortname/fileline - if (($a =~ m/$focus/) || SymbolMatches($symbols->{$a}, $focus)) { - AddEntry($result, $k, $count); - last; - } - } - } - return $result; -} - -# Focus only on paths not involving specified regexps -sub IgnoreProfile { - my $symbols = shift; - my $profile = shift; - my $ignore = shift; - my $result = {}; - foreach my $k (keys(%{$profile})) { - my $count = $profile->{$k}; - my @addrs = split(/\n/, $k); - my $matched = 0; - foreach my $a (@addrs) { - # Reply if it matches either the address/shortname/fileline - if (($a =~ m/$ignore/) || SymbolMatches($symbols->{$a}, $ignore)) { - $matched = 1; - last; - } - } - if (!$matched) { - AddEntry($result, $k, $count); - } - } - return $result; -} - -# Get total count in profile -sub TotalProfile { - my $profile = shift; - my $result = 0; - foreach my $k (keys(%{$profile})) { - $result += $profile->{$k}; - } - return $result; -} - -# Add A to B -sub AddProfile { - my $A = shift; - my $B = shift; - - my $R = {}; - # add all keys in A - foreach my $k (keys(%{$A})) { - my $v = $A->{$k}; - AddEntry($R, $k, $v); - } - # add all keys in B - foreach my $k (keys(%{$B})) { - my $v = $B->{$k}; - AddEntry($R, $k, $v); - } - return $R; -} - -# Merges symbol maps -sub MergeSymbols { - my $A = shift; - my $B = shift; - - my $R = {}; - foreach my $k (keys(%{$A})) { - $R->{$k} = $A->{$k}; - } - if (defined($B)) { - foreach my $k (keys(%{$B})) { - $R->{$k} = $B->{$k}; - } - } - return $R; -} - - -# Add A to B -sub AddPcs { - my $A = shift; - my $B = shift; - - my $R = {}; - # add all keys in A - foreach my $k (keys(%{$A})) { - $R->{$k} = 1 - } - # add all keys in B - foreach my $k (keys(%{$B})) { - $R->{$k} = 1 - } - return $R; -} - -# Subtract B from A -sub SubtractProfile { - my $A = shift; - my $B = shift; - - my $R = {}; - foreach my $k (keys(%{$A})) { - my $v = $A->{$k} - GetEntry($B, $k); - if ($v < 0 && $main::opt_drop_negative) { - $v = 0; - } - AddEntry($R, $k, $v); - } - if (!$main::opt_drop_negative) { - # Take care of when subtracted profile has more entries - foreach my $k (keys(%{$B})) { - if (!exists($A->{$k})) { - AddEntry($R, $k, 0 - $B->{$k}); - } - } - } - return $R; -} - -# Get entry from profile; zero if not present -sub GetEntry { - my $profile = shift; - my $k = shift; - if (exists($profile->{$k})) { - return $profile->{$k}; - } else { - return 0; - } -} - -# Add entry to specified profile -sub AddEntry { - my $profile = shift; - my $k = shift; - my $n = shift; - if (!exists($profile->{$k})) { - $profile->{$k} = 0; - } - $profile->{$k} += $n; -} - -# Add a stack of entries to specified profile, and add them to the $pcs -# list. -sub AddEntries { - my $profile = shift; - my $pcs = shift; - my $stack = shift; - my $count = shift; - my @k = (); - - foreach my $e (split(/\s+/, $stack)) { - my $pc = HexExtend($e); - $pcs->{$pc} = 1; - push @k, $pc; - } - AddEntry($profile, (join "\n", @k), $count); -} - -##### Code to profile a server dynamically ##### - -sub CheckSymbolPage { - my $url = SymbolPageURL(); - open(SYMBOL, "$URL_FETCHER '$url' |"); - my $line = ; - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - close(SYMBOL); - unless (defined($line)) { - error("$url doesn't exist\n"); - } - - if ($line =~ /^num_symbols:\s+(\d+)$/) { - if ($1 == 0) { - error("Stripped binary. No symbols available.\n"); - } - } else { - error("Failed to get the number of symbols from $url\n"); - } -} - -sub IsProfileURL { - my $profile_name = shift; - if (-f $profile_name) { - printf STDERR "Using local file $profile_name.\n"; - return 0; - } - return 1; -} - -sub ParseProfileURL { - my $profile_name = shift; - - if (!defined($profile_name) || $profile_name eq "") { - return (); - } - - # Split profile URL - matches all non-empty strings, so no test. - $profile_name =~ m,^(https?://)?([^/]+)(.*?)(/|$PROFILES)?$,; - - my $proto = $1 || "http://"; - my $hostport = $2; - my $prefix = $3; - my $profile = $4 || "/"; - - my $host = $hostport; - $host =~ s/:.*//; - - my $baseurl = "$proto$hostport$prefix"; - return ($host, $baseurl, $profile); -} - -# We fetch symbols from the first profile argument. -sub SymbolPageURL { - my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); - return "$baseURL$SYMBOL_PAGE"; -} - -sub FetchProgramName() { - my ($host, $baseURL, $path) = ParseProfileURL($main::pfile_args[0]); - my $url = "$baseURL$PROGRAM_NAME_PAGE"; - my $command_line = "$URL_FETCHER '$url'"; - open(CMDLINE, "$command_line |") or error($command_line); - my $cmdline = ; - $cmdline =~ s/\r//g; # turn windows-looking lines into unix-looking lines - close(CMDLINE); - error("Failed to get program name from $url\n") unless defined($cmdline); - $cmdline =~ s/\x00.+//; # Remove argv[1] and latters. - $cmdline =~ s!\n!!g; # Remove LFs. - return $cmdline; -} - -# Gee, curl's -L (--location) option isn't reliable at least -# with its 7.12.3 version. Curl will forget to post data if -# there is a redirection. This function is a workaround for -# curl. Redirection happens on borg hosts. -sub ResolveRedirectionForCurl { - my $url = shift; - my $command_line = "$URL_FETCHER --head '$url'"; - open(CMDLINE, "$command_line |") or error($command_line); - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (/^Location: (.*)/) { - $url = $1; - } - } - close(CMDLINE); - return $url; -} - -# Add a timeout flat to URL_FETCHER -sub AddFetchTimeout { - my $fetcher = shift; - my $timeout = shift; - if (defined($timeout)) { - if ($fetcher =~ m/\bcurl -s/) { - $fetcher .= sprintf(" --max-time %d", $timeout); - } elsif ($fetcher =~ m/\brpcget\b/) { - $fetcher .= sprintf(" --deadline=%d", $timeout); - } - } - return $fetcher; -} - -# Reads a symbol map from the file handle name given as $1, returning -# the resulting symbol map. Also processes variables relating to symbols. -# Currently, the only variable processed is 'binary=' which updates -# $main::prog to have the correct program name. -sub ReadSymbols { - my $in = shift; - my $map = {}; - while (<$in>) { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Removes all the leading zeroes from the symbols, see comment below. - if (m/^0x0*([0-9a-f]+)\s+(.+)/) { - $map->{$1} = $2; - } elsif (m/^---/) { - last; - } elsif (m/^([a-z][^=]*)=(.*)$/ ) { - my ($variable, $value) = ($1, $2); - for ($variable, $value) { - s/^\s+//; - s/\s+$//; - } - if ($variable eq "binary") { - if ($main::prog ne $UNKNOWN_BINARY && $main::prog ne $value) { - printf STDERR ("Warning: Mismatched binary name '%s', using '%s'.\n", - $main::prog, $value); - } - $main::prog = $value; - } else { - printf STDERR ("Ignoring unknown variable in symbols list: " . - "'%s' = '%s'\n", $variable, $value); - } - } - } - return $map; -} - -# Fetches and processes symbols to prepare them for use in the profile output -# code. If the optional 'symbol_map' arg is not given, fetches symbols from -# $SYMBOL_PAGE for all PC values found in profile. Otherwise, the raw symbols -# are assumed to have already been fetched into 'symbol_map' and are simply -# extracted and processed. -sub FetchSymbols { - my $pcset = shift; - my $symbol_map = shift; - - my %seen = (); - my @pcs = grep { !$seen{$_}++ } keys(%$pcset); # uniq - - if (!defined($symbol_map)) { - my $post_data = join("+", sort((map {"0x" . "$_"} @pcs))); - - open(POSTFILE, ">$main::tmpfile_sym"); - print POSTFILE $post_data; - close(POSTFILE); - - my $url = SymbolPageURL(); - - my $command_line; - if ($URL_FETCHER =~ m/\bcurl -s/) { - $url = ResolveRedirectionForCurl($url); - $command_line = "$URL_FETCHER -d '\@$main::tmpfile_sym' '$url'"; - } else { - $command_line = "$URL_FETCHER --post '$url' < '$main::tmpfile_sym'"; - } - # We use c++filt in case $SYMBOL_PAGE gives us mangled symbols. - my $cppfilt = $obj_tool_map{"c++filt"}; - open(SYMBOL, "$command_line | $cppfilt |") or error($command_line); - $symbol_map = ReadSymbols(*SYMBOL{IO}); - close(SYMBOL); - } - - my $symbols = {}; - foreach my $pc (@pcs) { - my $fullname; - # For 64 bits binaries, symbols are extracted with 8 leading zeroes. - # Then /symbol reads the long symbols in as uint64, and outputs - # the result with a "0x%08llx" format which get rid of the zeroes. - # By removing all the leading zeroes in both $pc and the symbols from - # /symbol, the symbols match and are retrievable from the map. - my $shortpc = $pc; - $shortpc =~ s/^0*//; - # Each line may have a list of names, which includes the function - # and also other functions it has inlined. They are separated - # (in PrintSymbolizedFile), by --, which is illegal in function names. - my $fullnames; - if (defined($symbol_map->{$shortpc})) { - $fullnames = $symbol_map->{$shortpc}; - } else { - $fullnames = "0x" . $pc; # Just use addresses - } - my $sym = []; - $symbols->{$pc} = $sym; - foreach my $fullname (split("--", $fullnames)) { - my $name = ShortFunctionName($fullname); - push(@{$sym}, $name, "?", $fullname); - } - } - return $symbols; -} - -sub BaseName { - my $file_name = shift; - $file_name =~ s!^.*/!!; # Remove directory name - return $file_name; -} - -sub MakeProfileBaseName { - my ($binary_name, $profile_name) = @_; - my ($host, $baseURL, $path) = ParseProfileURL($profile_name); - my $binary_shortname = BaseName($binary_name); - return sprintf("%s.%s.%s", - $binary_shortname, $main::op_time, $host); -} - -sub FetchDynamicProfile { - my $binary_name = shift; - my $profile_name = shift; - my $fetch_name_only = shift; - my $encourage_patience = shift; - - if (!IsProfileURL($profile_name)) { - return $profile_name; - } else { - my ($host, $baseURL, $path) = ParseProfileURL($profile_name); - if ($path eq "" || $path eq "/") { - # Missing type specifier defaults to cpu-profile - $path = $PROFILE_PAGE; - } - - my $profile_file = MakeProfileBaseName($binary_name, $profile_name); - - my $url = "$baseURL$path"; - my $fetch_timeout = undef; - if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE/) { - if ($path =~ m/[?]/) { - $url .= "&"; - } else { - $url .= "?"; - } - $url .= sprintf("seconds=%d", $main::opt_seconds); - $fetch_timeout = $main::opt_seconds * 1.01 + 60; - } else { - # For non-CPU profiles, we add a type-extension to - # the target profile file name. - my $suffix = $path; - $suffix =~ s,/,.,g; - $profile_file .= $suffix; - } - - my $profile_dir = $ENV{"PPROF_TMPDIR"} || ($ENV{HOME} . "/pprof"); - if (! -d $profile_dir) { - mkdir($profile_dir) - || die("Unable to create profile directory $profile_dir: $!\n"); - } - my $tmp_profile = "$profile_dir/.tmp.$profile_file"; - my $real_profile = "$profile_dir/$profile_file"; - - if ($fetch_name_only > 0) { - return $real_profile; - } - - my $fetcher = AddFetchTimeout($URL_FETCHER, $fetch_timeout); - my $cmd = "$fetcher '$url' > '$tmp_profile'"; - if ($path =~ m/$PROFILE_PAGE|$PMUPROFILE_PAGE|$CENSUSPROFILE_PAGE/){ - print STDERR "Gathering CPU profile from $url for $main::opt_seconds seconds to\n ${real_profile}\n"; - if ($encourage_patience) { - print STDERR "Be patient...\n"; - } - } else { - print STDERR "Fetching $path profile from $url to\n ${real_profile}\n"; - } - - (system($cmd) == 0) || error("Failed to get profile: $cmd: $!\n"); - (system("mv $tmp_profile $real_profile") == 0) || error("Unable to rename profile\n"); - print STDERR "Wrote profile to $real_profile\n"; - $main::collected_profile = $real_profile; - return $main::collected_profile; - } -} - -# Collect profiles in parallel -sub FetchDynamicProfiles { - my $items = scalar(@main::pfile_args); - my $levels = log($items) / log(2); - - if ($items == 1) { - $main::profile_files[0] = FetchDynamicProfile($main::prog, $main::pfile_args[0], 0, 1); - } else { - # math rounding issues - if ((2 ** $levels) < $items) { - $levels++; - } - my $count = scalar(@main::pfile_args); - for (my $i = 0; $i < $count; $i++) { - $main::profile_files[$i] = FetchDynamicProfile($main::prog, $main::pfile_args[$i], 1, 0); - } - print STDERR "Fetching $count profiles, Be patient...\n"; - FetchDynamicProfilesRecurse($levels, 0, 0); - $main::collected_profile = join(" \\\n ", @main::profile_files); - } -} - -# Recursively fork a process to get enough processes -# collecting profiles -sub FetchDynamicProfilesRecurse { - my $maxlevel = shift; - my $level = shift; - my $position = shift; - - if (my $pid = fork()) { - $position = 0 | ($position << 1); - TryCollectProfile($maxlevel, $level, $position); - wait; - } else { - $position = 1 | ($position << 1); - TryCollectProfile($maxlevel, $level, $position); - cleanup(); - exit(0); - } -} - -# Collect a single profile -sub TryCollectProfile { - my $maxlevel = shift; - my $level = shift; - my $position = shift; - - if ($level >= ($maxlevel - 1)) { - if ($position < scalar(@main::pfile_args)) { - FetchDynamicProfile($main::prog, $main::pfile_args[$position], 0, 0); - } - } else { - FetchDynamicProfilesRecurse($maxlevel, $level+1, $position); - } -} - -##### Parsing code ##### - -# Provide a small streaming-read module to handle very large -# cpu-profile files. Stream in chunks along a sliding window. -# Provides an interface to get one 'slot', correctly handling -# endian-ness differences. A slot is one 32-bit or 64-bit word -# (depending on the input profile). We tell endianness and bit-size -# for the profile by looking at the first 8 bytes: in cpu profiles, -# the second slot is always 3 (we'll accept anything that's not 0). -BEGIN { - package CpuProfileStream; - - sub new { - my ($class, $file, $fname) = @_; - my $self = { file => $file, - base => 0, - stride => 512 * 1024, # must be a multiple of bitsize/8 - slots => [], - unpack_code => "", # N for big-endian, V for little - perl_is_64bit => 1, # matters if profile is 64-bit - }; - bless $self, $class; - # Let unittests adjust the stride - if ($main::opt_test_stride > 0) { - $self->{stride} = $main::opt_test_stride; - } - # Read the first two slots to figure out bitsize and endianness. - my $slots = $self->{slots}; - my $str; - read($self->{file}, $str, 8); - # Set the global $address_length based on what we see here. - # 8 is 32-bit (8 hexadecimal chars); 16 is 64-bit (16 hexadecimal chars). - $address_length = ($str eq (chr(0)x8)) ? 16 : 8; - if ($address_length == 8) { - if (substr($str, 6, 2) eq chr(0)x2) { - $self->{unpack_code} = 'V'; # Little-endian. - } elsif (substr($str, 4, 2) eq chr(0)x2) { - $self->{unpack_code} = 'N'; # Big-endian - } else { - ::error("$fname: header size >= 2**16\n"); - } - @$slots = unpack($self->{unpack_code} . "*", $str); - } else { - # If we're a 64-bit profile, check if we're a 64-bit-capable - # perl. Otherwise, each slot will be represented as a float - # instead of an int64, losing precision and making all the - # 64-bit addresses wrong. We won't complain yet, but will - # later if we ever see a value that doesn't fit in 32 bits. - my $has_q = 0; - eval { $has_q = pack("Q", "1") ? 1 : 1; }; - if (!$has_q) { - $self->{perl_is_64bit} = 0; - } - read($self->{file}, $str, 8); - if (substr($str, 4, 4) eq chr(0)x4) { - # We'd love to use 'Q', but it's a) not universal, b) not endian-proof. - $self->{unpack_code} = 'V'; # Little-endian. - } elsif (substr($str, 0, 4) eq chr(0)x4) { - $self->{unpack_code} = 'N'; # Big-endian - } else { - ::error("$fname: header size >= 2**32\n"); - } - my @pair = unpack($self->{unpack_code} . "*", $str); - # Since we know one of the pair is 0, it's fine to just add them. - @$slots = (0, $pair[0] + $pair[1]); - } - return $self; - } - - # Load more data when we access slots->get(X) which is not yet in memory. - sub overflow { - my ($self) = @_; - my $slots = $self->{slots}; - $self->{base} += $#$slots + 1; # skip over data we're replacing - my $str; - read($self->{file}, $str, $self->{stride}); - if ($address_length == 8) { # the 32-bit case - # This is the easy case: unpack provides 32-bit unpacking primitives. - @$slots = unpack($self->{unpack_code} . "*", $str); - } else { - # We need to unpack 32 bits at a time and combine. - my @b32_values = unpack($self->{unpack_code} . "*", $str); - my @b64_values = (); - for (my $i = 0; $i < $#b32_values; $i += 2) { - # TODO(csilvers): if this is a 32-bit perl, the math below - # could end up in a too-large int, which perl will promote - # to a double, losing necessary precision. Deal with that. - # Right now, we just die. - my ($lo, $hi) = ($b32_values[$i], $b32_values[$i+1]); - if ($self->{unpack_code} eq 'N') { # big-endian - ($lo, $hi) = ($hi, $lo); - } - my $value = $lo + $hi * (2**32); - if (!$self->{perl_is_64bit} && # check value is exactly represented - (($value % (2**32)) != $lo || int($value / (2**32)) != $hi)) { - ::error("Need a 64-bit perl to process this 64-bit profile.\n"); - } - push(@b64_values, $value); - } - @$slots = @b64_values; - } - } - - # Access the i-th long in the file (logically), or -1 at EOF. - sub get { - my ($self, $idx) = @_; - my $slots = $self->{slots}; - while ($#$slots >= 0) { - if ($idx < $self->{base}) { - # The only time we expect a reference to $slots[$i - something] - # after referencing $slots[$i] is reading the very first header. - # Since $stride > |header|, that shouldn't cause any lookback - # errors. And everything after the header is sequential. - print STDERR "Unexpected look-back reading CPU profile"; - return -1; # shrug, don't know what better to return - } elsif ($idx > $self->{base} + $#$slots) { - $self->overflow(); - } else { - return $slots->[$idx - $self->{base}]; - } - } - # If we get here, $slots is [], which means we've reached EOF - return -1; # unique since slots is supposed to hold unsigned numbers - } -} - -# Reads the top, 'header' section of a profile, and returns the last -# line of the header, commonly called a 'header line'. The header -# section of a profile consists of zero or more 'command' lines that -# are instructions to pprof, which pprof executes when reading the -# header. All 'command' lines start with a %. After the command -# lines is the 'header line', which is a profile-specific line that -# indicates what type of profile it is, and perhaps other global -# information about the profile. For instance, here's a header line -# for a heap profile: -# heap profile: 53: 38236 [ 5525: 1284029] @ heapprofile -# For historical reasons, the CPU profile does not contain a text- -# readable header line. If the profile looks like a CPU profile, -# this function returns "". If no header line could be found, this -# function returns undef. -# -# The following commands are recognized: -# %warn -- emit the rest of this line to stderr, prefixed by 'WARNING:' -# -# The input file should be in binmode. -sub ReadProfileHeader { - local *PROFILE = shift; - my $firstchar = ""; - my $line = ""; - read(PROFILE, $firstchar, 1); - seek(PROFILE, -1, 1); # unread the firstchar - if ($firstchar !~ /[[:print:]]/) { # is not a text character - return ""; - } - while (defined($line = )) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ($line =~ /^%warn\s+(.*)/) { # 'warn' command - # Note this matches both '%warn blah\n' and '%warn\n'. - print STDERR "WARNING: $1\n"; # print the rest of the line - } elsif ($line =~ /^%/) { - print STDERR "Ignoring unknown command from profile header: $line"; - } else { - # End of commands, must be the header line. - return $line; - } - } - return undef; # got to EOF without seeing a header line -} - -sub IsSymbolizedProfileFile { - my $file_name = shift; - if (!(-e $file_name) || !(-r $file_name)) { - return 0; - } - # Check if the file contains a symbol-section marker. - open(TFILE, "<$file_name"); - binmode TFILE; - my $firstline = ReadProfileHeader(*TFILE); - close(TFILE); - if (!$firstline) { - return 0; - } - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - return $firstline =~ /^--- *$symbol_marker/; -} - -# Parse profile generated by common/profiler.cc and return a reference -# to a map: -# $result->{version} Version number of profile file -# $result->{period} Sampling period (in microseconds) -# $result->{profile} Profile object -# $result->{map} Memory map info from profile -# $result->{pcs} Hash of all PC values seen, key is hex address -sub ReadProfile { - my $prog = shift; - my $fname = shift; - my $result; # return value - - $CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $contention_marker = $&; - $GROWTH_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $growth_marker = $&; - $SYMBOL_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $symbol_marker = $&; - $PROFILE_PAGE =~ m,[^/]+$,; # matches everything after the last slash - my $profile_marker = $&; - - # Look at first line to see if it is a heap or a CPU profile. - # CPU profile may start with no header at all, and just binary data - # (starting with \0\0\0\0) -- in that case, don't try to read the - # whole firstline, since it may be gigabytes(!) of data. - open(PROFILE, "<$fname") || error("$fname: $!\n"); - binmode PROFILE; # New perls do UTF-8 processing - my $header = ReadProfileHeader(*PROFILE); - if (!defined($header)) { # means "at EOF" - error("Profile is empty.\n"); - } - - my $symbols; - if ($header =~ m/^--- *$symbol_marker/o) { - # Verify that the user asked for a symbolized profile - if (!$main::use_symbolized_profile) { - # we have both a binary and symbolized profiles, abort - error("FATAL ERROR: Symbolized profile\n $fname\ncannot be used with " . - "a binary arg. Try again without passing\n $prog\n"); - } - # Read the symbol section of the symbolized profile file. - $symbols = ReadSymbols(*PROFILE{IO}); - # Read the next line to get the header for the remaining profile. - $header = ReadProfileHeader(*PROFILE) || ""; - } - - $main::profile_type = ''; - if ($header =~ m/^heap profile:.*$growth_marker/o) { - $main::profile_type = 'growth'; - $result = ReadHeapProfile($prog, *PROFILE, $header); - } elsif ($header =~ m/^heap profile:/) { - $main::profile_type = 'heap'; - $result = ReadHeapProfile($prog, *PROFILE, $header); - } elsif ($header =~ m/^--- *$contention_marker/o) { - $main::profile_type = 'contention'; - $result = ReadSynchProfile($prog, *PROFILE); - } elsif ($header =~ m/^--- *Stacks:/) { - print STDERR - "Old format contention profile: mistakenly reports " . - "condition variable signals as lock contentions.\n"; - $main::profile_type = 'contention'; - $result = ReadSynchProfile($prog, *PROFILE); - } elsif ($header =~ m/^--- *$profile_marker/) { - # the binary cpu profile data starts immediately after this line - $main::profile_type = 'cpu'; - $result = ReadCPUProfile($prog, $fname, *PROFILE); - } else { - if (defined($symbols)) { - # a symbolized profile contains a format we don't recognize, bail out - error("$fname: Cannot recognize profile section after symbols.\n"); - } - # no ascii header present -- must be a CPU profile - $main::profile_type = 'cpu'; - $result = ReadCPUProfile($prog, $fname, *PROFILE); - } - - close(PROFILE); - - # if we got symbols along with the profile, return those as well - if (defined($symbols)) { - $result->{symbols} = $symbols; - } - - return $result; -} - -# Subtract one from caller pc so we map back to call instr. -# However, don't do this if we're reading a symbolized profile -# file, in which case the subtract-one was done when the file -# was written. -# -# We apply the same logic to all readers, though ReadCPUProfile uses an -# independent implementation. -sub FixCallerAddresses { - my $stack = shift; - if ($main::use_symbolized_profile) { - return $stack; - } else { - $stack =~ /(\s)/; - my $delimiter = $1; - my @addrs = split(' ', $stack); - my @fixedaddrs; - $#fixedaddrs = $#addrs; - if ($#addrs >= 0) { - $fixedaddrs[0] = $addrs[0]; - } - for (my $i = 1; $i <= $#addrs; $i++) { - $fixedaddrs[$i] = AddressSub($addrs[$i], "0x1"); - } - return join $delimiter, @fixedaddrs; - } -} - -# CPU profile reader -sub ReadCPUProfile { - my $prog = shift; - my $fname = shift; # just used for logging - local *PROFILE = shift; - my $version; - my $period; - my $i; - my $profile = {}; - my $pcs = {}; - - # Parse string into array of slots. - my $slots = CpuProfileStream->new(*PROFILE, $fname); - - # Read header. The current header version is a 5-element structure - # containing: - # 0: header count (always 0) - # 1: header "words" (after this one: 3) - # 2: format version (0) - # 3: sampling period (usec) - # 4: unused padding (always 0) - if ($slots->get(0) != 0 ) { - error("$fname: not a profile file, or old format profile file\n"); - } - $i = 2 + $slots->get(1); - $version = $slots->get(2); - $period = $slots->get(3); - # Do some sanity checking on these header values. - if ($version > (2**32) || $period > (2**32) || $i > (2**32) || $i < 5) { - error("$fname: not a profile file, or corrupted profile file\n"); - } - - # Parse profile - while ($slots->get($i) != -1) { - my $n = $slots->get($i++); - my $d = $slots->get($i++); - if ($d > (2**16)) { # TODO(csilvers): what's a reasonable max-stack-depth? - my $addr = sprintf("0%o", $i * ($address_length == 8 ? 4 : 8)); - print STDERR "At index $i (address $addr):\n"; - error("$fname: stack trace depth >= 2**32\n"); - } - if ($slots->get($i) == 0) { - # End of profile data marker - $i += $d; - last; - } - - # Make key out of the stack entries - my @k = (); - for (my $j = 0; $j < $d; $j++) { - my $pc = $slots->get($i+$j); - # Subtract one from caller pc so we map back to call instr. - # However, don't do this if we're reading a symbolized profile - # file, in which case the subtract-one was done when the file - # was written. - if ($j > 0 && !$main::use_symbolized_profile) { - $pc--; - } - $pc = sprintf("%0*x", $address_length, $pc); - $pcs->{$pc} = 1; - push @k, $pc; - } - - AddEntry($profile, (join "\n", @k), $n); - $i += $d; - } - - # Parse map - my $map = ''; - seek(PROFILE, $i * 4, 0); - read(PROFILE, $map, (stat PROFILE)[7]); - - my $r = {}; - $r->{version} = $version; - $r->{period} = $period; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - - return $r; -} - -sub ReadHeapProfile { - my $prog = shift; - local *PROFILE = shift; - my $header = shift; - - my $index = 1; - if ($main::opt_inuse_space) { - $index = 1; - } elsif ($main::opt_inuse_objects) { - $index = 0; - } elsif ($main::opt_alloc_space) { - $index = 3; - } elsif ($main::opt_alloc_objects) { - $index = 2; - } - - # Find the type of this profile. The header line looks like: - # heap profile: 1246: 8800744 [ 1246: 8800744] @ /266053 - # There are two pairs , the first inuse objects/space, and the - # second allocated objects/space. This is followed optionally by a profile - # type, and if that is present, optionally by a sampling frequency. - # For remote heap profiles (v1): - # The interpretation of the sampling frequency is that the profiler, for - # each sample, calculates a uniformly distributed random integer less than - # the given value, and records the next sample after that many bytes have - # been allocated. Therefore, the expected sample interval is half of the - # given frequency. By default, if not specified, the expected sample - # interval is 128KB. Only remote-heap-page profiles are adjusted for - # sample size. - # For remote heap profiles (v2): - # The sampling frequency is the rate of a Poisson process. This means that - # the probability of sampling an allocation of size X with sampling rate Y - # is 1 - exp(-X/Y) - # For version 2, a typical header line might look like this: - # heap profile: 1922: 127792360 [ 1922: 127792360] @ _v2/524288 - # the trailing number (524288) is the sampling rate. (Version 1 showed - # double the 'rate' here) - my $sampling_algorithm = 0; - my $sample_adjustment = 0; - chomp($header); - my $type = "unknown"; - if ($header =~ m"^heap profile:\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\](\s*@\s*([^/]*)(/(\d+))?)?") { - if (defined($6) && ($6 ne '')) { - $type = $6; - my $sample_period = $8; - # $type is "heapprofile" for profiles generated by the - # heap-profiler, and either "heap" or "heap_v2" for profiles - # generated by sampling directly within tcmalloc. It can also - # be "growth" for heap-growth profiles. The first is typically - # found for profiles generated locally, and the others for - # remote profiles. - if (($type eq "heapprofile") || ($type !~ /heap/) ) { - # No need to adjust for the sampling rate with heap-profiler-derived data - $sampling_algorithm = 0; - } elsif ($type =~ /_v2/) { - $sampling_algorithm = 2; # version 2 sampling - if (defined($sample_period) && ($sample_period ne '')) { - $sample_adjustment = int($sample_period); - } - } else { - $sampling_algorithm = 1; # version 1 sampling - if (defined($sample_period) && ($sample_period ne '')) { - $sample_adjustment = int($sample_period)/2; - } - } - } else { - # We detect whether or not this is a remote-heap profile by checking - # that the total-allocated stats ($n2,$s2) are exactly the - # same as the in-use stats ($n1,$s1). It is remotely conceivable - # that a non-remote-heap profile may pass this check, but it is hard - # to imagine how that could happen. - # In this case it's so old it's guaranteed to be remote-heap version 1. - my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - if (($n1 == $n2) && ($s1 == $s2)) { - # This is likely to be a remote-heap based sample profile - $sampling_algorithm = 1; - } - } - } - - if ($sampling_algorithm > 0) { - # For remote-heap generated profiles, adjust the counts and sizes to - # account for the sample rate (we sample once every 128KB by default). - if ($sample_adjustment == 0) { - # Turn on profile adjustment. - $sample_adjustment = 128*1024; - print STDERR "Adjusting heap profiles for 1-in-128KB sampling rate\n"; - } else { - printf STDERR ("Adjusting heap profiles for 1-in-%d sampling rate\n", - $sample_adjustment); - } - if ($sampling_algorithm > 1) { - # We don't bother printing anything for the original version (version 1) - printf STDERR "Heap version $sampling_algorithm\n"; - } - } - - my $profile = {}; - my $pcs = {}; - my $map = ""; - - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (/^MAPPED_LIBRARIES:/) { - # Read the /proc/self/maps data - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - $map .= $_; - } - last; - } - - if (/^--- Memory map:/) { - # Read /proc/self/maps data as formatted by DumpAddressMap() - my $buildvar = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Parse "build=" specification if supplied - if (m/^\s*build=(.*)\n/) { - $buildvar = $1; - } - - # Expand "$build" variable if available - $_ =~ s/\$build\b/$buildvar/g; - - $map .= $_; - } - last; - } - - # Read entry of the form: - # : [: ] @ a1 a2 a3 ... an - s/^\s*//; - s/\s*$//; - if (m/^\s*(\d+):\s+(\d+)\s+\[\s*(\d+):\s+(\d+)\]\s+@\s+(.*)$/) { - my $stack = $5; - my ($n1, $s1, $n2, $s2) = ($1, $2, $3, $4); - - if ($sample_adjustment) { - if ($sampling_algorithm == 2) { - # Remote-heap version 2 - # The sampling frequency is the rate of a Poisson process. - # This means that the probability of sampling an allocation of - # size X with sampling rate Y is 1 - exp(-X/Y) - if ($n1 != 0) { - my $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n1 *= $scale_factor; - $s1 *= $scale_factor; - } - if ($n2 != 0) { - my $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - my $scale_factor = 1/(1 - exp(-$ratio)); - $n2 *= $scale_factor; - $s2 *= $scale_factor; - } - } else { - # Remote-heap version 1 - my $ratio; - $ratio = (($s1*1.0)/$n1)/($sample_adjustment); - if ($ratio < 1) { - $n1 /= $ratio; - $s1 /= $ratio; - } - $ratio = (($s2*1.0)/$n2)/($sample_adjustment); - if ($ratio < 1) { - $n2 /= $ratio; - $s2 /= $ratio; - } - } - } - - my @counts = ($n1, $s1, $n2, $s2); - AddEntries($profile, $pcs, FixCallerAddresses($stack), $counts[$index]); - } - } - - my $r = {}; - $r->{version} = "heap"; - $r->{period} = 1; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -sub ReadSynchProfile { - my $prog = shift; - local *PROFILE = shift; - my $header = shift; - - my $map = ''; - my $profile = {}; - my $pcs = {}; - my $sampling_period = 1; - my $cyclespernanosec = 2.8; # Default assumption for old binaries - my $seen_clockrate = 0; - my $line; - - my $index = 0; - if ($main::opt_total_delay) { - $index = 0; - } elsif ($main::opt_contentions) { - $index = 1; - } elsif ($main::opt_mean_delay) { - $index = 2; - } - - while ( $line = ) { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - if ( $line =~ /^\s*(\d+)\s+(\d+) \@\s*(.*?)\s*$/ ) { - my ($cycles, $count, $stack) = ($1, $2, $3); - - # Convert cycles to nanoseconds - $cycles /= $cyclespernanosec; - - # Adjust for sampling done by application - $cycles *= $sampling_period; - $count *= $sampling_period; - - my @values = ($cycles, $count, $cycles / $count); - AddEntries($profile, $pcs, FixCallerAddresses($stack), $values[$index]); - - } elsif ( $line =~ /^(slow release).*thread \d+ \@\s*(.*?)\s*$/ || - $line =~ /^\s*(\d+) \@\s*(.*?)\s*$/ ) { - my ($cycles, $stack) = ($1, $2); - if ($cycles !~ /^\d+$/) { - next; - } - - # Convert cycles to nanoseconds - $cycles /= $cyclespernanosec; - - # Adjust for sampling done by application - $cycles *= $sampling_period; - - AddEntries($profile, $pcs, FixCallerAddresses($stack), $cycles); - - } elsif ( $line =~ m/^([a-z][^=]*)=(.*)$/ ) { - my ($variable, $value) = ($1,$2); - for ($variable, $value) { - s/^\s+//; - s/\s+$//; - } - if ($variable eq "cycles/second") { - $cyclespernanosec = $value / 1e9; - $seen_clockrate = 1; - } elsif ($variable eq "sampling period") { - $sampling_period = $value; - } elsif ($variable eq "ms since reset") { - # Currently nothing is done with this value in pprof - # So we just silently ignore it for now - } elsif ($variable eq "discarded samples") { - # Currently nothing is done with this value in pprof - # So we just silently ignore it for now - } else { - printf STDERR ("Ignoring unnknown variable in /contention output: " . - "'%s' = '%s'\n",$variable,$value); - } - } else { - # Memory map entry - $map .= $line; - } - } - - if (!$seen_clockrate) { - printf STDERR ("No cycles/second entry in profile; Guessing %.1f GHz\n", - $cyclespernanosec); - } - - my $r = {}; - $r->{version} = 0; - $r->{period} = $sampling_period; - $r->{profile} = $profile; - $r->{libs} = ParseLibraries($prog, $map, $pcs); - $r->{pcs} = $pcs; - return $r; -} - -# Given a hex value in the form "0x1abcd" return "0001abcd" or -# "000000000001abcd", depending on the current address length. -# There's probably a more idiomatic (or faster) way to do this... -sub HexExtend { - my $addr = shift; - - $addr =~ s/^0x//; - - if (length $addr > $address_length) { - printf STDERR "Warning: address $addr is longer than address length $address_length\n"; - } - - return substr("000000000000000".$addr, -$address_length); -} - -##### Symbol extraction ##### - -# Aggressively search the lib_prefix values for the given library -# If all else fails, just return the name of the library unmodified. -# If the lib_prefix is "/my/path,/other/path" and $file is "/lib/dir/mylib.so" -# it will search the following locations in this order, until it finds a file: -# /my/path/lib/dir/mylib.so -# /other/path/lib/dir/mylib.so -# /my/path/dir/mylib.so -# /other/path/dir/mylib.so -# /my/path/mylib.so -# /other/path/mylib.so -# /lib/dir/mylib.so (returned as last resort) -sub FindLibrary { - my $file = shift; - my $suffix = $file; - - # Search for the library as described above - do { - foreach my $prefix (@prefix_list) { - my $fullpath = $prefix . $suffix; - if (-e $fullpath) { - return $fullpath; - } - } - } while ($suffix =~ s|^/[^/]+/|/|); - return $file; -} - -# Return path to library with debugging symbols. -# For libc libraries, the copy in /usr/lib/debug contains debugging symbols -sub DebuggingLibrary { - my $file = shift; - if ($file =~ m|^/| && -f "/usr/lib/debug$file") { - return "/usr/lib/debug$file"; - } - return undef; -} - -# Parse text section header of a library using objdump -sub ParseTextSectionHeaderFromObjdump { - my $lib = shift; - - my $size = undef; - my $vma; - my $file_offset; - # Get objdump output from the library file to figure out how to - # map between mapped addresses and addresses in the library. - my $objdump = $obj_tool_map{"objdump"}; - open(OBJDUMP, "$objdump -h $lib |") - || error("$objdump $lib: $!\n"); - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - # Idx Name Size VMA LMA File off Algn - # 10 .text 00104b2c 420156f0 420156f0 000156f0 2**4 - # For 64-bit objects, VMA and LMA will be 16 hex digits, size and file - # offset may still be 8. But AddressSub below will still handle that. - my @x = split; - if (($#x >= 6) && ($x[1] eq '.text')) { - $size = $x[2]; - $vma = $x[3]; - $file_offset = $x[5]; - last; - } - } - close(OBJDUMP); - - if (!defined($size)) { - return undef; - } - - my $r = {}; - $r->{size} = $size; - $r->{vma} = $vma; - $r->{file_offset} = $file_offset; - - return $r; -} - -# Parse text section header of a library using otool (on OS X) -sub ParseTextSectionHeaderFromOtool { - my $lib = shift; - - my $size = undef; - my $vma = undef; - my $file_offset = undef; - # Get otool output from the library file to figure out how to - # map between mapped addresses and addresses in the library. - my $otool = $obj_tool_map{"otool"}; - open(OTOOL, "$otool -l $lib |") - || error("$otool $lib: $!\n"); - my $cmd = ""; - my $sectname = ""; - my $segname = ""; - foreach my $line () { - $line =~ s/\r//g; # turn windows-looking lines into unix-looking lines - # Load command <#> - # cmd LC_SEGMENT - # [...] - # Section - # sectname __text - # segname __TEXT - # addr 0x000009f8 - # size 0x00018b9e - # offset 2552 - # align 2^2 (4) - # We will need to strip off the leading 0x from the hex addresses, - # and convert the offset into hex. - if ($line =~ /Load command/) { - $cmd = ""; - $sectname = ""; - $segname = ""; - } elsif ($line =~ /Section/) { - $sectname = ""; - $segname = ""; - } elsif ($line =~ /cmd (\w+)/) { - $cmd = $1; - } elsif ($line =~ /sectname (\w+)/) { - $sectname = $1; - } elsif ($line =~ /segname (\w+)/) { - $segname = $1; - } elsif (!(($cmd eq "LC_SEGMENT" || $cmd eq "LC_SEGMENT_64") && - $sectname eq "__text" && - $segname eq "__TEXT")) { - next; - } elsif ($line =~ /\baddr 0x([0-9a-fA-F]+)/) { - $vma = $1; - } elsif ($line =~ /\bsize 0x([0-9a-fA-F]+)/) { - $size = $1; - } elsif ($line =~ /\boffset ([0-9]+)/) { - $file_offset = sprintf("%016x", $1); - } - if (defined($vma) && defined($size) && defined($file_offset)) { - last; - } - } - close(OTOOL); - - if (!defined($vma) || !defined($size) || !defined($file_offset)) { - return undef; - } - - my $r = {}; - $r->{size} = $size; - $r->{vma} = $vma; - $r->{file_offset} = $file_offset; - - return $r; -} - -sub ParseTextSectionHeader { - # obj_tool_map("otool") is only defined if we're in a Mach-O environment - if (defined($obj_tool_map{"otool"})) { - my $r = ParseTextSectionHeaderFromOtool(@_); - if (defined($r)){ - return $r; - } - } - # If otool doesn't work, or we don't have it, fall back to objdump - return ParseTextSectionHeaderFromObjdump(@_); -} - -# Split /proc/pid/maps dump into a list of libraries -sub ParseLibraries { - return if $main::use_symbol_page; # We don't need libraries info. - my $prog = shift; - my $map = shift; - my $pcs = shift; - - my $result = []; - my $h = "[a-f0-9]+"; - my $zero_offset = HexExtend("0"); - - my $buildvar = ""; - foreach my $l (split("\n", $map)) { - if ($l =~ m/^\s*build=(.*)$/) { - $buildvar = $1; - } - - my $start; - my $finish; - my $offset; - my $lib; - if ($l =~ /^($h)-($h)\s+..x.\s+($h)\s+\S+:\S+\s+\d+\s+(\S+\.(so|dll|dylib|bundle)((\.\d+)+\w*(\.\d+){0,3})?)$/i) { - # Full line from /proc/self/maps. Example: - # 40000000-40015000 r-xp 00000000 03:01 12845071 /lib/ld-2.3.2.so - $start = HexExtend($1); - $finish = HexExtend($2); - $offset = HexExtend($3); - $lib = $4; - $lib =~ s|\\|/|g; # turn windows-style paths into unix-style paths - } elsif ($l =~ /^\s*($h)-($h):\s*(\S+\.so(\.\d+)*)/) { - # Cooked line from DumpAddressMap. Example: - # 40000000-40015000: /lib/ld-2.3.2.so - $start = HexExtend($1); - $finish = HexExtend($2); - $offset = $zero_offset; - $lib = $3; - } else { - next; - } - - # Expand "$build" variable if available - $lib =~ s/\$build\b/$buildvar/g; - - $lib = FindLibrary($lib); - - # Check for pre-relocated libraries, which use pre-relocated symbol tables - # and thus require adjusting the offset that we'll use to translate - # VM addresses into symbol table addresses. - # Only do this if we're not going to fetch the symbol table from a - # debugging copy of the library. - if (!DebuggingLibrary($lib)) { - my $text = ParseTextSectionHeader($lib); - if (defined($text)) { - my $vma_offset = AddressSub($text->{vma}, $text->{file_offset}); - $offset = AddressAdd($offset, $vma_offset); - } - } - - push(@{$result}, [$lib, $start, $finish, $offset]); - } - - # Append special entry for additional library (not relocated) - if ($main::opt_lib ne "") { - my $text = ParseTextSectionHeader($main::opt_lib); - if (defined($text)) { - my $start = $text->{vma}; - my $finish = AddressAdd($start, $text->{size}); - - push(@{$result}, [$main::opt_lib, $start, $finish, $start]); - } - } - - # Append special entry for the main program. This covers - # 0..max_pc_value_seen, so that we assume pc values not found in one - # of the library ranges will be treated as coming from the main - # program binary. - my $min_pc = HexExtend("0"); - my $max_pc = $min_pc; # find the maximal PC value in any sample - foreach my $pc (keys(%{$pcs})) { - if (HexExtend($pc) gt $max_pc) { $max_pc = HexExtend($pc); } - } - push(@{$result}, [$prog, $min_pc, $max_pc, $zero_offset]); - - return $result; -} - -# Add two hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressAdd { - my $addr1 = shift; - my $addr2 = shift; - my $sum; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $sum = (hex($addr1)+hex($addr2)) % (0x10000000 * 16); - return sprintf("%08x", $sum); - - } else { - # Do the addition in 7-nibble chunks to trivialize carry handling. - - if ($main::opt_debug and $main::opt_test) { - print STDERR "AddressAdd $addr1 + $addr2 = "; - } - - my $a1 = substr($addr1,-7); - $addr1 = substr($addr1,0,-7); - my $a2 = substr($addr2,-7); - $addr2 = substr($addr2,0,-7); - $sum = hex($a1) + hex($a2); - my $c = 0; - if ($sum > 0xfffffff) { - $c = 1; - $sum -= 0x10000000; - } - my $r = sprintf("%07x", $sum); - - $a1 = substr($addr1,-7); - $addr1 = substr($addr1,0,-7); - $a2 = substr($addr2,-7); - $addr2 = substr($addr2,0,-7); - $sum = hex($a1) + hex($a2) + $c; - $c = 0; - if ($sum > 0xfffffff) { - $c = 1; - $sum -= 0x10000000; - } - $r = sprintf("%07x", $sum) . $r; - - $sum = hex($addr1) + hex($addr2) + $c; - if ($sum > 0xff) { $sum -= 0x100; } - $r = sprintf("%02x", $sum) . $r; - - if ($main::opt_debug and $main::opt_test) { print STDERR "$r\n"; } - - return $r; - } -} - - -# Subtract two hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressSub { - my $addr1 = shift; - my $addr2 = shift; - my $diff; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $diff = (hex($addr1)-hex($addr2)) % (0x10000000 * 16); - return sprintf("%08x", $diff); - - } else { - # Do the addition in 7-nibble chunks to trivialize borrow handling. - # if ($main::opt_debug) { print STDERR "AddressSub $addr1 - $addr2 = "; } - - my $a1 = hex(substr($addr1,-7)); - $addr1 = substr($addr1,0,-7); - my $a2 = hex(substr($addr2,-7)); - $addr2 = substr($addr2,0,-7); - my $b = 0; - if ($a2 > $a1) { - $b = 1; - $a1 += 0x10000000; - } - $diff = $a1 - $a2; - my $r = sprintf("%07x", $diff); - - $a1 = hex(substr($addr1,-7)); - $addr1 = substr($addr1,0,-7); - $a2 = hex(substr($addr2,-7)) + $b; - $addr2 = substr($addr2,0,-7); - $b = 0; - if ($a2 > $a1) { - $b = 1; - $a1 += 0x10000000; - } - $diff = $a1 - $a2; - $r = sprintf("%07x", $diff) . $r; - - $a1 = hex($addr1); - $a2 = hex($addr2) + $b; - if ($a2 > $a1) { $a1 += 0x100; } - $diff = $a1 - $a2; - $r = sprintf("%02x", $diff) . $r; - - # if ($main::opt_debug) { print STDERR "$r\n"; } - - return $r; - } -} - -# Increment a hex addresses of length $address_length. -# Run pprof --test for unit test if this is changed. -sub AddressInc { - my $addr = shift; - my $sum; - - if ($address_length == 8) { - # Perl doesn't cope with wraparound arithmetic, so do it explicitly: - $sum = (hex($addr)+1) % (0x10000000 * 16); - return sprintf("%08x", $sum); - - } else { - # Do the addition in 7-nibble chunks to trivialize carry handling. - # We are always doing this to step through the addresses in a function, - # and will almost never overflow the first chunk, so we check for this - # case and exit early. - - # if ($main::opt_debug) { print STDERR "AddressInc $addr1 = "; } - - my $a1 = substr($addr,-7); - $addr = substr($addr,0,-7); - $sum = hex($a1) + 1; - my $r = sprintf("%07x", $sum); - if ($sum <= 0xfffffff) { - $r = $addr . $r; - # if ($main::opt_debug) { print STDERR "$r\n"; } - return HexExtend($r); - } else { - $r = "0000000"; - } - - $a1 = substr($addr,-7); - $addr = substr($addr,0,-7); - $sum = hex($a1) + 1; - $r = sprintf("%07x", $sum) . $r; - if ($sum <= 0xfffffff) { - $r = $addr . $r; - # if ($main::opt_debug) { print STDERR "$r\n"; } - return HexExtend($r); - } else { - $r = "00000000000000"; - } - - $sum = hex($addr) + 1; - if ($sum > 0xff) { $sum -= 0x100; } - $r = sprintf("%02x", $sum) . $r; - - # if ($main::opt_debug) { print STDERR "$r\n"; } - return $r; - } -} - -# Extract symbols for all PC values found in profile -sub ExtractSymbols { - my $libs = shift; - my $pcset = shift; - - my $symbols = {}; - - # Map each PC value to the containing library. To make this faster, - # we sort libraries by their starting pc value (highest first), and - # advance through the libraries as we advance the pc. Sometimes the - # addresses of libraries may overlap with the addresses of the main - # binary, so to make sure the libraries 'win', we iterate over the - # libraries in reverse order (which assumes the binary doesn't start - # in the middle of a library, which seems a fair assumption). - my @pcs = (sort { $a cmp $b } keys(%{$pcset})); # pcset is 0-extended strings - foreach my $lib (sort {$b->[1] cmp $a->[1]} @{$libs}) { - my $libname = $lib->[0]; - my $start = $lib->[1]; - my $finish = $lib->[2]; - my $offset = $lib->[3]; - - # Get list of pcs that belong in this library. - my $contained = []; - my ($start_pc_index, $finish_pc_index); - # Find smallest finish_pc_index such that $finish < $pc[$finish_pc_index]. - for ($finish_pc_index = $#pcs + 1; $finish_pc_index > 0; - $finish_pc_index--) { - last if $pcs[$finish_pc_index - 1] le $finish; - } - # Find smallest start_pc_index such that $start <= $pc[$start_pc_index]. - for ($start_pc_index = $finish_pc_index; $start_pc_index > 0; - $start_pc_index--) { - last if $pcs[$start_pc_index - 1] lt $start; - } - # This keeps PC values higher than $pc[$finish_pc_index] in @pcs, - # in case there are overlaps in libraries and the main binary. - @{$contained} = splice(@pcs, $start_pc_index, - $finish_pc_index - $start_pc_index); - # Map to symbols - MapToSymbols($libname, AddressSub($start, $offset), $contained, $symbols); - } - - return $symbols; -} - -# Map list of PC values to symbols for a given image -sub MapToSymbols { - my $image = shift; - my $offset = shift; - my $pclist = shift; - my $symbols = shift; - - my $debug = 0; - - # Ignore empty binaries - if ($#{$pclist} < 0) { return; } - - # Figure out the addr2line command to use - my $addr2line = $obj_tool_map{"addr2line"}; - my $cmd = "$addr2line -f -C -e $image"; - if (exists $obj_tool_map{"addr2line_pdb"}) { - $addr2line = $obj_tool_map{"addr2line_pdb"}; - $cmd = "$addr2line --demangle -f -C -e $image"; - } - - # If "addr2line" isn't installed on the system at all, just use - # nm to get what info we can (function names, but not line numbers). - if (system("$addr2line --help >/dev/null 2>&1") != 0) { - MapSymbolsWithNM($image, $offset, $pclist, $symbols); - return; - } - - # "addr2line -i" can produce a variable number of lines per input - # address, with no separator that allows us to tell when data for - # the next address starts. So we find the address for a special - # symbol (_fini) and interleave this address between all real - # addresses passed to addr2line. The name of this special symbol - # can then be used as a separator. - $sep_address = undef; # May be filled in by MapSymbolsWithNM() - my $nm_symbols = {}; - MapSymbolsWithNM($image, $offset, $pclist, $nm_symbols); - # TODO(csilvers): only add '-i' if addr2line supports it. - if (defined($sep_address)) { - # Only add " -i" to addr2line if the binary supports it. - # addr2line --help returns 0, but not if it sees an unknown flag first. - if (system("$cmd -i --help >/dev/null 2>&1") == 0) { - $cmd .= " -i"; - } else { - $sep_address = undef; # no need for sep_address if we don't support -i - } - } - - # Make file with all PC values with intervening 'sep_address' so - # that we can reliably detect the end of inlined function list - open(ADDRESSES, ">$main::tmpfile_sym") || error("$main::tmpfile_sym: $!\n"); - if ($debug) { print("---- $image ---\n"); } - for (my $i = 0; $i <= $#{$pclist}; $i++) { - # addr2line always reads hex addresses, and does not need '0x' prefix. - if ($debug) { printf STDERR ("%s\n", $pclist->[$i]); } - printf ADDRESSES ("%s\n", AddressSub($pclist->[$i], $offset)); - if (defined($sep_address)) { - printf ADDRESSES ("%s\n", $sep_address); - } - } - close(ADDRESSES); - if ($debug) { - print("----\n"); - system("cat $main::tmpfile_sym"); - print("----\n"); - system("$cmd <$main::tmpfile_sym"); - print("----\n"); - } - - open(SYMBOLS, "$cmd <$main::tmpfile_sym |") || error("$cmd: $!\n"); - my $count = 0; # Index in pclist - while () { - # Read fullfunction and filelineinfo from next pair of lines - s/\r?\n$//g; - my $fullfunction = $_; - $_ = ; - s/\r?\n$//g; - my $filelinenum = $_; - - if (defined($sep_address) && $fullfunction eq $sep_symbol) { - # Terminating marker for data for this address - $count++; - next; - } - - $filelinenum =~ s|\\|/|g; # turn windows-style paths into unix-style paths - - my $pcstr = $pclist->[$count]; - my $function = ShortFunctionName($fullfunction); - if ($fullfunction eq '??') { - # See if nm found a symbol - my $nms = $nm_symbols->{$pcstr}; - if (defined($nms)) { - $function = $nms->[0]; - $fullfunction = $nms->[2]; - } - } - - # Prepend to accumulated symbols for pcstr - # (so that caller comes before callee) - my $sym = $symbols->{$pcstr}; - if (!defined($sym)) { - $sym = []; - $symbols->{$pcstr} = $sym; - } - unshift(@{$sym}, $function, $filelinenum, $fullfunction); - if ($debug) { printf STDERR ("%s => [%s]\n", $pcstr, join(" ", @{$sym})); } - if (!defined($sep_address)) { - # Inlining is off, se this entry ends immediately - $count++; - } - } - close(SYMBOLS); -} - -# Use nm to map the list of referenced PCs to symbols. Return true iff we -# are able to read procedure information via nm. -sub MapSymbolsWithNM { - my $image = shift; - my $offset = shift; - my $pclist = shift; - my $symbols = shift; - - # Get nm output sorted by increasing address - my $symbol_table = GetProcedureBoundaries($image, "."); - if (!%{$symbol_table}) { - return 0; - } - # Start addresses are already the right length (8 or 16 hex digits). - my @names = sort { $symbol_table->{$a}->[0] cmp $symbol_table->{$b}->[0] } - keys(%{$symbol_table}); - - if ($#names < 0) { - # No symbols: just use addresses - foreach my $pc (@{$pclist}) { - my $pcstr = "0x" . $pc; - $symbols->{$pc} = [$pcstr, "?", $pcstr]; - } - return 0; - } - - # Sort addresses so we can do a join against nm output - my $index = 0; - my $fullname = $names[0]; - my $name = ShortFunctionName($fullname); - foreach my $pc (sort { $a cmp $b } @{$pclist}) { - # Adjust for mapped offset - my $mpc = AddressSub($pc, $offset); - while (($index < $#names) && ($mpc ge $symbol_table->{$fullname}->[1])){ - $index++; - $fullname = $names[$index]; - $name = ShortFunctionName($fullname); - } - if ($mpc lt $symbol_table->{$fullname}->[1]) { - $symbols->{$pc} = [$name, "?", $fullname]; - } else { - my $pcstr = "0x" . $pc; - $symbols->{$pc} = [$pcstr, "?", $pcstr]; - } - } - return 1; -} - -sub ShortFunctionName { - my $function = shift; - while ($function =~ s/\([^()]*\)(\s*const)?//g) { } # Argument types - while ($function =~ s/<[^<>]*>//g) { } # Remove template arguments - $function =~ s/^.*\s+(\w+::)/$1/; # Remove leading type - return $function; -} - -##### Miscellaneous ##### - -# Find the right versions of the above object tools to use. The -# argument is the program file being analyzed, and should be an ELF -# 32-bit or ELF 64-bit executable file. The location of the tools -# is determined by considering the following options in this order: -# 1) --tools option, if set -# 2) PPROF_TOOLS environment variable, if set -# 3) the environment -sub ConfigureObjTools { - my $prog_file = shift; - - # Check for the existence of $prog_file because /usr/bin/file does not - # predictably return error status in prod. - (-e $prog_file) || error("$prog_file does not exist.\n"); - - # Follow symlinks (at least for systems where "file" supports that) - my $file_type = `/usr/bin/file -L $prog_file 2>/dev/null || /usr/bin/file $prog_file`; - if ($file_type =~ /64-bit/) { - # Change $address_length to 16 if the program file is ELF 64-bit. - # We can't detect this from many (most?) heap or lock contention - # profiles, since the actual addresses referenced are generally in low - # memory even for 64-bit programs. - $address_length = 16; - } - - if ($file_type =~ /MS Windows/) { - # For windows, we provide a version of nm and addr2line as part of - # the opensource release, which is capable of parsing - # Windows-style PDB executables. It should live in the path, or - # in the same directory as pprof. - $obj_tool_map{"nm_pdb"} = "nm-pdb"; - $obj_tool_map{"addr2line_pdb"} = "addr2line-pdb"; - } - - if ($file_type =~ /Mach-O/) { - # OS X uses otool to examine Mach-O files, rather than objdump. - $obj_tool_map{"otool"} = "otool"; - $obj_tool_map{"addr2line"} = "false"; # no addr2line - $obj_tool_map{"objdump"} = "false"; # no objdump - } - - # Go fill in %obj_tool_map with the pathnames to use: - foreach my $tool (keys %obj_tool_map) { - $obj_tool_map{$tool} = ConfigureTool($obj_tool_map{$tool}); - } -} - -# Returns the path of a caller-specified object tool. If --tools or -# PPROF_TOOLS are specified, then returns the full path to the tool -# with that prefix. Otherwise, returns the path unmodified (which -# means we will look for it on PATH). -sub ConfigureTool { - my $tool = shift; - my $path; - - # --tools (or $PPROF_TOOLS) is a comma separated list, where each - # item is either a) a pathname prefix, or b) a map of the form - # :. First we look for an entry of type (b) for our - # tool. If one is found, we use it. Otherwise, we consider all the - # pathname prefixes in turn, until one yields an existing file. If - # none does, we use a default path. - my $tools = $main::opt_tools || $ENV{"PPROF_TOOLS"} || ""; - if ($tools =~ m/(,|^)\Q$tool\E:([^,]*)/) { - $path = $2; - # TODO(csilvers): sanity-check that $path exists? Hard if it's relative. - } elsif ($tools ne '') { - foreach my $prefix (split(',', $tools)) { - next if ($prefix =~ /:/); # ignore "tool:fullpath" entries in the list - if (-x $prefix . $tool) { - $path = $prefix . $tool; - last; - } - } - if (!$path) { - error("No '$tool' found with prefix specified by " . - "--tools (or \$PPROF_TOOLS) '$tools'\n"); - } - } else { - # ... otherwise use the version that exists in the same directory as - # pprof. If there's nothing there, use $PATH. - $0 =~ m,[^/]*$,; # this is everything after the last slash - my $dirname = $`; # this is everything up to and including the last slash - if (-x "$dirname$tool") { - $path = "$dirname$tool"; - } else { - $path = $tool; - } - } - if ($main::opt_debug) { print STDERR "Using '$path' for '$tool'.\n"; } - return $path; -} - -sub cleanup { - unlink($main::tmpfile_sym); - unlink(keys %main::tempnames); - - # We leave any collected profiles in $HOME/pprof in case the user wants - # to look at them later. We print a message informing them of this. - if ((scalar(@main::profile_files) > 0) && - defined($main::collected_profile)) { - if (scalar(@main::profile_files) == 1) { - print STDERR "Dynamically gathered profile is in $main::collected_profile\n"; - } - print STDERR "If you want to investigate this profile further, you can do:\n"; - print STDERR "\n"; - print STDERR " pprof \\\n"; - print STDERR " $main::prog \\\n"; - print STDERR " $main::collected_profile\n"; - print STDERR "\n"; - } -} - -sub sighandler { - cleanup(); - exit(1); -} - -sub error { - my $msg = shift; - print STDERR $msg; - cleanup(); - exit(1); -} - - -# Run $nm_command and get all the resulting procedure boundaries whose -# names match "$regexp" and returns them in a hashtable mapping from -# procedure name to a two-element vector of [start address, end address] -sub GetProcedureBoundariesViaNm { - my $nm_command = shift; - my $regexp = shift; - - my $symbol_table = {}; - open(NM, "$nm_command |") || error("$nm_command: $!\n"); - my $last_start = "0"; - my $routine = ""; - while () { - s/\r//g; # turn windows-looking lines into unix-looking lines - if (m/^\s*([0-9a-f]+) (.) (..*)/) { - my $start_val = $1; - my $type = $2; - my $this_routine = $3; - - # It's possible for two symbols to share the same address, if - # one is a zero-length variable (like __start_google_malloc) or - # one symbol is a weak alias to another (like __libc_malloc). - # In such cases, we want to ignore all values except for the - # actual symbol, which in nm-speak has type "T". The logic - # below does this, though it's a bit tricky: what happens when - # we have a series of lines with the same address, is the first - # one gets queued up to be processed. However, it won't - # *actually* be processed until later, when we read a line with - # a different address. That means that as long as we're reading - # lines with the same address, we have a chance to replace that - # item in the queue, which we do whenever we see a 'T' entry -- - # that is, a line with type 'T'. If we never see a 'T' entry, - # we'll just go ahead and process the first entry (which never - # got touched in the queue), and ignore the others. - if ($start_val eq $last_start && $type =~ /t/i) { - # We are the 'T' symbol at this address, replace previous symbol. - $routine = $this_routine; - next; - } elsif ($start_val eq $last_start) { - # We're not the 'T' symbol at this address, so ignore us. - next; - } - - if ($this_routine eq $sep_symbol) { - $sep_address = HexExtend($start_val); - } - - # Tag this routine with the starting address in case the image - # has multiple occurrences of this routine. We use a syntax - # that resembles template paramters that are automatically - # stripped out by ShortFunctionName() - $this_routine .= "<$start_val>"; - - if (defined($routine) && $routine =~ m/$regexp/) { - $symbol_table->{$routine} = [HexExtend($last_start), - HexExtend($start_val)]; - } - $last_start = $start_val; - $routine = $this_routine; - } elsif (m/^Loaded image name: (.+)/) { - # The win32 nm workalike emits information about the binary it is using. - if ($main::opt_debug) { print STDERR "Using Image $1\n"; } - } elsif (m/^PDB file name: (.+)/) { - # The win32 nm workalike emits information about the pdb it is using. - if ($main::opt_debug) { print STDERR "Using PDB $1\n"; } - } - } - close(NM); - # Handle the last line in the nm output. Unfortunately, we don't know - # how big this last symbol is, because we don't know how big the file - # is. For now, we just give it a size of 0. - # TODO(csilvers): do better here. - if (defined($routine) && $routine =~ m/$regexp/) { - $symbol_table->{$routine} = [HexExtend($last_start), - HexExtend($last_start)]; - } - return $symbol_table; -} - -# Gets the procedure boundaries for all routines in "$image" whose names -# match "$regexp" and returns them in a hashtable mapping from procedure -# name to a two-element vector of [start address, end address]. -# Will return an empty map if nm is not installed or not working properly. -sub GetProcedureBoundaries { - my $image = shift; - my $regexp = shift; - - # For libc libraries, the copy in /usr/lib/debug contains debugging symbols - my $debugging = DebuggingLibrary($image); - if ($debugging) { - $image = $debugging; - } - - my $nm = $obj_tool_map{"nm"}; - my $cppfilt = $obj_tool_map{"c++filt"}; - - # nm can fail for two reasons: 1) $image isn't a debug library; 2) nm - # binary doesn't support --demangle. In addition, for OS X we need - # to use the -f flag to get 'flat' nm output (otherwise we don't sort - # properly and get incorrect results). Unfortunately, GNU nm uses -f - # in an incompatible way. So first we test whether our nm supports - # --demangle and -f. - my $demangle_flag = ""; - my $cppfilt_flag = ""; - if (system("$nm --demangle $image >/dev/null 2>&1") == 0) { - # In this mode, we do "nm --demangle " - $demangle_flag = "--demangle"; - $cppfilt_flag = ""; - } elsif (system("$cppfilt $image >/dev/null 2>&1") == 0) { - # In this mode, we do "nm | c++filt" - $cppfilt_flag = " | $cppfilt"; - }; - my $flatten_flag = ""; - if (system("$nm -f $image >/dev/null 2>&1") == 0) { - $flatten_flag = "-f"; - } - - # Finally, in the case $imagie isn't a debug library, we try again with - # -D to at least get *exported* symbols. If we can't use --demangle, - # we use c++filt instead, if it exists on this system. - my @nm_commands = ("$nm -n $flatten_flag $demangle_flag" . - " $image 2>/dev/null $cppfilt_flag", - "$nm -D -n $flatten_flag $demangle_flag" . - " $image 2>/dev/null $cppfilt_flag", - # 6nm is for Go binaries - "6nm $image 2>/dev/null | sort", - ); - - # If the executable is an MS Windows PDB-format executable, we'll - # have set up obj_tool_map("nm_pdb"). In this case, we actually - # want to use both unix nm and windows-specific nm_pdb, since - # PDB-format executables can apparently include dwarf .o files. - if (exists $obj_tool_map{"nm_pdb"}) { - my $nm_pdb = $obj_tool_map{"nm_pdb"}; - push(@nm_commands, "$nm_pdb --demangle $image 2>/dev/null"); - } - - foreach my $nm_command (@nm_commands) { - my $symbol_table = GetProcedureBoundariesViaNm($nm_command, $regexp); - return $symbol_table if (%{$symbol_table}); - } - my $symbol_table = {}; - return $symbol_table; -} - - -# The test vectors for AddressAdd/Sub/Inc are 8-16-nibble hex strings. -# To make them more readable, we add underscores at interesting places. -# This routine removes the underscores, producing the canonical representation -# used by pprof to represent addresses, particularly in the tested routines. -sub CanonicalHex { - my $arg = shift; - return join '', (split '_',$arg); -} - - -# Unit test for AddressAdd: -sub AddressAddUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressAddUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressAdd ($row->[0], $row->[1]); - if ($sum ne $row->[2]) { - printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, - $row->[0], $row->[1], $row->[2]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressAdd 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressAdd (CanonicalHex($row->[0]), CanonicalHex($row->[1])); - my $expected = join '', (split '_',$row->[2]); - if ($sum ne CanonicalHex($row->[2])) { - printf STDERR "ERROR: %s != %s + %s = %s\n", $sum, - $row->[0], $row->[1], $row->[2]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressAdd 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Unit test for AddressSub: -sub AddressSubUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressSubUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressSub ($row->[0], $row->[1]); - if ($sum ne $row->[3]) { - printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, - $row->[0], $row->[1], $row->[3]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressSub 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressSub (CanonicalHex($row->[0]), CanonicalHex($row->[1])); - if ($sum ne CanonicalHex($row->[3])) { - printf STDERR "ERROR: %s != %s - %s = %s\n", $sum, - $row->[0], $row->[1], $row->[3]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressSub 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Unit test for AddressInc: -sub AddressIncUnitTest { - my $test_data_8 = shift; - my $test_data_16 = shift; - my $error_count = 0; - my $fail_count = 0; - my $pass_count = 0; - # print STDERR "AddressIncUnitTest: ", 1+$#{$test_data_8}, " tests\n"; - - # First a few 8-nibble addresses. Note that this implementation uses - # plain old arithmetic, so a quick sanity check along with verifying what - # happens to overflow (we want it to wrap): - $address_length = 8; - foreach my $row (@{$test_data_8}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressInc ($row->[0]); - if ($sum ne $row->[4]) { - printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, - $row->[0], $row->[4]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressInc 32-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count = $fail_count; - $fail_count = 0; - $pass_count = 0; - - # Now 16-nibble addresses. - $address_length = 16; - foreach my $row (@{$test_data_16}) { - if ($main::opt_debug and $main::opt_test) { print STDERR "@{$row}\n"; } - my $sum = AddressInc (CanonicalHex($row->[0])); - if ($sum ne CanonicalHex($row->[4])) { - printf STDERR "ERROR: %s != %s + 1 = %s\n", $sum, - $row->[0], $row->[4]; - ++$fail_count; - } else { - ++$pass_count; - } - } - printf STDERR "AddressInc 64-bit tests: %d passes, %d failures\n", - $pass_count, $fail_count; - $error_count += $fail_count; - - return $error_count; -} - - -# Driver for unit tests. -# Currently just the address add/subtract/increment routines for 64-bit. -sub RunUnitTests { - my $error_count = 0; - - # This is a list of tuples [a, b, a+b, a-b, a+1] - my $unit_test_data_8 = [ - [qw(aaaaaaaa 50505050 fafafafa 5a5a5a5a aaaaaaab)], - [qw(50505050 aaaaaaaa fafafafa a5a5a5a6 50505051)], - [qw(ffffffff aaaaaaaa aaaaaaa9 55555555 00000000)], - [qw(00000001 ffffffff 00000000 00000002 00000002)], - [qw(00000001 fffffff0 fffffff1 00000011 00000002)], - ]; - my $unit_test_data_16 = [ - # The implementation handles data in 7-nibble chunks, so those are the - # interesting boundaries. - [qw(aaaaaaaa 50505050 - 00_000000f_afafafa 00_0000005_a5a5a5a 00_000000a_aaaaaab)], - [qw(50505050 aaaaaaaa - 00_000000f_afafafa ff_ffffffa_5a5a5a6 00_0000005_0505051)], - [qw(ffffffff aaaaaaaa - 00_000001a_aaaaaa9 00_0000005_5555555 00_0000010_0000000)], - [qw(00000001 ffffffff - 00_0000010_0000000 ff_ffffff0_0000002 00_0000000_0000002)], - [qw(00000001 fffffff0 - 00_000000f_ffffff1 ff_ffffff0_0000011 00_0000000_0000002)], - - [qw(00_a00000a_aaaaaaa 50505050 - 00_a00000f_afafafa 00_a000005_a5a5a5a 00_a00000a_aaaaaab)], - [qw(0f_fff0005_0505050 aaaaaaaa - 0f_fff000f_afafafa 0f_ffefffa_5a5a5a6 0f_fff0005_0505051)], - [qw(00_000000f_fffffff 01_800000a_aaaaaaa - 01_800001a_aaaaaa9 fe_8000005_5555555 00_0000010_0000000)], - [qw(00_0000000_0000001 ff_fffffff_fffffff - 00_0000000_0000000 00_0000000_0000002 00_0000000_0000002)], - [qw(00_0000000_0000001 ff_fffffff_ffffff0 - ff_fffffff_ffffff1 00_0000000_0000011 00_0000000_0000002)], - ]; - - $error_count += AddressAddUnitTest($unit_test_data_8, $unit_test_data_16); - $error_count += AddressSubUnitTest($unit_test_data_8, $unit_test_data_16); - $error_count += AddressIncUnitTest($unit_test_data_8, $unit_test_data_16); - if ($error_count > 0) { - print STDERR $error_count, " errors: FAILED\n"; - } else { - print STDERR "PASS\n"; - } - exit ($error_count); -} diff --git a/deps/jemalloc.orig/config.guess b/deps/jemalloc.orig/config.guess deleted file mode 100755 index 0773d0f6312..00000000000 --- a/deps/jemalloc.orig/config.guess +++ /dev/null @@ -1,1456 +0,0 @@ -#! /bin/sh -# Attempt to guess a canonical system name. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. - -timestamp='2004-03-03' - -# This file is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, but -# WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# Originally written by Per Bothner . -# Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. -# -# This script attempts to guess a canonical system name similar to -# config.sub. If it succeeds, it prints the system name on stdout, and -# exits with 0. Otherwise, it exits with 1. -# -# The plan is that this can be called by configure scripts if you -# don't specify an explicit build system type. - -me=`echo "$0" | sed -e 's,.*/,,'` - -usage="\ -Usage: $0 [OPTION] - -Output the configuration name of the system \`$me' is run on. - -Operation modes: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.guess ($timestamp) - -Originally written by Per Bothner. -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 -Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; - --version | -v ) - echo "$version" ; exit 0 ;; - --help | --h* | -h ) - echo "$usage"; exit 0 ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" >&2 - exit 1 ;; - * ) - break ;; - esac -done - -if test $# != 0; then - echo "$me: too many arguments$help" >&2 - exit 1 -fi - -trap 'exit 1' 1 2 15 - -# CC_FOR_BUILD -- compiler used by this script. Note that the use of a -# compiler to aid in system detection is discouraged as it requires -# temporary files to be created and, as you can see below, it is a -# headache to deal with in a portable fashion. - -# Historically, `CC_FOR_BUILD' used to be named `HOST_CC'. We still -# use `HOST_CC' if defined, but it is deprecated. - -# Portable tmp directory creation inspired by the Autoconf team. - -set_cc_for_build=' -trap "exitcode=\$?; (rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null) && exit \$exitcode" 0 ; -trap "rm -f \$tmpfiles 2>/dev/null; rmdir \$tmp 2>/dev/null; exit 1" 1 2 13 15 ; -: ${TMPDIR=/tmp} ; - { tmp=`(umask 077 && mktemp -d -q "$TMPDIR/cgXXXXXX") 2>/dev/null` && test -n "$tmp" && test -d "$tmp" ; } || - { test -n "$RANDOM" && tmp=$TMPDIR/cg$$-$RANDOM && (umask 077 && mkdir $tmp) ; } || - { tmp=$TMPDIR/cg-$$ && (umask 077 && mkdir $tmp) && echo "Warning: creating insecure temp directory" >&2 ; } || - { echo "$me: cannot create a temporary directory in $TMPDIR" >&2 ; exit 1 ; } ; -dummy=$tmp/dummy ; -tmpfiles="$dummy.c $dummy.o $dummy.rel $dummy" ; -case $CC_FOR_BUILD,$HOST_CC,$CC in - ,,) echo "int x;" > $dummy.c ; - for c in cc gcc c89 c99 ; do - if ($c -c -o $dummy.o $dummy.c) >/dev/null 2>&1 ; then - CC_FOR_BUILD="$c"; break ; - fi ; - done ; - if test x"$CC_FOR_BUILD" = x ; then - CC_FOR_BUILD=no_compiler_found ; - fi - ;; - ,,*) CC_FOR_BUILD=$CC ;; - ,*,*) CC_FOR_BUILD=$HOST_CC ;; -esac ;' - -# This is needed to find uname on a Pyramid OSx when run in the BSD universe. -# (ghazi@noc.rutgers.edu 1994-08-24) -if (test -f /.attbin/uname) >/dev/null 2>&1 ; then - PATH=$PATH:/.attbin ; export PATH -fi - -UNAME_MACHINE=`(uname -m) 2>/dev/null` || UNAME_MACHINE=unknown -UNAME_RELEASE=`(uname -r) 2>/dev/null` || UNAME_RELEASE=unknown -UNAME_SYSTEM=`(uname -s) 2>/dev/null` || UNAME_SYSTEM=unknown -UNAME_VERSION=`(uname -v) 2>/dev/null` || UNAME_VERSION=unknown - -# Note: order is significant - the case branches are not exclusive. - -case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in - *:NetBSD:*:*) - # NetBSD (nbsd) targets should (where applicable) match one or - # more of the tupples: *-*-netbsdelf*, *-*-netbsdaout*, - # *-*-netbsdecoff* and *-*-netbsd*. For targets that recently - # switched to ELF, *-*-netbsd* would select the old - # object file format. This provides both forward - # compatibility and a consistent mechanism for selecting the - # object file format. - # - # Note: NetBSD doesn't particularly care about the vendor - # portion of the name. We always set it to "unknown". - sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` - case "${UNAME_MACHINE_ARCH}" in - armeb) machine=armeb-unknown ;; - arm*) machine=arm-unknown ;; - sh3el) machine=shl-unknown ;; - sh3eb) machine=sh-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; - esac - # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. - case "${UNAME_MACHINE_ARCH}" in - arm*|i386|m68k|ns32k|sh3*|sparc|vax) - eval $set_cc_for_build - if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ - | grep __ELF__ >/dev/null - then - # Once all utilities can be ECOFF (netbsdecoff) or a.out (netbsdaout). - # Return netbsd for either. FIX? - os=netbsd - else - os=netbsdelf - fi - ;; - *) - os=netbsd - ;; - esac - # The OS release - # Debian GNU/NetBSD machines have a different userland, and - # thus, need a distinct triplet. However, they do not need - # kernel version information, so it can be replaced with a - # suitable tag, in the style of linux-gnu. - case "${UNAME_VERSION}" in - Debian*) - release='-gnu' - ;; - *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` - ;; - esac - # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: - # contains redundant information, the shorter form: - # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" - exit 0 ;; - amd64:OpenBSD:*:*) - echo x86_64-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - amiga:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - arc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - cats:OpenBSD:*:*) - echo arm-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - hp300:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mac68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - macppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme68k:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvme88k:OpenBSD:*:*) - echo m88k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - mvmeppc:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pegasos:OpenBSD:*:*) - echo powerpc-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - pmax:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sgi:OpenBSD:*:*) - echo mipseb-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - sun3:OpenBSD:*:*) - echo m68k-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - wgrisc:OpenBSD:*:*) - echo mipsel-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - *:OpenBSD:*:*) - echo ${UNAME_MACHINE}-unknown-openbsd${UNAME_RELEASE} - exit 0 ;; - *:ekkoBSD:*:*) - echo ${UNAME_MACHINE}-unknown-ekkobsd${UNAME_RELEASE} - exit 0 ;; - macppc:MirBSD:*:*) - echo powerppc-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; - *:MirBSD:*:*) - echo ${UNAME_MACHINE}-unknown-mirbsd${UNAME_RELEASE} - exit 0 ;; - alpha:OSF1:*:*) - case $UNAME_RELEASE in - *4.0) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $3}'` - ;; - *5.*) - UNAME_RELEASE=`/usr/sbin/sizer -v | awk '{print $4}'` - ;; - esac - # According to Compaq, /usr/sbin/psrinfo has been available on - # OSF/1 and Tru64 systems produced since 1995. I hope that - # covers most systems running today. This code pipes the CPU - # types through head -n 1, so we only detect the type of CPU 0. - ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` - case "$ALPHA_CPU_TYPE" in - "EV4 (21064)") - UNAME_MACHINE="alpha" ;; - "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; - "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; - "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; - "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; - "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; - "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; - "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; - "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; - "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; - "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; - "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; - "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; - "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; - "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; - esac - # A Pn.n version is a patched version. - # A Vn.n version is a released version. - # A Tn.n version is a released field test version. - # A Xn.n version is an unreleased experimental baselevel. - # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - exit 0 ;; - Alpha*:OpenVMS:*:*) - echo alpha-hp-vms - exit 0 ;; - Alpha\ *:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # Should we change UNAME_MACHINE based on the output of uname instead - # of the specific Alpha model? - echo alpha-pc-interix - exit 0 ;; - 21064:Windows_NT:50:3) - echo alpha-dec-winnt3.5 - exit 0 ;; - Amiga*:UNIX_System_V:4.0:*) - echo m68k-unknown-sysv4 - exit 0;; - *:[Aa]miga[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-amigaos - exit 0 ;; - *:[Mm]orph[Oo][Ss]:*:*) - echo ${UNAME_MACHINE}-unknown-morphos - exit 0 ;; - *:OS/390:*:*) - echo i370-ibm-openedition - exit 0 ;; - *:OS400:*:*) - echo powerpc-ibm-os400 - exit 0 ;; - arm:RISC*:1.[012]*:*|arm:riscix:1.[012]*:*) - echo arm-acorn-riscix${UNAME_RELEASE} - exit 0;; - SR2?01:HI-UX/MPP:*:* | SR8000:HI-UX/MPP:*:*) - echo hppa1.1-hitachi-hiuxmpp - exit 0;; - Pyramid*:OSx*:*:* | MIS*:OSx*:*:* | MIS*:SMP_DC-OSx*:*:*) - # akee@wpdis03.wpafb.af.mil (Earle F. Ake) contributed MIS and NILE. - if test "`(/bin/universe) 2>/dev/null`" = att ; then - echo pyramid-pyramid-sysv3 - else - echo pyramid-pyramid-bsd - fi - exit 0 ;; - NILE*:*:*:dcosx) - echo pyramid-pyramid-svr4 - exit 0 ;; - DRS?6000:unix:4.0:6*) - echo sparc-icl-nx6 - exit 0 ;; - DRS?6000:UNIX_SV:4.2*:7*) - case `/usr/bin/uname -p` in - sparc) echo sparc-icl-nx7 && exit 0 ;; - esac ;; - sun4H:SunOS:5.*:*) - echo sparc-hal-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - sun4*:SunOS:5.*:* | tadpole*:SunOS:5.*:*) - echo sparc-sun-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - i86pc:SunOS:5.*:*) - echo i386-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - sun4*:SunOS:6*:*) - # According to config.sub, this is the proper way to canonicalize - # SunOS6. Hard to guess exactly what SunOS6 will be like, but - # it's likely to be more like Solaris than SunOS4. - echo sparc-sun-solaris3`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - sun4*:SunOS:*:*) - case "`/usr/bin/arch -k`" in - Series*|S4*) - UNAME_RELEASE=`uname -v` - ;; - esac - # Japanese Language versions have a version number like `4.1.3-JL'. - echo sparc-sun-sunos`echo ${UNAME_RELEASE}|sed -e 's/-/_/'` - exit 0 ;; - sun3*:SunOS:*:*) - echo m68k-sun-sunos${UNAME_RELEASE} - exit 0 ;; - sun*:*:4.2BSD:*) - UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 - case "`/bin/arch`" in - sun3) - echo m68k-sun-sunos${UNAME_RELEASE} - ;; - sun4) - echo sparc-sun-sunos${UNAME_RELEASE} - ;; - esac - exit 0 ;; - aushp:SunOS:*:*) - echo sparc-auspex-sunos${UNAME_RELEASE} - exit 0 ;; - # The situation for MiNT is a little confusing. The machine name - # can be virtually everything (everything which is not - # "atarist" or "atariste" at least should have a processor - # > m68000). The system name ranges from "MiNT" over "FreeMiNT" - # to the lowercase version "mint" (or "freemint"). Finally - # the system name "TOS" denotes a system which is actually not - # MiNT. But MiNT is downward compatible to TOS, so this should - # be no problem. - atarist[e]:*MiNT:*:* | atarist[e]:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; - atari*:*MiNT:*:* | atari*:*mint:*:* | atarist[e]:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; - *falcon*:*MiNT:*:* | *falcon*:*mint:*:* | *falcon*:*TOS:*:*) - echo m68k-atari-mint${UNAME_RELEASE} - exit 0 ;; - milan*:*MiNT:*:* | milan*:*mint:*:* | *milan*:*TOS:*:*) - echo m68k-milan-mint${UNAME_RELEASE} - exit 0 ;; - hades*:*MiNT:*:* | hades*:*mint:*:* | *hades*:*TOS:*:*) - echo m68k-hades-mint${UNAME_RELEASE} - exit 0 ;; - *:*MiNT:*:* | *:*mint:*:* | *:*TOS:*:*) - echo m68k-unknown-mint${UNAME_RELEASE} - exit 0 ;; - m68k:machten:*:*) - echo m68k-apple-machten${UNAME_RELEASE} - exit 0 ;; - powerpc:machten:*:*) - echo powerpc-apple-machten${UNAME_RELEASE} - exit 0 ;; - RISC*:Mach:*:*) - echo mips-dec-mach_bsd4.3 - exit 0 ;; - RISC*:ULTRIX:*:*) - echo mips-dec-ultrix${UNAME_RELEASE} - exit 0 ;; - VAX*:ULTRIX*:*:*) - echo vax-dec-ultrix${UNAME_RELEASE} - exit 0 ;; - 2020:CLIX:*:* | 2430:CLIX:*:*) - echo clipper-intergraph-clix${UNAME_RELEASE} - exit 0 ;; - mips:*:*:UMIPS | mips:*:*:RISCos) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c -#ifdef __cplusplus -#include /* for printf() prototype */ - int main (int argc, char *argv[]) { -#else - int main (argc, argv) int argc; char *argv[]; { -#endif - #if defined (host_mips) && defined (MIPSEB) - #if defined (SYSTYPE_SYSV) - printf ("mips-mips-riscos%ssysv\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_SVR4) - printf ("mips-mips-riscos%ssvr4\n", argv[1]); exit (0); - #endif - #if defined (SYSTYPE_BSD43) || defined(SYSTYPE_BSD) - printf ("mips-mips-riscos%sbsd\n", argv[1]); exit (0); - #endif - #endif - exit (-1); - } -EOF - $CC_FOR_BUILD -o $dummy $dummy.c \ - && $dummy `echo "${UNAME_RELEASE}" | sed -n 's/\([0-9]*\).*/\1/p'` \ - && exit 0 - echo mips-mips-riscos${UNAME_RELEASE} - exit 0 ;; - Motorola:PowerMAX_OS:*:*) - echo powerpc-motorola-powermax - exit 0 ;; - Motorola:*:4.3:PL8-*) - echo powerpc-harris-powermax - exit 0 ;; - Night_Hawk:*:*:PowerMAX_OS | Synergy:PowerMAX_OS:*:*) - echo powerpc-harris-powermax - exit 0 ;; - Night_Hawk:Power_UNIX:*:*) - echo powerpc-harris-powerunix - exit 0 ;; - m88k:CX/UX:7*:*) - echo m88k-harris-cxux7 - exit 0 ;; - m88k:*:4*:R4*) - echo m88k-motorola-sysv4 - exit 0 ;; - m88k:*:3*:R3*) - echo m88k-motorola-sysv3 - exit 0 ;; - AViiON:dgux:*:*) - # DG/UX returns AViiON for all architectures - UNAME_PROCESSOR=`/usr/bin/uname -p` - if [ $UNAME_PROCESSOR = mc88100 ] || [ $UNAME_PROCESSOR = mc88110 ] - then - if [ ${TARGET_BINARY_INTERFACE}x = m88kdguxelfx ] || \ - [ ${TARGET_BINARY_INTERFACE}x = x ] - then - echo m88k-dg-dgux${UNAME_RELEASE} - else - echo m88k-dg-dguxbcs${UNAME_RELEASE} - fi - else - echo i586-dg-dgux${UNAME_RELEASE} - fi - exit 0 ;; - M88*:DolphinOS:*:*) # DolphinOS (SVR3) - echo m88k-dolphin-sysv3 - exit 0 ;; - M88*:*:R3*:*) - # Delta 88k system running SVR3 - echo m88k-motorola-sysv3 - exit 0 ;; - XD88*:*:*:*) # Tektronix XD88 system running UTekV (SVR3) - echo m88k-tektronix-sysv3 - exit 0 ;; - Tek43[0-9][0-9]:UTek:*:*) # Tektronix 4300 system running UTek (BSD) - echo m68k-tektronix-bsd - exit 0 ;; - *:IRIX*:*:*) - echo mips-sgi-irix`echo ${UNAME_RELEASE}|sed -e 's/-/_/g'` - exit 0 ;; - ????????:AIX?:[12].1:2) # AIX 2.2.1 or AIX 2.1.1 is RT/PC AIX. - echo romp-ibm-aix # uname -m gives an 8 hex-code CPU id - exit 0 ;; # Note that: echo "'`uname -s`'" gives 'AIX ' - i*86:AIX:*:*) - echo i386-ibm-aix - exit 0 ;; - ia64:AIX:*:*) - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` - else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} - fi - echo ${UNAME_MACHINE}-ibm-aix${IBM_REV} - exit 0 ;; - *:AIX:2:3) - if grep bos325 /usr/include/stdio.h >/dev/null 2>&1; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - - main() - { - if (!__power_pc()) - exit(1); - puts("powerpc-ibm-aix3.2.5"); - exit(0); - } -EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 - echo rs6000-ibm-aix3.2.5 - elif grep bos324 /usr/include/stdio.h >/dev/null 2>&1; then - echo rs6000-ibm-aix3.2.4 - else - echo rs6000-ibm-aix3.2 - fi - exit 0 ;; - *:AIX:*:[45]) - IBM_CPU_ID=`/usr/sbin/lsdev -C -c processor -S available | sed 1q | awk '{ print $1 }'` - if /usr/sbin/lsattr -El ${IBM_CPU_ID} | grep ' POWER' >/dev/null 2>&1; then - IBM_ARCH=rs6000 - else - IBM_ARCH=powerpc - fi - if [ -x /usr/bin/oslevel ] ; then - IBM_REV=`/usr/bin/oslevel` - else - IBM_REV=${UNAME_VERSION}.${UNAME_RELEASE} - fi - echo ${IBM_ARCH}-ibm-aix${IBM_REV} - exit 0 ;; - *:AIX:*:*) - echo rs6000-ibm-aix - exit 0 ;; - ibmrt:4.4BSD:*|romp-ibm:BSD:*) - echo romp-ibm-bsd4.4 - exit 0 ;; - ibmrt:*BSD:*|romp-ibm:BSD:*) # covers RT/PC BSD and - echo romp-ibm-bsd${UNAME_RELEASE} # 4.3 with uname added to - exit 0 ;; # report: romp-ibm BSD 4.3 - *:BOSX:*:*) - echo rs6000-bull-bosx - exit 0 ;; - DPX/2?00:B.O.S.:*:*) - echo m68k-bull-sysv3 - exit 0 ;; - 9000/[34]??:4.3bsd:1.*:*) - echo m68k-hp-bsd - exit 0 ;; - hp300:4.4BSD:*:* | 9000/[34]??:4.3bsd:2.*:*) - echo m68k-hp-bsd4.4 - exit 0 ;; - 9000/[34678]??:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - case "${UNAME_MACHINE}" in - 9000/31? ) HP_ARCH=m68000 ;; - 9000/[34]?? ) HP_ARCH=m68k ;; - 9000/[678][0-9][0-9]) - if [ -x /usr/bin/getconf ]; then - sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` - sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` - case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 - 532) # CPU_PA_RISC2_0 - case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 - esac ;; - esac - fi - if [ "${HP_ARCH}" = "" ]; then - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - - #define _HPUX_SOURCE - #include - #include - - int main () - { - #if defined(_SC_KERNEL_BITS) - long bits = sysconf(_SC_KERNEL_BITS); - #endif - long cpu = sysconf (_SC_CPU_VERSION); - - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1"); break; - case CPU_PA_RISC2_0: - #if defined(_SC_KERNEL_BITS) - switch (bits) - { - case 64: puts ("hppa2.0w"); break; - case 32: puts ("hppa2.0n"); break; - default: puts ("hppa2.0"); break; - } break; - #else /* !defined(_SC_KERNEL_BITS) */ - puts ("hppa2.0"); break; - #endif - default: puts ("hppa1.0"); break; - } - exit (0); - } -EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` - test -z "$HP_ARCH" && HP_ARCH=hppa - fi ;; - esac - if [ ${HP_ARCH} = "hppa2.0w" ] - then - # avoid double evaluation of $set_cc_for_build - test -n "$CC_FOR_BUILD" || eval $set_cc_for_build - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E -) | grep __LP64__ >/dev/null - then - HP_ARCH="hppa2.0w" - else - HP_ARCH="hppa64" - fi - fi - echo ${HP_ARCH}-hp-hpux${HPUX_REV} - exit 0 ;; - ia64:HP-UX:*:*) - HPUX_REV=`echo ${UNAME_RELEASE}|sed -e 's/[^.]*.[0B]*//'` - echo ia64-hp-hpux${HPUX_REV} - exit 0 ;; - 3050*:HI-UX:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - int - main () - { - long cpu = sysconf (_SC_CPU_VERSION); - /* The order matters, because CPU_IS_HP_MC68K erroneously returns - true for CPU_PA_RISC1_0. CPU_IS_PA_RISC returns correct - results, however. */ - if (CPU_IS_PA_RISC (cpu)) - { - switch (cpu) - { - case CPU_PA_RISC1_0: puts ("hppa1.0-hitachi-hiuxwe2"); break; - case CPU_PA_RISC1_1: puts ("hppa1.1-hitachi-hiuxwe2"); break; - case CPU_PA_RISC2_0: puts ("hppa2.0-hitachi-hiuxwe2"); break; - default: puts ("hppa-hitachi-hiuxwe2"); break; - } - } - else if (CPU_IS_HP_MC68K (cpu)) - puts ("m68k-hitachi-hiuxwe2"); - else puts ("unknown-hitachi-hiuxwe2"); - exit (0); - } -EOF - $CC_FOR_BUILD -o $dummy $dummy.c && $dummy && exit 0 - echo unknown-hitachi-hiuxwe2 - exit 0 ;; - 9000/7??:4.3bsd:*:* | 9000/8?[79]:4.3bsd:*:* ) - echo hppa1.1-hp-bsd - exit 0 ;; - 9000/8??:4.3bsd:*:*) - echo hppa1.0-hp-bsd - exit 0 ;; - *9??*:MPE/iX:*:* | *3000*:MPE/iX:*:*) - echo hppa1.0-hp-mpeix - exit 0 ;; - hp7??:OSF1:*:* | hp8?[79]:OSF1:*:* ) - echo hppa1.1-hp-osf - exit 0 ;; - hp8??:OSF1:*:*) - echo hppa1.0-hp-osf - exit 0 ;; - i*86:OSF1:*:*) - if [ -x /usr/sbin/sysversion ] ; then - echo ${UNAME_MACHINE}-unknown-osf1mk - else - echo ${UNAME_MACHINE}-unknown-osf1 - fi - exit 0 ;; - parisc*:Lites*:*:*) - echo hppa1.1-hp-lites - exit 0 ;; - C1*:ConvexOS:*:* | convex:ConvexOS:C1*:*) - echo c1-convex-bsd - exit 0 ;; - C2*:ConvexOS:*:* | convex:ConvexOS:C2*:*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit 0 ;; - C34*:ConvexOS:*:* | convex:ConvexOS:C34*:*) - echo c34-convex-bsd - exit 0 ;; - C38*:ConvexOS:*:* | convex:ConvexOS:C38*:*) - echo c38-convex-bsd - exit 0 ;; - C4*:ConvexOS:*:* | convex:ConvexOS:C4*:*) - echo c4-convex-bsd - exit 0 ;; - CRAY*Y-MP:*:*:*) - echo ymp-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; - CRAY*[A-Z]90:*:*:*) - echo ${UNAME_MACHINE}-cray-unicos${UNAME_RELEASE} \ - | sed -e 's/CRAY.*\([A-Z]90\)/\1/' \ - -e y/ABCDEFGHIJKLMNOPQRSTUVWXYZ/abcdefghijklmnopqrstuvwxyz/ \ - -e 's/\.[^.]*$/.X/' - exit 0 ;; - CRAY*TS:*:*:*) - echo t90-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; - CRAY*T3E:*:*:*) - echo alphaev5-cray-unicosmk${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; - CRAY*SV1:*:*:*) - echo sv1-cray-unicos${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; - *:UNICOS/mp:*:*) - echo nv1-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' - exit 0 ;; - F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` - echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; - 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` - echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" - exit 0 ;; - i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) - echo ${UNAME_MACHINE}-pc-bsdi${UNAME_RELEASE} - exit 0 ;; - sparc*:BSD/OS:*:*) - echo sparc-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; - *:BSD/OS:*:*) - echo ${UNAME_MACHINE}-unknown-bsdi${UNAME_RELEASE} - exit 0 ;; - *:FreeBSD:*:*) - # Determine whether the default compiler uses glibc. - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #if __GLIBC__ >= 2 - LIBC=gnu - #else - LIBC= - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - # GNU/KFreeBSD systems have a "k" prefix to indicate we are using - # FreeBSD's kernel, but not the complete OS. - case ${LIBC} in gnu) kernel_only='k' ;; esac - echo ${UNAME_MACHINE}-unknown-${kernel_only}freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`${LIBC:+-$LIBC} - exit 0 ;; - i*:CYGWIN*:*) - echo ${UNAME_MACHINE}-pc-cygwin - exit 0 ;; - i*:MINGW*:*) - echo ${UNAME_MACHINE}-pc-mingw32 - exit 0 ;; - i*:PW*:*) - echo ${UNAME_MACHINE}-pc-pw32 - exit 0 ;; - x86:Interix*:[34]*) - echo i586-pc-interix${UNAME_RELEASE}|sed -e 's/\..*//' - exit 0 ;; - [345]86:Windows_95:* | [345]86:Windows_98:* | [345]86:Windows_NT:*) - echo i${UNAME_MACHINE}-pc-mks - exit 0 ;; - i*:Windows_NT*:* | Pentium*:Windows_NT*:*) - # How do we know it's Interix rather than the generic POSIX subsystem? - # It also conflicts with pre-2.0 versions of AT&T UWIN. Should we - # UNAME_MACHINE based on the output of uname instead of i386? - echo i586-pc-interix - exit 0 ;; - i*:UWIN*:*) - echo ${UNAME_MACHINE}-pc-uwin - exit 0 ;; - p*:CYGWIN*:*) - echo powerpcle-unknown-cygwin - exit 0 ;; - prep*:SunOS:5.*:*) - echo powerpcle-unknown-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` - exit 0 ;; - *:GNU:*:*) - # the GNU system - echo `echo ${UNAME_MACHINE}|sed -e 's,[-/].*$,,'`-unknown-gnu`echo ${UNAME_RELEASE}|sed -e 's,/.*$,,'` - exit 0 ;; - *:GNU/*:*:*) - # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-unknown-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-gnu - exit 0 ;; - i*86:Minix:*:*) - echo ${UNAME_MACHINE}-pc-minix - exit 0 ;; - arm*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - cris:Linux:*:*) - echo cris-axis-linux-gnu - exit 0 ;; - ia64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - m68*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - mips:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #undef CPU - #undef mips - #undef mipsel - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mipsel - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips - #else - CPU= - #endif - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 - ;; - mips64:Linux:*:*) - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #undef CPU - #undef mips64 - #undef mips64el - #if defined(__MIPSEL__) || defined(__MIPSEL) || defined(_MIPSEL) || defined(MIPSEL) - CPU=mips64el - #else - #if defined(__MIPSEB__) || defined(__MIPSEB) || defined(_MIPSEB) || defined(MIPSEB) - CPU=mips64 - #else - CPU= - #endif - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^CPU=` - test x"${CPU}" != x && echo "${CPU}-unknown-linux-gnu" && exit 0 - ;; - ppc:Linux:*:*) - echo powerpc-unknown-linux-gnu - exit 0 ;; - ppc64:Linux:*:*) - echo powerpc64-unknown-linux-gnu - exit 0 ;; - alpha:Linux:*:*) - case `sed -n '/^cpu model/s/^.*: \(.*\)/\1/p' < /proc/cpuinfo` in - EV5) UNAME_MACHINE=alphaev5 ;; - EV56) UNAME_MACHINE=alphaev56 ;; - PCA56) UNAME_MACHINE=alphapca56 ;; - PCA57) UNAME_MACHINE=alphapca56 ;; - EV6) UNAME_MACHINE=alphaev6 ;; - EV67) UNAME_MACHINE=alphaev67 ;; - EV68*) UNAME_MACHINE=alphaev68 ;; - esac - objdump --private-headers /bin/sh | grep ld.so.1 >/dev/null - if test "$?" = 0 ; then LIBC="libc1" ; else LIBC="" ; fi - echo ${UNAME_MACHINE}-unknown-linux-gnu${LIBC} - exit 0 ;; - parisc:Linux:*:* | hppa:Linux:*:*) - # Look for CPU level - case `grep '^cpu[^a-z]*:' /proc/cpuinfo 2>/dev/null | cut -d' ' -f2` in - PA7*) echo hppa1.1-unknown-linux-gnu ;; - PA8*) echo hppa2.0-unknown-linux-gnu ;; - *) echo hppa-unknown-linux-gnu ;; - esac - exit 0 ;; - parisc64:Linux:*:* | hppa64:Linux:*:*) - echo hppa64-unknown-linux-gnu - exit 0 ;; - s390:Linux:*:* | s390x:Linux:*:*) - echo ${UNAME_MACHINE}-ibm-linux - exit 0 ;; - sh64*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - sh*:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - sparc:Linux:*:* | sparc64:Linux:*:*) - echo ${UNAME_MACHINE}-unknown-linux-gnu - exit 0 ;; - x86_64:Linux:*:*) - echo x86_64-unknown-linux-gnu - exit 0 ;; - i*86:Linux:*:*) - # The BFD linker knows what the default object file format is, so - # first see if it will tell us. cd to the root directory to prevent - # problems with other programs or directories called `ld' in the path. - # Set LC_ALL=C to ensure ld outputs messages in English. - ld_supported_targets=`cd /; LC_ALL=C ld --help 2>&1 \ - | sed -ne '/supported targets:/!d - s/[ ][ ]*/ /g - s/.*supported targets: *// - s/ .*// - p'` - case "$ld_supported_targets" in - elf32-i386) - TENTATIVE="${UNAME_MACHINE}-pc-linux-gnu" - ;; - a.out-i386-linux) - echo "${UNAME_MACHINE}-pc-linux-gnuaout" - exit 0 ;; - coff-i386) - echo "${UNAME_MACHINE}-pc-linux-gnucoff" - exit 0 ;; - "") - # Either a pre-BFD a.out linker (linux-gnuoldld) or - # one that does not give us useful --help. - echo "${UNAME_MACHINE}-pc-linux-gnuoldld" - exit 0 ;; - esac - # Determine whether the default compiler is a.out or elf - eval $set_cc_for_build - sed 's/^ //' << EOF >$dummy.c - #include - #ifdef __ELF__ - # ifdef __GLIBC__ - # if __GLIBC__ >= 2 - LIBC=gnu - # else - LIBC=gnulibc1 - # endif - # else - LIBC=gnulibc1 - # endif - #else - #ifdef __INTEL_COMPILER - LIBC=gnu - #else - LIBC=gnuaout - #endif - #endif - #ifdef __dietlibc__ - LIBC=dietlibc - #endif -EOF - eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep ^LIBC=` - test x"${LIBC}" != x && echo "${UNAME_MACHINE}-pc-linux-${LIBC}" && exit 0 - test x"${TENTATIVE}" != x && echo "${TENTATIVE}" && exit 0 - ;; - i*86:DYNIX/ptx:4*:*) - # ptx 4.0 does uname -s correctly, with DYNIX/ptx in there. - # earlier versions are messed up and put the nodename in both - # sysname and nodename. - echo i386-sequent-sysv4 - exit 0 ;; - i*86:UNIX_SV:4.2MP:2.*) - # Unixware is an offshoot of SVR4, but it has its own version - # number series starting with 2... - # I am not positive that other SVR4 systems won't match this, - # I just have to hope. -- rms. - # Use sysv4.2uw... so that sysv4* matches it. - echo ${UNAME_MACHINE}-pc-sysv4.2uw${UNAME_VERSION} - exit 0 ;; - i*86:OS/2:*:*) - # If we were able to find `uname', then EMX Unix compatibility - # is probably installed. - echo ${UNAME_MACHINE}-pc-os2-emx - exit 0 ;; - i*86:XTS-300:*:STOP) - echo ${UNAME_MACHINE}-unknown-stop - exit 0 ;; - i*86:atheos:*:*) - echo ${UNAME_MACHINE}-unknown-atheos - exit 0 ;; - i*86:syllable:*:*) - echo ${UNAME_MACHINE}-pc-syllable - exit 0 ;; - i*86:LynxOS:2.*:* | i*86:LynxOS:3.[01]*:* | i*86:LynxOS:4.0*:*) - echo i386-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - i*86:*DOS:*:*) - echo ${UNAME_MACHINE}-pc-msdosdjgpp - exit 0 ;; - i*86:*:4.*:* | i*86:SYSTEM_V:4.*:*) - UNAME_REL=`echo ${UNAME_RELEASE} | sed 's/\/MP$//'` - if grep Novell /usr/include/link.h >/dev/null 2>/dev/null; then - echo ${UNAME_MACHINE}-univel-sysv${UNAME_REL} - else - echo ${UNAME_MACHINE}-pc-sysv${UNAME_REL} - fi - exit 0 ;; - i*86:*:5:[78]*) - case `/bin/uname -X | grep "^Machine"` in - *486*) UNAME_MACHINE=i486 ;; - *Pentium) UNAME_MACHINE=i586 ;; - *Pent*|*Celeron) UNAME_MACHINE=i686 ;; - esac - echo ${UNAME_MACHINE}-unknown-sysv${UNAME_RELEASE}${UNAME_SYSTEM}${UNAME_VERSION} - exit 0 ;; - i*86:*:3.2:*) - if test -f /usr/options/cb.name; then - UNAME_REL=`sed -n 's/.*Version //p' /dev/null >/dev/null ; then - UNAME_REL=`(/bin/uname -X|grep Release|sed -e 's/.*= //')` - (/bin/uname -X|grep i80486 >/dev/null) && UNAME_MACHINE=i486 - (/bin/uname -X|grep '^Machine.*Pentium' >/dev/null) \ - && UNAME_MACHINE=i586 - (/bin/uname -X|grep '^Machine.*Pent *II' >/dev/null) \ - && UNAME_MACHINE=i686 - (/bin/uname -X|grep '^Machine.*Pentium Pro' >/dev/null) \ - && UNAME_MACHINE=i686 - echo ${UNAME_MACHINE}-pc-sco$UNAME_REL - else - echo ${UNAME_MACHINE}-pc-sysv32 - fi - exit 0 ;; - pc:*:*:*) - # Left here for compatibility: - # uname -m prints for DJGPP always 'pc', but it prints nothing about - # the processor, so we play safe by assuming i386. - echo i386-pc-msdosdjgpp - exit 0 ;; - Intel:Mach:3*:*) - echo i386-pc-mach3 - exit 0 ;; - paragon:*:*:*) - echo i860-intel-osf1 - exit 0 ;; - i860:*:4.*:*) # i860-SVR4 - if grep Stardent /usr/include/sys/uadmin.h >/dev/null 2>&1 ; then - echo i860-stardent-sysv${UNAME_RELEASE} # Stardent Vistra i860-SVR4 - else # Add other i860-SVR4 vendors below as they are discovered. - echo i860-unknown-sysv${UNAME_RELEASE} # Unknown i860-SVR4 - fi - exit 0 ;; - mini*:CTIX:SYS*5:*) - # "miniframe" - echo m68010-convergent-sysv - exit 0 ;; - mc68k:UNIX:SYSTEM5:3.51m) - echo m68k-convergent-sysv - exit 0 ;; - M680?0:D-NIX:5.3:*) - echo m68k-diab-dnix - exit 0 ;; - M68*:*:R3V[567]*:*) - test -r /sysV68 && echo 'm68k-motorola-sysv' && exit 0 ;; - 3[345]??:*:4.0:3.0 | 3[34]??A:*:4.0:3.0 | 3[34]??,*:*:4.0:3.0 | 3[34]??/*:*:4.0:3.0 | 4400:*:4.0:3.0 | 4850:*:4.0:3.0 | SKA40:*:4.0:3.0 | SDS2:*:4.0:3.0 | SHG2:*:4.0:3.0) - OS_REL='' - test -r /etc/.relid \ - && OS_REL=.`sed -n 's/[^ ]* [^ ]* \([0-9][0-9]\).*/\1/p' < /etc/.relid` - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4.3${OS_REL} && exit 0 - /bin/uname -p 2>/dev/null | /bin/grep entium >/dev/null \ - && echo i586-ncr-sysv4.3${OS_REL} && exit 0 ;; - 3[34]??:*:4.0:* | 3[34]??,*:*:4.0:*) - /bin/uname -p 2>/dev/null | grep 86 >/dev/null \ - && echo i486-ncr-sysv4 && exit 0 ;; - m68*:LynxOS:2.*:* | m68*:LynxOS:3.0*:*) - echo m68k-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - mc68030:UNIX_System_V:4.*:*) - echo m68k-atari-sysv4 - exit 0 ;; - TSUNAMI:LynxOS:2.*:*) - echo sparc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - rs6000:LynxOS:2.*:*) - echo rs6000-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - PowerPC:LynxOS:2.*:* | PowerPC:LynxOS:3.[01]*:* | PowerPC:LynxOS:4.0*:*) - echo powerpc-unknown-lynxos${UNAME_RELEASE} - exit 0 ;; - SM[BE]S:UNIX_SV:*:*) - echo mips-dde-sysv${UNAME_RELEASE} - exit 0 ;; - RM*:ReliantUNIX-*:*:*) - echo mips-sni-sysv4 - exit 0 ;; - RM*:SINIX-*:*:*) - echo mips-sni-sysv4 - exit 0 ;; - *:SINIX-*:*:*) - if uname -p 2>/dev/null >/dev/null ; then - UNAME_MACHINE=`(uname -p) 2>/dev/null` - echo ${UNAME_MACHINE}-sni-sysv4 - else - echo ns32k-sni-sysv - fi - exit 0 ;; - PENTIUM:*:4.0*:*) # Unisys `ClearPath HMP IX 4000' SVR4/MP effort - # says - echo i586-unisys-sysv4 - exit 0 ;; - *:UNIX_System_V:4*:FTX*) - # From Gerald Hewes . - # How about differentiating between stratus architectures? -djm - echo hppa1.1-stratus-sysv4 - exit 0 ;; - *:*:*:FTX*) - # From seanf@swdc.stratus.com. - echo i860-stratus-sysv4 - exit 0 ;; - *:VOS:*:*) - # From Paul.Green@stratus.com. - echo hppa1.1-stratus-vos - exit 0 ;; - mc68*:A/UX:*:*) - echo m68k-apple-aux${UNAME_RELEASE} - exit 0 ;; - news*:NEWS-OS:6*:*) - echo mips-sony-newsos6 - exit 0 ;; - R[34]000:*System_V*:*:* | R4000:UNIX_SYSV:*:* | R*000:UNIX_SV:*:*) - if [ -d /usr/nec ]; then - echo mips-nec-sysv${UNAME_RELEASE} - else - echo mips-unknown-sysv${UNAME_RELEASE} - fi - exit 0 ;; - BeBox:BeOS:*:*) # BeOS running on hardware made by Be, PPC only. - echo powerpc-be-beos - exit 0 ;; - BeMac:BeOS:*:*) # BeOS running on Mac or Mac clone, PPC only. - echo powerpc-apple-beos - exit 0 ;; - BePC:BeOS:*:*) # BeOS running on Intel PC compatible. - echo i586-pc-beos - exit 0 ;; - SX-4:SUPER-UX:*:*) - echo sx4-nec-superux${UNAME_RELEASE} - exit 0 ;; - SX-5:SUPER-UX:*:*) - echo sx5-nec-superux${UNAME_RELEASE} - exit 0 ;; - SX-6:SUPER-UX:*:*) - echo sx6-nec-superux${UNAME_RELEASE} - exit 0 ;; - Power*:Rhapsody:*:*) - echo powerpc-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; - *:Rhapsody:*:*) - echo ${UNAME_MACHINE}-apple-rhapsody${UNAME_RELEASE} - exit 0 ;; - *:Darwin:*:*) - case `uname -p` in - *86) UNAME_PROCESSOR=i686 ;; - powerpc) UNAME_PROCESSOR=powerpc ;; - esac - echo ${UNAME_PROCESSOR}-apple-darwin${UNAME_RELEASE} - exit 0 ;; - *:procnto*:*:* | *:QNX:[0123456789]*:*) - UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then - UNAME_PROCESSOR=i386 - UNAME_MACHINE=pc - fi - echo ${UNAME_PROCESSOR}-${UNAME_MACHINE}-nto-qnx${UNAME_RELEASE} - exit 0 ;; - *:QNX:*:4*) - echo i386-pc-qnx - exit 0 ;; - NSR-?:NONSTOP_KERNEL:*:*) - echo nsr-tandem-nsk${UNAME_RELEASE} - exit 0 ;; - *:NonStop-UX:*:*) - echo mips-compaq-nonstopux - exit 0 ;; - BS2000:POSIX*:*:*) - echo bs2000-siemens-sysv - exit 0 ;; - DS/*:UNIX_System_V:*:*) - echo ${UNAME_MACHINE}-${UNAME_SYSTEM}-${UNAME_RELEASE} - exit 0 ;; - *:Plan9:*:*) - # "uname -m" is not consistent, so use $cputype instead. 386 - # is converted to i386 for consistency with other x86 - # operating systems. - if test "$cputype" = "386"; then - UNAME_MACHINE=i386 - else - UNAME_MACHINE="$cputype" - fi - echo ${UNAME_MACHINE}-unknown-plan9 - exit 0 ;; - *:TOPS-10:*:*) - echo pdp10-unknown-tops10 - exit 0 ;; - *:TENEX:*:*) - echo pdp10-unknown-tenex - exit 0 ;; - KS10:TOPS-20:*:* | KL10:TOPS-20:*:* | TYPE4:TOPS-20:*:*) - echo pdp10-dec-tops20 - exit 0 ;; - XKL-1:TOPS-20:*:* | TYPE5:TOPS-20:*:*) - echo pdp10-xkl-tops20 - exit 0 ;; - *:TOPS-20:*:*) - echo pdp10-unknown-tops20 - exit 0 ;; - *:ITS:*:*) - echo pdp10-unknown-its - exit 0 ;; - SEI:*:*:SEIUX) - echo mips-sei-seiux${UNAME_RELEASE} - exit 0 ;; - *:DragonFly:*:*) - echo ${UNAME_MACHINE}-unknown-dragonfly`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` - exit 0 ;; -esac - -#echo '(No uname command or uname output not recognized.)' 1>&2 -#echo "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" 1>&2 - -eval $set_cc_for_build -cat >$dummy.c < -# include -#endif -main () -{ -#if defined (sony) -#if defined (MIPSEB) - /* BFD wants "bsd" instead of "newsos". Perhaps BFD should be changed, - I don't know.... */ - printf ("mips-sony-bsd\n"); exit (0); -#else -#include - printf ("m68k-sony-newsos%s\n", -#ifdef NEWSOS4 - "4" -#else - "" -#endif - ); exit (0); -#endif -#endif - -#if defined (__arm) && defined (__acorn) && defined (__unix) - printf ("arm-acorn-riscix"); exit (0); -#endif - -#if defined (hp300) && !defined (hpux) - printf ("m68k-hp-bsd\n"); exit (0); -#endif - -#if defined (NeXT) -#if !defined (__ARCHITECTURE__) -#define __ARCHITECTURE__ "m68k" -#endif - int version; - version=`(hostinfo | sed -n 's/.*NeXT Mach \([0-9]*\).*/\1/p') 2>/dev/null`; - if (version < 4) - printf ("%s-next-nextstep%d\n", __ARCHITECTURE__, version); - else - printf ("%s-next-openstep%d\n", __ARCHITECTURE__, version); - exit (0); -#endif - -#if defined (MULTIMAX) || defined (n16) -#if defined (UMAXV) - printf ("ns32k-encore-sysv\n"); exit (0); -#else -#if defined (CMU) - printf ("ns32k-encore-mach\n"); exit (0); -#else - printf ("ns32k-encore-bsd\n"); exit (0); -#endif -#endif -#endif - -#if defined (__386BSD__) - printf ("i386-pc-bsd\n"); exit (0); -#endif - -#if defined (sequent) -#if defined (i386) - printf ("i386-sequent-dynix\n"); exit (0); -#endif -#if defined (ns32000) - printf ("ns32k-sequent-dynix\n"); exit (0); -#endif -#endif - -#if defined (_SEQUENT_) - struct utsname un; - - uname(&un); - - if (strncmp(un.version, "V2", 2) == 0) { - printf ("i386-sequent-ptx2\n"); exit (0); - } - if (strncmp(un.version, "V1", 2) == 0) { /* XXX is V1 correct? */ - printf ("i386-sequent-ptx1\n"); exit (0); - } - printf ("i386-sequent-ptx\n"); exit (0); - -#endif - -#if defined (vax) -# if !defined (ultrix) -# include -# if defined (BSD) -# if BSD == 43 - printf ("vax-dec-bsd4.3\n"); exit (0); -# else -# if BSD == 199006 - printf ("vax-dec-bsd4.3reno\n"); exit (0); -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# endif -# else - printf ("vax-dec-bsd\n"); exit (0); -# endif -# else - printf ("vax-dec-ultrix\n"); exit (0); -# endif -#endif - -#if defined (alliant) && defined (i860) - printf ("i860-alliant-bsd\n"); exit (0); -#endif - - exit (1); -} -EOF - -$CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null && $dummy && exit 0 - -# Apollos put the system type in the environment. - -test -d /usr/apollo && { echo ${ISP}-apollo-${SYSTYPE}; exit 0; } - -# Convex versions that predate uname can use getsysinfo(1) - -if [ -x /usr/convex/getsysinfo ] -then - case `getsysinfo -f cpu_type` in - c1*) - echo c1-convex-bsd - exit 0 ;; - c2*) - if getsysinfo -f scalar_acc - then echo c32-convex-bsd - else echo c2-convex-bsd - fi - exit 0 ;; - c34*) - echo c34-convex-bsd - exit 0 ;; - c38*) - echo c38-convex-bsd - exit 0 ;; - c4*) - echo c4-convex-bsd - exit 0 ;; - esac -fi - -cat >&2 < in order to provide the needed -information to handle your system. - -config.guess timestamp = $timestamp - -uname -m = `(uname -m) 2>/dev/null || echo unknown` -uname -r = `(uname -r) 2>/dev/null || echo unknown` -uname -s = `(uname -s) 2>/dev/null || echo unknown` -uname -v = `(uname -v) 2>/dev/null || echo unknown` - -/usr/bin/uname -p = `(/usr/bin/uname -p) 2>/dev/null` -/bin/uname -X = `(/bin/uname -X) 2>/dev/null` - -hostinfo = `(hostinfo) 2>/dev/null` -/bin/universe = `(/bin/universe) 2>/dev/null` -/usr/bin/arch -k = `(/usr/bin/arch -k) 2>/dev/null` -/bin/arch = `(/bin/arch) 2>/dev/null` -/usr/bin/oslevel = `(/usr/bin/oslevel) 2>/dev/null` -/usr/convex/getsysinfo = `(/usr/convex/getsysinfo) 2>/dev/null` - -UNAME_MACHINE = ${UNAME_MACHINE} -UNAME_RELEASE = ${UNAME_RELEASE} -UNAME_SYSTEM = ${UNAME_SYSTEM} -UNAME_VERSION = ${UNAME_VERSION} -EOF - -exit 1 - -# Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: diff --git a/deps/jemalloc.orig/config.stamp.in b/deps/jemalloc.orig/config.stamp.in deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/deps/jemalloc.orig/config.sub b/deps/jemalloc.orig/config.sub deleted file mode 100755 index 264f820aa55..00000000000 --- a/deps/jemalloc.orig/config.sub +++ /dev/null @@ -1,1549 +0,0 @@ -#! /bin/sh -# Configuration validation subroutine script. -# Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, -# 2000, 2001, 2002, 2003 Free Software Foundation, Inc. - -timestamp='2004-02-23' - -# This file is (in principle) common to ALL GNU software. -# The presence of a machine in this file suggests that SOME GNU software -# can handle that machine. It does not imply ALL GNU software can. -# -# This file is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -# As a special exception to the GNU General Public License, if you -# distribute this file as part of a program that contains a -# configuration script generated by Autoconf, you may include it under -# the same distribution terms that you use for the rest of that program. - -# Please send patches to . Submit a context -# diff and a properly formatted ChangeLog entry. -# -# Configuration subroutine to validate and canonicalize a configuration type. -# Supply the specified configuration type as an argument. -# If it is invalid, we print an error message on stderr and exit with code 1. -# Otherwise, we print the canonical config type on stdout and succeed. - -# This file is supposed to be the same for all GNU packages -# and recognize all the CPU types, system types and aliases -# that are meaningful with *any* GNU software. -# Each package is responsible for reporting which valid configurations -# it does not support. The user should be able to distinguish -# a failure to support a valid configuration from a meaningless -# configuration. - -# The goal of this file is to map all the various variations of a given -# machine specification into a single specification in the form: -# CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM -# or in some cases, the newer four-part form: -# CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM -# It is wrong to echo any other type of specification. - -me=`echo "$0" | sed -e 's,.*/,,'` - -usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS - -Canonicalize a configuration name. - -Operation modes: - -h, --help print this help, then exit - -t, --time-stamp print date of last modification, then exit - -v, --version print version number, then exit - -Report bugs and patches to ." - -version="\ -GNU config.sub ($timestamp) - -Copyright (C) 1992, 1993, 1994, 1995, 1996, 1997, 1998, 1999, 2000, 2001 -Free Software Foundation, Inc. - -This is free software; see the source for copying conditions. There is NO -warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." - -help=" -Try \`$me --help' for more information." - -# Parse command line -while test $# -gt 0 ; do - case $1 in - --time-stamp | --time* | -t ) - echo "$timestamp" ; exit 0 ;; - --version | -v ) - echo "$version" ; exit 0 ;; - --help | --h* | -h ) - echo "$usage"; exit 0 ;; - -- ) # Stop option processing - shift; break ;; - - ) # Use stdin as input. - break ;; - -* ) - echo "$me: invalid option $1$help" - exit 1 ;; - - *local*) - # First pass through any local machine types. - echo $1 - exit 0;; - - * ) - break ;; - esac -done - -case $# in - 0) echo "$me: missing argument$help" >&2 - exit 1;; - 1) ;; - *) echo "$me: too many arguments$help" >&2 - exit 1;; -esac - -# Separate what the user gave into CPU-COMPANY and OS or KERNEL-OS (if any). -# Here we must recognize all the valid KERNEL-OS combinations. -maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` -case $maybe_os in - nto-qnx* | linux-gnu* | linux-dietlibc | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | \ - kfreebsd*-gnu* | knetbsd*-gnu* | netbsd*-gnu* | storm-chaos* | os2-emx* | rtmk-nova*) - os=-$maybe_os - basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` - ;; - *) - basic_machine=`echo $1 | sed 's/-[^-]*$//'` - if [ $basic_machine != $1 ] - then os=`echo $1 | sed 's/.*-/-/'` - else os=; fi - ;; -esac - -### Let's recognize common machines as not being operating systems so -### that things like config.sub decstation-3100 work. We also -### recognize some manufacturers as not being operating systems, so we -### can provide default operating systems below. -case $os in - -sun*os*) - # Prevent following clause from handling this invalid input. - ;; - -dec* | -mips* | -sequent* | -encore* | -pc532* | -sgi* | -sony* | \ - -att* | -7300* | -3300* | -delta* | -motorola* | -sun[234]* | \ - -unicom* | -ibm* | -next | -hp | -isi* | -apollo | -altos* | \ - -convergent* | -ncr* | -news | -32* | -3600* | -3100* | -hitachi* |\ - -c[123]* | -convex* | -sun | -crds | -omron* | -dg | -ultra | -tti* | \ - -harris | -dolphin | -highlevel | -gould | -cbm | -ns | -masscomp | \ - -apple | -axis) - os= - basic_machine=$1 - ;; - -sim | -cisco | -oki | -wec | -winbond) - os= - basic_machine=$1 - ;; - -scout) - ;; - -wrs) - os=-vxworks - basic_machine=$1 - ;; - -chorusos*) - os=-chorusos - basic_machine=$1 - ;; - -chorusrdb) - os=-chorusrdb - basic_machine=$1 - ;; - -hiux*) - os=-hiuxwe2 - ;; - -sco5) - os=-sco3.2v5 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco4) - os=-sco3.2v4 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2.[4-9]*) - os=`echo $os | sed -e 's/sco3.2./sco3.2v/'` - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco3.2v[4-9]*) - # Don't forget version if it is 3.2v4 or newer. - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -sco*) - os=-sco3.2v2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -udk*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -isc) - os=-isc2.2 - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -clix*) - basic_machine=clipper-intergraph - ;; - -isc*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-pc/'` - ;; - -lynx*) - os=-lynxos - ;; - -ptx*) - basic_machine=`echo $1 | sed -e 's/86-.*/86-sequent/'` - ;; - -windowsnt*) - os=`echo $os | sed -e 's/windowsnt/winnt/'` - ;; - -psos*) - os=-psos - ;; - -mint | -mint[0-9]*) - basic_machine=m68k-atari - os=-mint - ;; -esac - -# Decode aliases for certain CPU-COMPANY combinations. -case $basic_machine in - # Recognize the basic CPU types without company name. - # Some are omitted here because they have special meanings below. - 1750a | 580 \ - | a29k \ - | alpha | alphaev[4-8] | alphaev56 | alphaev6[78] | alphapca5[67] \ - | alpha64 | alpha64ev[4-8] | alpha64ev56 | alpha64ev6[78] | alpha64pca5[67] \ - | am33_2.0 \ - | arc | arm | arm[bl]e | arme[lb] | armv[2345] | armv[345][lb] | avr \ - | c4x | clipper \ - | d10v | d30v | dlx | dsp16xx \ - | fr30 | frv \ - | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ - | i370 | i860 | i960 | ia64 \ - | ip2k | iq2000 \ - | m32r | m68000 | m68k | m88k | mcore \ - | mips | mipsbe | mipseb | mipsel | mipsle \ - | mips16 \ - | mips64 | mips64el \ - | mips64vr | mips64vrel \ - | mips64orion | mips64orionel \ - | mips64vr4100 | mips64vr4100el \ - | mips64vr4300 | mips64vr4300el \ - | mips64vr5000 | mips64vr5000el \ - | mipsisa32 | mipsisa32el \ - | mipsisa32r2 | mipsisa32r2el \ - | mipsisa64 | mipsisa64el \ - | mipsisa64r2 | mipsisa64r2el \ - | mipsisa64sb1 | mipsisa64sb1el \ - | mipsisa64sr71k | mipsisa64sr71kel \ - | mipstx39 | mipstx39el \ - | mn10200 | mn10300 \ - | msp430 \ - | ns16k | ns32k \ - | openrisc | or32 \ - | pdp10 | pdp11 | pj | pjl \ - | powerpc | powerpc64 | powerpc64le | powerpcle | ppcbe \ - | pyramid \ - | sh | sh[1234] | sh[23]e | sh[34]eb | shbe | shle | sh[1234]le | sh3ele \ - | sh64 | sh64le \ - | sparc | sparc64 | sparc86x | sparclet | sparclite | sparcv9 | sparcv9b \ - | strongarm \ - | tahoe | thumb | tic4x | tic80 | tron \ - | v850 | v850e \ - | we32k \ - | x86 | xscale | xstormy16 | xtensa \ - | z8k) - basic_machine=$basic_machine-unknown - ;; - m6811 | m68hc11 | m6812 | m68hc12) - # Motorola 68HC11/12. - basic_machine=$basic_machine-unknown - os=-none - ;; - m88110 | m680[12346]0 | m683?2 | m68360 | m5200 | v70 | w65 | z8k) - ;; - - # We use `pc' rather than `unknown' - # because (1) that's what they normally are, and - # (2) the word "unknown" tends to confuse beginning users. - i*86 | x86_64) - basic_machine=$basic_machine-pc - ;; - # Object if more than one company name word. - *-*-*) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 - ;; - # Recognize the basic CPU types with company name. - 580-* \ - | a29k-* \ - | alpha-* | alphaev[4-8]-* | alphaev56-* | alphaev6[78]-* \ - | alpha64-* | alpha64ev[4-8]-* | alpha64ev56-* | alpha64ev6[78]-* \ - | alphapca5[67]-* | alpha64pca5[67]-* | arc-* \ - | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ - | avr-* \ - | bs2000-* \ - | c[123]* | c30-* | [cjt]90-* | c4x-* | c54x-* | c55x-* | c6x-* \ - | clipper-* | cydra-* \ - | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ - | f30[01]-* | f700-* | fr30-* | frv-* | fx80-* \ - | h8300-* | h8500-* \ - | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ - | i*86-* | i860-* | i960-* | ia64-* \ - | ip2k-* | iq2000-* \ - | m32r-* \ - | m68000-* | m680[012346]0-* | m68360-* | m683?2-* | m68k-* \ - | m88110-* | m88k-* | mcore-* \ - | mips-* | mipsbe-* | mipseb-* | mipsel-* | mipsle-* \ - | mips16-* \ - | mips64-* | mips64el-* \ - | mips64vr-* | mips64vrel-* \ - | mips64orion-* | mips64orionel-* \ - | mips64vr4100-* | mips64vr4100el-* \ - | mips64vr4300-* | mips64vr4300el-* \ - | mips64vr5000-* | mips64vr5000el-* \ - | mipsisa32-* | mipsisa32el-* \ - | mipsisa32r2-* | mipsisa32r2el-* \ - | mipsisa64-* | mipsisa64el-* \ - | mipsisa64r2-* | mipsisa64r2el-* \ - | mipsisa64sb1-* | mipsisa64sb1el-* \ - | mipsisa64sr71k-* | mipsisa64sr71kel-* \ - | mipstx39-* | mipstx39el-* \ - | msp430-* \ - | none-* | np1-* | nv1-* | ns16k-* | ns32k-* \ - | orion-* \ - | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ - | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* | ppcbe-* \ - | pyramid-* \ - | romp-* | rs6000-* \ - | sh-* | sh[1234]-* | sh[23]e-* | sh[34]eb-* | shbe-* \ - | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ - | sparc-* | sparc64-* | sparc86x-* | sparclet-* | sparclite-* \ - | sparcv9-* | sparcv9b-* | strongarm-* | sv1-* | sx?-* \ - | tahoe-* | thumb-* \ - | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ - | tron-* \ - | v850-* | v850e-* | vax-* \ - | we32k-* \ - | x86-* | x86_64-* | xps100-* | xscale-* | xstormy16-* \ - | xtensa-* \ - | ymp-* \ - | z8k-*) - ;; - # Recognize the various machine names and aliases which stand - # for a CPU type and a company and sometimes even an OS. - 386bsd) - basic_machine=i386-unknown - os=-bsd - ;; - 3b1 | 7300 | 7300-att | att-7300 | pc7300 | safari | unixpc) - basic_machine=m68000-att - ;; - 3b*) - basic_machine=we32k-att - ;; - a29khif) - basic_machine=a29k-amd - os=-udi - ;; - abacus) - basic_machine=abacus-unknown - ;; - adobe68k) - basic_machine=m68010-adobe - os=-scout - ;; - alliant | fx80) - basic_machine=fx80-alliant - ;; - altos | altos3068) - basic_machine=m68k-altos - ;; - am29k) - basic_machine=a29k-none - os=-bsd - ;; - amd64) - basic_machine=x86_64-pc - ;; - amd64-*) - basic_machine=x86_64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - amdahl) - basic_machine=580-amdahl - os=-sysv - ;; - amiga | amiga-*) - basic_machine=m68k-unknown - ;; - amigaos | amigados) - basic_machine=m68k-unknown - os=-amigaos - ;; - amigaunix | amix) - basic_machine=m68k-unknown - os=-sysv4 - ;; - apollo68) - basic_machine=m68k-apollo - os=-sysv - ;; - apollo68bsd) - basic_machine=m68k-apollo - os=-bsd - ;; - aux) - basic_machine=m68k-apple - os=-aux - ;; - balance) - basic_machine=ns32k-sequent - os=-dynix - ;; - c90) - basic_machine=c90-cray - os=-unicos - ;; - convex-c1) - basic_machine=c1-convex - os=-bsd - ;; - convex-c2) - basic_machine=c2-convex - os=-bsd - ;; - convex-c32) - basic_machine=c32-convex - os=-bsd - ;; - convex-c34) - basic_machine=c34-convex - os=-bsd - ;; - convex-c38) - basic_machine=c38-convex - os=-bsd - ;; - cray | j90) - basic_machine=j90-cray - os=-unicos - ;; - cr16c) - basic_machine=cr16c-unknown - os=-elf - ;; - crds | unos) - basic_machine=m68k-crds - ;; - cris | cris-* | etrax*) - basic_machine=cris-axis - ;; - crx) - basic_machine=crx-unknown - os=-elf - ;; - da30 | da30-*) - basic_machine=m68k-da30 - ;; - decstation | decstation-3100 | pmax | pmax-* | pmin | dec3100 | decstatn) - basic_machine=mips-dec - ;; - decsystem10* | dec10*) - basic_machine=pdp10-dec - os=-tops10 - ;; - decsystem20* | dec20*) - basic_machine=pdp10-dec - os=-tops20 - ;; - delta | 3300 | motorola-3300 | motorola-delta \ - | 3300-motorola | delta-motorola) - basic_machine=m68k-motorola - ;; - delta88) - basic_machine=m88k-motorola - os=-sysv3 - ;; - dpx20 | dpx20-*) - basic_machine=rs6000-bull - os=-bosx - ;; - dpx2* | dpx2*-bull) - basic_machine=m68k-bull - os=-sysv3 - ;; - ebmon29k) - basic_machine=a29k-amd - os=-ebmon - ;; - elxsi) - basic_machine=elxsi-elxsi - os=-bsd - ;; - encore | umax | mmax) - basic_machine=ns32k-encore - ;; - es1800 | OSE68k | ose68k | ose | OSE) - basic_machine=m68k-ericsson - os=-ose - ;; - fx2800) - basic_machine=i860-alliant - ;; - genix) - basic_machine=ns32k-ns - ;; - gmicro) - basic_machine=tron-gmicro - os=-sysv - ;; - go32) - basic_machine=i386-pc - os=-go32 - ;; - h3050r* | hiux*) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - h8300hms) - basic_machine=h8300-hitachi - os=-hms - ;; - h8300xray) - basic_machine=h8300-hitachi - os=-xray - ;; - h8500hms) - basic_machine=h8500-hitachi - os=-hms - ;; - harris) - basic_machine=m88k-harris - os=-sysv3 - ;; - hp300-*) - basic_machine=m68k-hp - ;; - hp300bsd) - basic_machine=m68k-hp - os=-bsd - ;; - hp300hpux) - basic_machine=m68k-hp - os=-hpux - ;; - hp3k9[0-9][0-9] | hp9[0-9][0-9]) - basic_machine=hppa1.0-hp - ;; - hp9k2[0-9][0-9] | hp9k31[0-9]) - basic_machine=m68000-hp - ;; - hp9k3[2-9][0-9]) - basic_machine=m68k-hp - ;; - hp9k6[0-9][0-9] | hp6[0-9][0-9]) - basic_machine=hppa1.0-hp - ;; - hp9k7[0-79][0-9] | hp7[0-79][0-9]) - basic_machine=hppa1.1-hp - ;; - hp9k78[0-9] | hp78[0-9]) - # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp - ;; - hp9k8[67]1 | hp8[67]1 | hp9k80[24] | hp80[24] | hp9k8[78]9 | hp8[78]9 | hp9k893 | hp893) - # FIXME: really hppa2.0-hp - basic_machine=hppa1.1-hp - ;; - hp9k8[0-9][13679] | hp8[0-9][13679]) - basic_machine=hppa1.1-hp - ;; - hp9k8[0-9][0-9] | hp8[0-9][0-9]) - basic_machine=hppa1.0-hp - ;; - hppa-next) - os=-nextstep3 - ;; - hppaosf) - basic_machine=hppa1.1-hp - os=-osf - ;; - hppro) - basic_machine=hppa1.1-hp - os=-proelf - ;; - i370-ibm* | ibm*) - basic_machine=i370-ibm - ;; -# I'm not sure what "Sysv32" means. Should this be sysv3.2? - i*86v32) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv32 - ;; - i*86v4*) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv4 - ;; - i*86v) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-sysv - ;; - i*86sol2) - basic_machine=`echo $1 | sed -e 's/86.*/86-pc/'` - os=-solaris2 - ;; - i386mach) - basic_machine=i386-mach - os=-mach - ;; - i386-vsta | vsta) - basic_machine=i386-unknown - os=-vsta - ;; - iris | iris4d) - basic_machine=mips-sgi - case $os in - -irix*) - ;; - *) - os=-irix4 - ;; - esac - ;; - isi68 | isi) - basic_machine=m68k-isi - os=-sysv - ;; - m88k-omron*) - basic_machine=m88k-omron - ;; - magnum | m3230) - basic_machine=mips-mips - os=-sysv - ;; - merlin) - basic_machine=ns32k-utek - os=-sysv - ;; - mingw32) - basic_machine=i386-pc - os=-mingw32 - ;; - miniframe) - basic_machine=m68000-convergent - ;; - *mint | -mint[0-9]* | *MiNT | *MiNT[0-9]*) - basic_machine=m68k-atari - os=-mint - ;; - mips3*-*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'` - ;; - mips3*) - basic_machine=`echo $basic_machine | sed -e 's/mips3/mips64/'`-unknown - ;; - mmix*) - basic_machine=mmix-knuth - os=-mmixware - ;; - monitor) - basic_machine=m68k-rom68k - os=-coff - ;; - morphos) - basic_machine=powerpc-unknown - os=-morphos - ;; - msdos) - basic_machine=i386-pc - os=-msdos - ;; - mvs) - basic_machine=i370-ibm - os=-mvs - ;; - ncr3000) - basic_machine=i486-ncr - os=-sysv4 - ;; - netbsd386) - basic_machine=i386-unknown - os=-netbsd - ;; - netwinder) - basic_machine=armv4l-rebel - os=-linux - ;; - news | news700 | news800 | news900) - basic_machine=m68k-sony - os=-newsos - ;; - news1000) - basic_machine=m68030-sony - os=-newsos - ;; - news-3600 | risc-news) - basic_machine=mips-sony - os=-newsos - ;; - necv70) - basic_machine=v70-nec - os=-sysv - ;; - next | m*-next ) - basic_machine=m68k-next - case $os in - -nextstep* ) - ;; - -ns2*) - os=-nextstep2 - ;; - *) - os=-nextstep3 - ;; - esac - ;; - nh3000) - basic_machine=m68k-harris - os=-cxux - ;; - nh[45]000) - basic_machine=m88k-harris - os=-cxux - ;; - nindy960) - basic_machine=i960-intel - os=-nindy - ;; - mon960) - basic_machine=i960-intel - os=-mon960 - ;; - nonstopux) - basic_machine=mips-compaq - os=-nonstopux - ;; - np1) - basic_machine=np1-gould - ;; - nv1) - basic_machine=nv1-cray - os=-unicosmp - ;; - nsr-tandem) - basic_machine=nsr-tandem - ;; - op50n-* | op60c-*) - basic_machine=hppa1.1-oki - os=-proelf - ;; - or32 | or32-*) - basic_machine=or32-unknown - os=-coff - ;; - os400) - basic_machine=powerpc-ibm - os=-os400 - ;; - OSE68000 | ose68000) - basic_machine=m68000-ericsson - os=-ose - ;; - os68k) - basic_machine=m68k-none - os=-os68k - ;; - pa-hitachi) - basic_machine=hppa1.1-hitachi - os=-hiuxwe2 - ;; - paragon) - basic_machine=i860-intel - os=-osf - ;; - pbd) - basic_machine=sparc-tti - ;; - pbb) - basic_machine=m68k-tti - ;; - pc532 | pc532-*) - basic_machine=ns32k-pc532 - ;; - pentium | p5 | k5 | k6 | nexgen | viac3) - basic_machine=i586-pc - ;; - pentiumpro | p6 | 6x86 | athlon | athlon_*) - basic_machine=i686-pc - ;; - pentiumii | pentium2 | pentiumiii | pentium3) - basic_machine=i686-pc - ;; - pentium4) - basic_machine=i786-pc - ;; - pentium-* | p5-* | k5-* | k6-* | nexgen-* | viac3-*) - basic_machine=i586-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentiumpro-* | p6-* | 6x86-* | athlon-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentiumii-* | pentium2-* | pentiumiii-* | pentium3-*) - basic_machine=i686-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pentium4-*) - basic_machine=i786-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - pn) - basic_machine=pn-gould - ;; - power) basic_machine=power-ibm - ;; - ppc) basic_machine=powerpc-unknown - ;; - ppc-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppcle | powerpclittle | ppc-le | powerpc-little) - basic_machine=powerpcle-unknown - ;; - ppcle-* | powerpclittle-*) - basic_machine=powerpcle-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppc64) basic_machine=powerpc64-unknown - ;; - ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) - basic_machine=powerpc64le-unknown - ;; - ppc64le-* | powerpc64little-*) - basic_machine=powerpc64le-`echo $basic_machine | sed 's/^[^-]*-//'` - ;; - ps2) - basic_machine=i386-ibm - ;; - pw32) - basic_machine=i586-unknown - os=-pw32 - ;; - rom68k) - basic_machine=m68k-rom68k - os=-coff - ;; - rm[46]00) - basic_machine=mips-siemens - ;; - rtpc | rtpc-*) - basic_machine=romp-ibm - ;; - s390 | s390-*) - basic_machine=s390-ibm - ;; - s390x | s390x-*) - basic_machine=s390x-ibm - ;; - sa29200) - basic_machine=a29k-amd - os=-udi - ;; - sb1) - basic_machine=mipsisa64sb1-unknown - ;; - sb1el) - basic_machine=mipsisa64sb1el-unknown - ;; - sei) - basic_machine=mips-sei - os=-seiux - ;; - sequent) - basic_machine=i386-sequent - ;; - sh) - basic_machine=sh-hitachi - os=-hms - ;; - sh64) - basic_machine=sh64-unknown - ;; - sparclite-wrs | simso-wrs) - basic_machine=sparclite-wrs - os=-vxworks - ;; - sps7) - basic_machine=m68k-bull - os=-sysv2 - ;; - spur) - basic_machine=spur-unknown - ;; - st2000) - basic_machine=m68k-tandem - ;; - stratus) - basic_machine=i860-stratus - os=-sysv4 - ;; - sun2) - basic_machine=m68000-sun - ;; - sun2os3) - basic_machine=m68000-sun - os=-sunos3 - ;; - sun2os4) - basic_machine=m68000-sun - os=-sunos4 - ;; - sun3os3) - basic_machine=m68k-sun - os=-sunos3 - ;; - sun3os4) - basic_machine=m68k-sun - os=-sunos4 - ;; - sun4os3) - basic_machine=sparc-sun - os=-sunos3 - ;; - sun4os4) - basic_machine=sparc-sun - os=-sunos4 - ;; - sun4sol2) - basic_machine=sparc-sun - os=-solaris2 - ;; - sun3 | sun3-*) - basic_machine=m68k-sun - ;; - sun4) - basic_machine=sparc-sun - ;; - sun386 | sun386i | roadrunner) - basic_machine=i386-sun - ;; - sv1) - basic_machine=sv1-cray - os=-unicos - ;; - symmetry) - basic_machine=i386-sequent - os=-dynix - ;; - t3e) - basic_machine=alphaev5-cray - os=-unicos - ;; - t90) - basic_machine=t90-cray - os=-unicos - ;; - tic54x | c54x*) - basic_machine=tic54x-unknown - os=-coff - ;; - tic55x | c55x*) - basic_machine=tic55x-unknown - os=-coff - ;; - tic6x | c6x*) - basic_machine=tic6x-unknown - os=-coff - ;; - tx39) - basic_machine=mipstx39-unknown - ;; - tx39el) - basic_machine=mipstx39el-unknown - ;; - toad1) - basic_machine=pdp10-xkl - os=-tops20 - ;; - tower | tower-32) - basic_machine=m68k-ncr - ;; - tpf) - basic_machine=s390x-ibm - os=-tpf - ;; - udi29k) - basic_machine=a29k-amd - os=-udi - ;; - ultra3) - basic_machine=a29k-nyu - os=-sym1 - ;; - v810 | necv810) - basic_machine=v810-nec - os=-none - ;; - vaxv) - basic_machine=vax-dec - os=-sysv - ;; - vms) - basic_machine=vax-dec - os=-vms - ;; - vpp*|vx|vx-*) - basic_machine=f301-fujitsu - ;; - vxworks960) - basic_machine=i960-wrs - os=-vxworks - ;; - vxworks68) - basic_machine=m68k-wrs - os=-vxworks - ;; - vxworks29k) - basic_machine=a29k-wrs - os=-vxworks - ;; - w65*) - basic_machine=w65-wdc - os=-none - ;; - w89k-*) - basic_machine=hppa1.1-winbond - os=-proelf - ;; - xps | xps100) - basic_machine=xps100-honeywell - ;; - ymp) - basic_machine=ymp-cray - os=-unicos - ;; - z8k-*-coff) - basic_machine=z8k-unknown - os=-sim - ;; - none) - basic_machine=none-none - os=-none - ;; - -# Here we handle the default manufacturer of certain CPU types. It is in -# some cases the only manufacturer, in others, it is the most popular. - w89k) - basic_machine=hppa1.1-winbond - ;; - op50n) - basic_machine=hppa1.1-oki - ;; - op60c) - basic_machine=hppa1.1-oki - ;; - romp) - basic_machine=romp-ibm - ;; - rs6000) - basic_machine=rs6000-ibm - ;; - vax) - basic_machine=vax-dec - ;; - pdp10) - # there are many clones, so DEC is not a safe bet - basic_machine=pdp10-unknown - ;; - pdp11) - basic_machine=pdp11-dec - ;; - we32k) - basic_machine=we32k-att - ;; - sh3 | sh4 | sh[34]eb | sh[1234]le | sh[23]ele) - basic_machine=sh-unknown - ;; - sh64) - basic_machine=sh64-unknown - ;; - sparc | sparcv9 | sparcv9b) - basic_machine=sparc-sun - ;; - cydra) - basic_machine=cydra-cydrome - ;; - orion) - basic_machine=orion-highlevel - ;; - orion105) - basic_machine=clipper-highlevel - ;; - mac | mpw | mac-mpw) - basic_machine=m68k-apple - ;; - pmac | pmac-mpw) - basic_machine=powerpc-apple - ;; - *-unknown) - # Make sure to match an already-canonicalized machine name. - ;; - *) - echo Invalid configuration \`$1\': machine \`$basic_machine\' not recognized 1>&2 - exit 1 - ;; -esac - -# Here we canonicalize certain aliases for manufacturers. -case $basic_machine in - *-digital*) - basic_machine=`echo $basic_machine | sed 's/digital.*/dec/'` - ;; - *-commodore*) - basic_machine=`echo $basic_machine | sed 's/commodore.*/cbm/'` - ;; - *) - ;; -esac - -# Decode manufacturer-specific aliases for certain operating systems. - -if [ x"$os" != x"" ] -then -case $os in - # First match some system type aliases - # that might get confused with valid system types. - # -solaris* is a basic system type, with this one exception. - -solaris1 | -solaris1.*) - os=`echo $os | sed -e 's|solaris1|sunos4|'` - ;; - -solaris) - os=-solaris2 - ;; - -svr4*) - os=-sysv4 - ;; - -unixware*) - os=-sysv4.2uw - ;; - -gnu/linux*) - os=`echo $os | sed -e 's|gnu/linux|linux-gnu|'` - ;; - # First accept the basic system types. - # The portable systems comes first. - # Each alternative MUST END IN A *, to match a version number. - # -sysv* is not here because it comes later, after sysvr4. - -gnu* | -bsd* | -mach* | -minix* | -genix* | -ultrix* | -irix* \ - | -*vms* | -sco* | -esix* | -isc* | -aix* | -sunos | -sunos[34]*\ - | -hpux* | -unos* | -osf* | -luna* | -dgux* | -solaris* | -sym* \ - | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* \ - | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ - | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ - | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* | -openbsd* \ - | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ - | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ - | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ - | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* \ - | -cygwin* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -linux-gnu* | -linux-uclibc* | -uxpv* | -beos* | -mpeix* | -udk* \ - | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ - | -openstep* | -oskit* | -conix* | -pw32* | -nonstopux* \ - | -storm-chaos* | -tops10* | -tenex* | -tops20* | -its* \ - | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ - | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ - | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly*) - # Remember, each alternative MUST END IN *, to match a version number. - ;; - -qnx*) - case $basic_machine in - x86-* | i*86-*) - ;; - *) - os=-nto$os - ;; - esac - ;; - -nto-qnx*) - ;; - -nto*) - os=`echo $os | sed -e 's|nto|nto-qnx|'` - ;; - -sim | -es1800* | -hms* | -xray | -os68k* | -none* | -v88r* \ - | -windows* | -osx | -abug | -netware* | -os9* | -beos* \ - | -macos* | -mpw* | -magic* | -mmixware* | -mon960* | -lnews*) - ;; - -mac*) - os=`echo $os | sed -e 's|mac|macos|'` - ;; - -linux-dietlibc) - os=-linux-dietlibc - ;; - -linux*) - os=`echo $os | sed -e 's|linux|linux-gnu|'` - ;; - -sunos5*) - os=`echo $os | sed -e 's|sunos5|solaris2|'` - ;; - -sunos6*) - os=`echo $os | sed -e 's|sunos6|solaris3|'` - ;; - -opened*) - os=-openedition - ;; - -os400*) - os=-os400 - ;; - -wince*) - os=-wince - ;; - -osfrose*) - os=-osfrose - ;; - -osf*) - os=-osf - ;; - -utek*) - os=-bsd - ;; - -dynix*) - os=-bsd - ;; - -acis*) - os=-aos - ;; - -atheos*) - os=-atheos - ;; - -syllable*) - os=-syllable - ;; - -386bsd) - os=-bsd - ;; - -ctix* | -uts*) - os=-sysv - ;; - -nova*) - os=-rtmk-nova - ;; - -ns2 ) - os=-nextstep2 - ;; - -nsk*) - os=-nsk - ;; - # Preserve the version number of sinix5. - -sinix5.*) - os=`echo $os | sed -e 's|sinix|sysv|'` - ;; - -sinix*) - os=-sysv4 - ;; - -tpf*) - os=-tpf - ;; - -triton*) - os=-sysv3 - ;; - -oss*) - os=-sysv3 - ;; - -svr4) - os=-sysv4 - ;; - -svr3) - os=-sysv3 - ;; - -sysvr4) - os=-sysv4 - ;; - # This must come after -sysvr4. - -sysv*) - ;; - -ose*) - os=-ose - ;; - -es1800*) - os=-ose - ;; - -xenix) - os=-xenix - ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - os=-mint - ;; - -aros*) - os=-aros - ;; - -kaos*) - os=-kaos - ;; - -none) - ;; - *) - # Get rid of the `-' at the beginning of $os. - os=`echo $os | sed 's/[^-]*-//'` - echo Invalid configuration \`$1\': system \`$os\' not recognized 1>&2 - exit 1 - ;; -esac -else - -# Here we handle the default operating systems that come with various machines. -# The value should be what the vendor currently ships out the door with their -# machine or put another way, the most popular os provided with the machine. - -# Note that if you're going to try to match "-MANUFACTURER" here (say, -# "-sun"), then you have to tell the case statement up towards the top -# that MANUFACTURER isn't an operating system. Otherwise, code above -# will signal an error saying that MANUFACTURER isn't an operating -# system, and we'll never get to this point. - -case $basic_machine in - *-acorn) - os=-riscix1.2 - ;; - arm*-rebel) - os=-linux - ;; - arm*-semi) - os=-aout - ;; - c4x-* | tic4x-*) - os=-coff - ;; - # This must come before the *-dec entry. - pdp10-*) - os=-tops20 - ;; - pdp11-*) - os=-none - ;; - *-dec | vax-*) - os=-ultrix4.2 - ;; - m68*-apollo) - os=-domain - ;; - i386-sun) - os=-sunos4.0.2 - ;; - m68000-sun) - os=-sunos3 - # This also exists in the configure program, but was not the - # default. - # os=-sunos4 - ;; - m68*-cisco) - os=-aout - ;; - mips*-cisco) - os=-elf - ;; - mips*-*) - os=-elf - ;; - or32-*) - os=-coff - ;; - *-tti) # must be before sparc entry or we get the wrong os. - os=-sysv3 - ;; - sparc-* | *-sun) - os=-sunos4.1.1 - ;; - *-be) - os=-beos - ;; - *-ibm) - os=-aix - ;; - *-wec) - os=-proelf - ;; - *-winbond) - os=-proelf - ;; - *-oki) - os=-proelf - ;; - *-hp) - os=-hpux - ;; - *-hitachi) - os=-hiux - ;; - i860-* | *-att | *-ncr | *-altos | *-motorola | *-convergent) - os=-sysv - ;; - *-cbm) - os=-amigaos - ;; - *-dg) - os=-dgux - ;; - *-dolphin) - os=-sysv3 - ;; - m68k-ccur) - os=-rtu - ;; - m88k-omron*) - os=-luna - ;; - *-next ) - os=-nextstep - ;; - *-sequent) - os=-ptx - ;; - *-crds) - os=-unos - ;; - *-ns) - os=-genix - ;; - i370-*) - os=-mvs - ;; - *-next) - os=-nextstep3 - ;; - *-gould) - os=-sysv - ;; - *-highlevel) - os=-bsd - ;; - *-encore) - os=-bsd - ;; - *-sgi) - os=-irix - ;; - *-siemens) - os=-sysv4 - ;; - *-masscomp) - os=-rtu - ;; - f30[01]-fujitsu | f700-fujitsu) - os=-uxpv - ;; - *-rom68k) - os=-coff - ;; - *-*bug) - os=-coff - ;; - *-apple) - os=-macos - ;; - *-atari*) - os=-mint - ;; - *) - os=-none - ;; -esac -fi - -# Here we handle the case where we know the os, and the CPU type, but not the -# manufacturer. We pick the logical manufacturer. -vendor=unknown -case $basic_machine in - *-unknown) - case $os in - -riscix*) - vendor=acorn - ;; - -sunos*) - vendor=sun - ;; - -aix*) - vendor=ibm - ;; - -beos*) - vendor=be - ;; - -hpux*) - vendor=hp - ;; - -mpeix*) - vendor=hp - ;; - -hiux*) - vendor=hitachi - ;; - -unos*) - vendor=crds - ;; - -dgux*) - vendor=dg - ;; - -luna*) - vendor=omron - ;; - -genix*) - vendor=ns - ;; - -mvs* | -opened*) - vendor=ibm - ;; - -os400*) - vendor=ibm - ;; - -ptx*) - vendor=sequent - ;; - -tpf*) - vendor=ibm - ;; - -vxsim* | -vxworks* | -windiss*) - vendor=wrs - ;; - -aux*) - vendor=apple - ;; - -hms*) - vendor=hitachi - ;; - -mpw* | -macos*) - vendor=apple - ;; - -*mint | -mint[0-9]* | -*MiNT | -MiNT[0-9]*) - vendor=atari - ;; - -vos*) - vendor=stratus - ;; - esac - basic_machine=`echo $basic_machine | sed "s/unknown/$vendor/"` - ;; -esac - -echo $basic_machine$os -exit 0 - -# Local variables: -# eval: (add-hook 'write-file-hooks 'time-stamp) -# time-stamp-start: "timestamp='" -# time-stamp-format: "%:y-%02m-%02d" -# time-stamp-end: "'" -# End: diff --git a/deps/jemalloc.orig/configure.ac b/deps/jemalloc.orig/configure.ac deleted file mode 100644 index b58aa520998..00000000000 --- a/deps/jemalloc.orig/configure.ac +++ /dev/null @@ -1,938 +0,0 @@ -dnl Process this file with autoconf to produce a configure script. -AC_INIT([Makefile.in]) - -dnl ============================================================================ -dnl Custom macro definitions. - -dnl JE_CFLAGS_APPEND(cflag) -AC_DEFUN([JE_CFLAGS_APPEND], -[ -AC_MSG_CHECKING([whether compiler supports $1]) -TCFLAGS="${CFLAGS}" -if test "x${CFLAGS}" = "x" ; then - CFLAGS="$1" -else - CFLAGS="${CFLAGS} $1" -fi -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[[ -]], [[ - return 0; -]])], - AC_MSG_RESULT([yes]), - AC_MSG_RESULT([no]) - [CFLAGS="${TCFLAGS}"] -) -]) - -dnl JE_COMPILABLE(label, hcode, mcode, rvar) -AC_DEFUN([JE_COMPILABLE], -[ -AC_MSG_CHECKING([whether $1 is compilable]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[$2], [$3])], - AC_MSG_RESULT([yes]) - [$4="yes"], - AC_MSG_RESULT([no]) - [$4="no"] -) -]) - -dnl ============================================================================ - -srcroot=$srcdir -if test "x${srcroot}" = "x." ; then - srcroot="" -else - srcroot="${srcroot}/" -fi -AC_SUBST([srcroot]) -abs_srcroot="`cd \"${srcdir}\"; pwd`/" -AC_SUBST([abs_srcroot]) - -objroot="" -AC_SUBST([objroot]) -abs_objroot="`pwd`/" -AC_SUBST([abs_objroot]) - -dnl Munge install path variables. -if test "x$prefix" = "xNONE" ; then - prefix="/usr/local" -fi -if test "x$exec_prefix" = "xNONE" ; then - exec_prefix=$prefix -fi -PREFIX=$prefix -AC_SUBST([PREFIX]) -BINDIR=`eval echo $bindir` -BINDIR=`eval echo $BINDIR` -AC_SUBST([BINDIR]) -INCLUDEDIR=`eval echo $includedir` -INCLUDEDIR=`eval echo $INCLUDEDIR` -AC_SUBST([INCLUDEDIR]) -LIBDIR=`eval echo $libdir` -LIBDIR=`eval echo $LIBDIR` -AC_SUBST([LIBDIR]) -DATADIR=`eval echo $datadir` -DATADIR=`eval echo $DATADIR` -AC_SUBST([DATADIR]) -MANDIR=`eval echo $mandir` -MANDIR=`eval echo $MANDIR` -AC_SUBST([MANDIR]) - -dnl Support for building documentation. -AC_PATH_PROG([XSLTPROC], [xsltproc], , [$PATH]) -AC_ARG_WITH([xslroot], - [AS_HELP_STRING([--with-xslroot=], [XSL stylesheet root path])], -if test "x$with_xslroot" = "xno" ; then - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" -else - XSLROOT="${with_xslroot}" -fi, - XSLROOT="/usr/share/xml/docbook/stylesheet/docbook-xsl" -) -AC_SUBST([XSLROOT]) - -dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, -dnl just prevent autoconf from molesting CFLAGS. -CFLAGS=$CFLAGS -AC_PROG_CC -if test "x$CFLAGS" = "x" ; then - no_CFLAGS="yes" - if test "x$GCC" = "xyes" ; then - JE_CFLAGS_APPEND([-std=gnu99]) - JE_CFLAGS_APPEND([-Wall]) - JE_CFLAGS_APPEND([-pipe]) - JE_CFLAGS_APPEND([-g3]) - fi -fi -dnl Append EXTRA_CFLAGS to CFLAGS, if defined. -if test "x$EXTRA_CFLAGS" != "x" ; then - JE_CFLAGS_APPEND([$EXTRA_CFLAGS]) -fi -AC_PROG_CPP - -AC_CHECK_SIZEOF([void *]) -if test "x${ac_cv_sizeof_void_p}" = "x8" ; then - LG_SIZEOF_PTR=3 -elif test "x${ac_cv_sizeof_void_p}" = "x4" ; then - LG_SIZEOF_PTR=2 -else - AC_MSG_ERROR([Unsupported pointer size: ${ac_cv_sizeof_void_p}]) -fi -AC_DEFINE_UNQUOTED([LG_SIZEOF_PTR], [$LG_SIZEOF_PTR]) - -AC_CHECK_SIZEOF([int]) -if test "x${ac_cv_sizeof_int}" = "x8" ; then - LG_SIZEOF_INT=3 -elif test "x${ac_cv_sizeof_int}" = "x4" ; then - LG_SIZEOF_INT=2 -else - AC_MSG_ERROR([Unsupported int size: ${ac_cv_sizeof_int}]) -fi -AC_DEFINE_UNQUOTED([LG_SIZEOF_INT], [$LG_SIZEOF_INT]) - -AC_CHECK_SIZEOF([long]) -if test "x${ac_cv_sizeof_long}" = "x8" ; then - LG_SIZEOF_LONG=3 -elif test "x${ac_cv_sizeof_long}" = "x4" ; then - LG_SIZEOF_LONG=2 -else - AC_MSG_ERROR([Unsupported long size: ${ac_cv_sizeof_long}]) -fi -AC_DEFINE_UNQUOTED([LG_SIZEOF_LONG], [$LG_SIZEOF_LONG]) - -AC_CANONICAL_HOST -dnl CPU-specific settings. -CPU_SPINWAIT="" -case "${host_cpu}" in - i[[345]]86) - ;; - i686) - JE_COMPILABLE([__asm__], [], [[__asm__ volatile("pause"); return 0;]], - [asm]) - if test "x${asm}" = "xyes" ; then - CPU_SPINWAIT='__asm__ volatile("pause")' - fi - ;; - x86_64) - JE_COMPILABLE([__asm__ syntax], [], - [[__asm__ volatile("pause"); return 0;]], [asm]) - if test "x${asm}" = "xyes" ; then - CPU_SPINWAIT='__asm__ volatile("pause")' - fi - ;; - *) - ;; -esac -AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT]) - -dnl Platform-specific settings. abi and RPATH can probably be determined -dnl programmatically, but doing so is error-prone, which makes it generally -dnl not worth the trouble. -dnl -dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the -dnl definitions need to be seen before any headers are included, which is a pain -dnl to make happen otherwise. -case "${host}" in - *-*-darwin*) - CFLAGS="$CFLAGS -fno-common -no-cpp-precomp" - abi="macho" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="" - ;; - *-*-freebsd*) - CFLAGS="$CFLAGS" - abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="-Wl,-rpath," - ;; - *-*-linux*) - CFLAGS="$CFLAGS" - CPPFLAGS="$CPPFLAGS -D_GNU_SOURCE" - abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED]) - RPATH="-Wl,-rpath," - ;; - *-*-netbsd*) - AC_MSG_CHECKING([ABI]) - AC_COMPILE_IFELSE([AC_LANG_PROGRAM( -[[#ifdef __ELF__ -/* ELF */ -#else -#error aout -#endif -]])], - [CFLAGS="$CFLAGS"; abi="elf"], - [abi="aout"]) - AC_MSG_RESULT([$abi]) - AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE]) - RPATH="-Wl,-rpath," - ;; - *-*-solaris2*) - CFLAGS="$CFLAGS" - abi="elf" - RPATH="-Wl,-R," - dnl Solaris needs this for sigwait(). - CPPFLAGS="$CPPFLAGS -D_POSIX_PTHREAD_SEMANTICS" - LIBS="$LIBS -lposix4 -lsocket -lnsl" - ;; - *) - AC_MSG_RESULT([Unsupported operating system: ${host}]) - abi="elf" - RPATH="-Wl,-rpath," - ;; -esac -AC_SUBST([abi]) -AC_SUBST([RPATH]) - -JE_COMPILABLE([__attribute__ syntax], - [static __attribute__((unused)) void foo(void){}], - [], - [attribute]) -if test "x${attribute}" = "xyes" ; then - AC_DEFINE([JEMALLOC_HAVE_ATTR], [ ]) - if test "x${GCC}" = "xyes" -a "x${abi}" = "xelf"; then - JE_CFLAGS_APPEND([-fvisibility=hidden]) - fi -fi - -JE_COMPILABLE([mremap(...MREMAP_FIXED...)], [ -#define _GNU_SOURCE -#include -], [ -void *p = mremap((void *)0, 0, 0, MREMAP_MAYMOVE|MREMAP_FIXED, (void *)0); -], [mremap_fixed]) -if test "x${mremap_fixed}" = "xyes" ; then - AC_DEFINE([JEMALLOC_MREMAP_FIXED]) -fi - -dnl Support optional additions to rpath. -AC_ARG_WITH([rpath], - [AS_HELP_STRING([--with-rpath=], [Colon-separated rpath (ELF systems only)])], -if test "x$with_rpath" = "xno" ; then - RPATH_EXTRA= -else - RPATH_EXTRA="`echo $with_rpath | tr \":\" \" \"`" -fi, - RPATH_EXTRA= -) -AC_SUBST([RPATH_EXTRA]) - -dnl Disable rules that do automatic regeneration of configure output by default. -AC_ARG_ENABLE([autogen], - [AS_HELP_STRING([--enable-autogen], [Automatically regenerate configure output])], -if test "x$enable_autogen" = "xno" ; then - enable_autogen="0" -else - enable_autogen="1" -fi -, -enable_autogen="0" -) -AC_SUBST([enable_autogen]) - -AC_PROG_INSTALL -AC_PROG_RANLIB -AC_PATH_PROG([AR], [ar], , [$PATH]) -AC_PATH_PROG([LD], [ld], , [$PATH]) -AC_PATH_PROG([AUTOCONF], [autoconf], , [$PATH]) - -dnl Do not prefix public APIs by default. -AC_ARG_WITH([jemalloc_prefix], - [AS_HELP_STRING([--with-jemalloc-prefix=], [Prefix to prepend to all public APIs])], - [JEMALLOC_PREFIX="$with_jemalloc_prefix"], - [if test "x$abi" != "xmacho" ; then - JEMALLOC_PREFIX="" -else - JEMALLOC_PREFIX="je_" -fi] -) -if test "x$JEMALLOC_PREFIX" != "x" ; then - JEMALLOC_CPREFIX=`echo ${JEMALLOC_PREFIX} | tr "a-z" "A-Z"` - AC_DEFINE_UNQUOTED([JEMALLOC_PREFIX], ["$JEMALLOC_PREFIX"]) - AC_DEFINE_UNQUOTED([JEMALLOC_CPREFIX], ["$JEMALLOC_CPREFIX"]) - AC_DEFINE_UNQUOTED([JEMALLOC_P(string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix)], [${JEMALLOC_PREFIX}##string_that_no_one_should_want_to_use_as_a_jemalloc_API_prefix]) -fi - -dnl Do not mangle library-private APIs by default. -AC_ARG_WITH([private_namespace], - [AS_HELP_STRING([--with-private-namespace=], [Prefix to prepend to all library-private APIs])], - [JEMALLOC_PRIVATE_NAMESPACE="$with_private_namespace"], - [JEMALLOC_PRIVATE_NAMESPACE=""] -) -AC_DEFINE_UNQUOTED([JEMALLOC_PRIVATE_NAMESPACE], ["$JEMALLOC_PRIVATE_NAMESPACE"]) -if test "x$JEMALLOC_PRIVATE_NAMESPACE" != "x" ; then - AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [${JEMALLOC_PRIVATE_NAMESPACE}##string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix]) -else - AC_DEFINE_UNQUOTED([JEMALLOC_N(string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix)], [string_that_no_one_should_want_to_use_as_a_jemalloc_private_namespace_prefix]) -fi - -dnl Do not add suffix to installed files by default. -AC_ARG_WITH([install_suffix], - [AS_HELP_STRING([--with-install-suffix=], [Suffix to append to all installed files])], - [INSTALL_SUFFIX="$with_install_suffix"], - [INSTALL_SUFFIX=] -) -install_suffix="$INSTALL_SUFFIX" -AC_SUBST([install_suffix]) - -cfgoutputs_in="${srcroot}Makefile.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/html.xsl.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/manpages.xsl.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}doc/jemalloc.xml.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/jemalloc.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}include/jemalloc/internal/jemalloc_internal.h.in" -cfgoutputs_in="${cfgoutputs_in} ${srcroot}test/jemalloc_test.h.in" - -cfgoutputs_out="Makefile" -cfgoutputs_out="${cfgoutputs_out} doc/html.xsl" -cfgoutputs_out="${cfgoutputs_out} doc/manpages.xsl" -cfgoutputs_out="${cfgoutputs_out} doc/jemalloc${install_suffix}.xml" -cfgoutputs_out="${cfgoutputs_out} include/jemalloc/jemalloc${install_suffix}.h" -cfgoutputs_out="${cfgoutputs_out} include/jemalloc/internal/jemalloc_internal.h" -cfgoutputs_out="${cfgoutputs_out} test/jemalloc_test.h" - -cfgoutputs_tup="Makefile" -cfgoutputs_tup="${cfgoutputs_tup} doc/html.xsl:doc/html.xsl.in" -cfgoutputs_tup="${cfgoutputs_tup} doc/manpages.xsl:doc/manpages.xsl.in" -cfgoutputs_tup="${cfgoutputs_tup} doc/jemalloc${install_suffix}.xml:doc/jemalloc.xml.in" -cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/jemalloc${install_suffix}.h:include/jemalloc/jemalloc.h.in" -cfgoutputs_tup="${cfgoutputs_tup} include/jemalloc/internal/jemalloc_internal.h" -cfgoutputs_tup="${cfgoutputs_tup} test/jemalloc_test.h:test/jemalloc_test.h.in" - -cfghdrs_in="${srcroot}include/jemalloc/jemalloc_defs.h.in" - -cfghdrs_out="include/jemalloc/jemalloc_defs${install_suffix}.h" - -cfghdrs_tup="include/jemalloc/jemalloc_defs${install_suffix}.h:include/jemalloc/jemalloc_defs.h.in" - -dnl Do not silence irrelevant compiler warnings by default, since enabling this -dnl option incurs a performance penalty. -AC_ARG_ENABLE([cc-silence], - [AS_HELP_STRING([--enable-cc-silence], - [Silence irrelevant compiler warnings])], -[if test "x$enable_cc_silence" = "xno" ; then - enable_cc_silence="0" -else - enable_cc_silence="1" -fi -], -[enable_cc_silence="0"] -) -if test "x$enable_cc_silence" = "x1" ; then - AC_DEFINE([JEMALLOC_CC_SILENCE]) -fi - -dnl Do not compile with debugging by default. -AC_ARG_ENABLE([debug], - [AS_HELP_STRING([--enable-debug], [Build debugging code])], -[if test "x$enable_debug" = "xno" ; then - enable_debug="0" -else - enable_debug="1" -fi -], -[enable_debug="0"] -) -if test "x$enable_debug" = "x1" ; then - AC_DEFINE([JEMALLOC_DEBUG], [ ]) - AC_DEFINE([JEMALLOC_IVSALLOC], [ ]) -fi -AC_SUBST([enable_debug]) - -dnl Only optimize if not debugging. -if test "x$enable_debug" = "x0" -a "x$no_CFLAGS" = "xyes" ; then - dnl Make sure that an optimization flag was not specified in EXTRA_CFLAGS. - optimize="no" - echo "$EXTRA_CFLAGS" | grep "\-O" >/dev/null || optimize="yes" - if test "x${optimize}" = "xyes" ; then - if test "x$GCC" = "xyes" ; then - JE_CFLAGS_APPEND([-O3]) - JE_CFLAGS_APPEND([-funroll-loops]) - else - JE_CFLAGS_APPEND([-O]) - fi - fi -fi - -dnl Do not enable statistics calculation by default. -AC_ARG_ENABLE([stats], - [AS_HELP_STRING([--enable-stats], [Enable statistics calculation/reporting])], -[if test "x$enable_stats" = "xno" ; then - enable_stats="0" -else - enable_stats="1" -fi -], -[enable_stats="0"] -) -if test "x$enable_stats" = "x1" ; then - AC_DEFINE([JEMALLOC_STATS], [ ]) -fi -AC_SUBST([enable_stats]) - -dnl Do not enable profiling by default. -AC_ARG_ENABLE([prof], - [AS_HELP_STRING([--enable-prof], [Enable allocation profiling])], -[if test "x$enable_prof" = "xno" ; then - enable_prof="0" -else - enable_prof="1" -fi -], -[enable_prof="0"] -) -if test "x$enable_prof" = "x1" ; then - backtrace_method="" -else - backtrace_method="N/A" -fi - -AC_ARG_ENABLE([prof-libunwind], - [AS_HELP_STRING([--enable-prof-libunwind], [Use libunwind for backtracing])], -[if test "x$enable_prof_libunwind" = "xno" ; then - enable_prof_libunwind="0" -else - enable_prof_libunwind="1" -fi -], -[enable_prof_libunwind="0"] -) -AC_ARG_WITH([static_libunwind], - [AS_HELP_STRING([--with-static-libunwind=], - [Path to static libunwind library; use rather than dynamically linking])], -if test "x$with_static_libunwind" = "xno" ; then - LUNWIND="-lunwind" -else - if test ! -f "$with_static_libunwind" ; then - AC_MSG_ERROR([Static libunwind not found: $with_static_libunwind]) - fi - LUNWIND="$with_static_libunwind" -fi, - LUNWIND="-lunwind" -) -if test "x$backtrace_method" = "x" -a "x$enable_prof_libunwind" = "x1" ; then - AC_CHECK_HEADERS([libunwind.h], , [enable_prof_libunwind="0"]) - if test "x$LUNWIND" = "x-lunwind" ; then - AC_CHECK_LIB([unwind], [backtrace], [LIBS="$LIBS $LUNWIND"], - [enable_prof_libunwind="0"]) - else - LIBS="$LIBS $LUNWIND" - fi - if test "x${enable_prof_libunwind}" = "x1" ; then - backtrace_method="libunwind" - AC_DEFINE([JEMALLOC_PROF_LIBUNWIND], [ ]) - fi -fi - -AC_ARG_ENABLE([prof-libgcc], - [AS_HELP_STRING([--disable-prof-libgcc], - [Do not use libgcc for backtracing])], -[if test "x$enable_prof_libgcc" = "xno" ; then - enable_prof_libgcc="0" -else - enable_prof_libgcc="1" -fi -], -[enable_prof_libgcc="1"] -) -if test "x$backtrace_method" = "x" -a "x$enable_prof_libgcc" = "x1" \ - -a "x$GCC" = "xyes" ; then - AC_CHECK_HEADERS([unwind.h], , [enable_prof_libgcc="0"]) - AC_CHECK_LIB([gcc], [_Unwind_Backtrace], [LIBS="$LIBS -lgcc"], [enable_prof_libgcc="0"]) - dnl The following is conservative, in that it only has entries for CPUs on - dnl which jemalloc has been tested. - AC_MSG_CHECKING([libgcc-based backtracing reliability on ${host_cpu}]) - case "${host_cpu}" in - i[[3456]]86) - AC_MSG_RESULT([unreliable]) - enable_prof_libgcc="0"; - ;; - x86_64) - AC_MSG_RESULT([reliable]) - ;; - *) - AC_MSG_RESULT([unreliable]) - enable_prof_libgcc="0"; - ;; - esac - if test "x${enable_prof_libgcc}" = "x1" ; then - backtrace_method="libgcc" - AC_DEFINE([JEMALLOC_PROF_LIBGCC], [ ]) - fi -else - enable_prof_libgcc="0" -fi - -AC_ARG_ENABLE([prof-gcc], - [AS_HELP_STRING([--disable-prof-gcc], - [Do not use gcc intrinsics for backtracing])], -[if test "x$enable_prof_gcc" = "xno" ; then - enable_prof_gcc="0" -else - enable_prof_gcc="1" -fi -], -[enable_prof_gcc="1"] -) -if test "x$backtrace_method" = "x" -a "x$enable_prof_gcc" = "x1" \ - -a "x$GCC" = "xyes" ; then - backtrace_method="gcc intrinsics" - AC_DEFINE([JEMALLOC_PROF_GCC], [ ]) -else - enable_prof_gcc="0" -fi - -if test "x$backtrace_method" = "x" ; then - backtrace_method="none (disabling profiling)" - enable_prof="0" -fi -AC_MSG_CHECKING([configured backtracing method]) -AC_MSG_RESULT([$backtrace_method]) -if test "x$enable_prof" = "x1" ; then - LIBS="$LIBS -lm" - AC_DEFINE([JEMALLOC_PROF], [ ]) -fi -AC_SUBST([enable_prof]) - -dnl Enable tiny allocations by default. -AC_ARG_ENABLE([tiny], - [AS_HELP_STRING([--disable-tiny], [Disable tiny (sub-quantum) allocations])], -[if test "x$enable_tiny" = "xno" ; then - enable_tiny="0" -else - enable_tiny="1" -fi -], -[enable_tiny="1"] -) -if test "x$enable_tiny" = "x1" ; then - AC_DEFINE([JEMALLOC_TINY], [ ]) -fi -AC_SUBST([enable_tiny]) - -dnl Enable thread-specific caching by default. -AC_ARG_ENABLE([tcache], - [AS_HELP_STRING([--disable-tcache], [Disable per thread caches])], -[if test "x$enable_tcache" = "xno" ; then - enable_tcache="0" -else - enable_tcache="1" -fi -], -[enable_tcache="1"] -) -if test "x$enable_tcache" = "x1" ; then - AC_DEFINE([JEMALLOC_TCACHE], [ ]) -fi -AC_SUBST([enable_tcache]) - -dnl Do not enable mmap()ped swap files by default. -AC_ARG_ENABLE([swap], - [AS_HELP_STRING([--enable-swap], [Enable mmap()ped swap files])], -[if test "x$enable_swap" = "xno" ; then - enable_swap="0" -else - enable_swap="1" -fi -], -[enable_swap="0"] -) -if test "x$enable_swap" = "x1" ; then - AC_DEFINE([JEMALLOC_SWAP], [ ]) -fi -AC_SUBST([enable_swap]) - -dnl Do not enable allocation from DSS by default. -AC_ARG_ENABLE([dss], - [AS_HELP_STRING([--enable-dss], [Enable allocation from DSS])], -[if test "x$enable_dss" = "xno" ; then - enable_dss="0" -else - enable_dss="1" -fi -], -[enable_dss="0"] -) -if test "x$enable_dss" = "x1" ; then - AC_DEFINE([JEMALLOC_DSS], [ ]) -fi -AC_SUBST([enable_dss]) - -dnl Do not support the junk/zero filling option by default. -AC_ARG_ENABLE([fill], - [AS_HELP_STRING([--enable-fill], [Support junk/zero filling option])], -[if test "x$enable_fill" = "xno" ; then - enable_fill="0" -else - enable_fill="1" -fi -], -[enable_fill="0"] -) -if test "x$enable_fill" = "x1" ; then - AC_DEFINE([JEMALLOC_FILL], [ ]) -fi -AC_SUBST([enable_fill]) - -dnl Do not support the xmalloc option by default. -AC_ARG_ENABLE([xmalloc], - [AS_HELP_STRING([--enable-xmalloc], [Support xmalloc option])], -[if test "x$enable_xmalloc" = "xno" ; then - enable_xmalloc="0" -else - enable_xmalloc="1" -fi -], -[enable_xmalloc="0"] -) -if test "x$enable_xmalloc" = "x1" ; then - AC_DEFINE([JEMALLOC_XMALLOC], [ ]) -fi -AC_SUBST([enable_xmalloc]) - -dnl Do not support the SYSV option by default. -AC_ARG_ENABLE([sysv], - [AS_HELP_STRING([--enable-sysv], [Support SYSV semantics option])], -[if test "x$enable_sysv" = "xno" ; then - enable_sysv="0" -else - enable_sysv="1" -fi -], -[enable_sysv="0"] -) -if test "x$enable_sysv" = "x1" ; then - AC_DEFINE([JEMALLOC_SYSV], [ ]) -fi -AC_SUBST([enable_sysv]) - -dnl Do not determine page shift at run time by default. -AC_ARG_ENABLE([dynamic_page_shift], - [AS_HELP_STRING([--enable-dynamic-page-shift], - [Determine page size at run time (don't trust configure result)])], -[if test "x$enable_dynamic_page_shift" = "xno" ; then - enable_dynamic_page_shift="0" -else - enable_dynamic_page_shift="1" -fi -], -[enable_dynamic_page_shift="0"] -) -if test "x$enable_dynamic_page_shift" = "x1" ; then - AC_DEFINE([DYNAMIC_PAGE_SHIFT], [ ]) -fi -AC_SUBST([enable_dynamic_page_shift]) - -AC_MSG_CHECKING([STATIC_PAGE_SHIFT]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[[#include -#include -#include -]], [[ - long result; - FILE *f; - - result = sysconf(_SC_PAGESIZE); - if (result == -1) { - return 1; - } - f = fopen("conftest.out", "w"); - if (f == NULL) { - return 1; - } - fprintf(f, "%u\n", ffs((int)result) - 1); - close(f); - - return 0; -]])], - [STATIC_PAGE_SHIFT=`cat conftest.out`] - AC_MSG_RESULT([$STATIC_PAGE_SHIFT]) - AC_DEFINE_UNQUOTED([STATIC_PAGE_SHIFT], [$STATIC_PAGE_SHIFT]), - AC_MSG_RESULT([error])) - -dnl ============================================================================ -dnl jemalloc configuration. -dnl - -dnl Set VERSION if source directory has an embedded git repository. -if test -d "${srcroot}.git" ; then - git describe --long --abbrev=40 > ${srcroot}VERSION -fi -jemalloc_version=`cat ${srcroot}VERSION` -jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` -jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` -jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` -jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` -jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` -AC_SUBST([jemalloc_version]) -AC_SUBST([jemalloc_version_major]) -AC_SUBST([jemalloc_version_minor]) -AC_SUBST([jemalloc_version_bugfix]) -AC_SUBST([jemalloc_version_nrev]) -AC_SUBST([jemalloc_version_gid]) - -dnl ============================================================================ -dnl Configure pthreads. - -AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) -AC_CHECK_LIB([pthread], [pthread_create], [LIBS="$LIBS -lpthread"], - [AC_MSG_ERROR([libpthread is missing])]) - -CPPFLAGS="$CPPFLAGS -D_REENTRANT" - -dnl Enable lazy locking by default. -AC_ARG_ENABLE([lazy_lock], - [AS_HELP_STRING([--disable-lazy-lock], - [Disable lazy locking (always lock, even when single-threaded)])], -[if test "x$enable_lazy_lock" = "xno" ; then - enable_lazy_lock="0" -else - enable_lazy_lock="1" -fi -], -[enable_lazy_lock="1"] -) -if test "x$enable_lazy_lock" = "x1" ; then - AC_CHECK_HEADERS([dlfcn.h], , [AC_MSG_ERROR([dlfcn.h is missing])]) - AC_CHECK_LIB([dl], [dlopen], [LIBS="$LIBS -ldl"], - [AC_MSG_ERROR([libdl is missing])]) - AC_DEFINE([JEMALLOC_LAZY_LOCK], [ ]) -fi -AC_SUBST([enable_lazy_lock]) - -AC_ARG_ENABLE([tls], - [AS_HELP_STRING([--disable-tls], [Disable thread-local storage (__thread keyword)])], -if test "x$enable_tls" = "xno" ; then - enable_tls="0" -else - enable_tls="1" -fi -, -enable_tls="1" -) -if test "x${enable_tls}" = "x1" ; then -AC_MSG_CHECKING([for TLS]) -AC_RUN_IFELSE([AC_LANG_PROGRAM( -[[ - __thread int x; -]], [[ - x = 42; - - return 0; -]])], - AC_MSG_RESULT([yes]), - AC_MSG_RESULT([no]) - enable_tls="0") -fi -AC_SUBST([enable_tls]) -if test "x${enable_tls}" = "x0" ; then - AC_DEFINE_UNQUOTED([NO_TLS], [ ]) -fi - -dnl ============================================================================ -dnl Check for ffsl(3), and fail if not found. This function exists on all -dnl platforms that jemalloc currently has a chance of functioning on without -dnl modification. - -AC_CHECK_FUNC([ffsl], [], - [AC_MSG_ERROR([Cannot build without ffsl(3)])]) - -dnl ============================================================================ -dnl Check for atomic(3) operations as provided on Darwin. - -JE_COMPILABLE([Darwin OSAtomic*()], [ -#include -#include -], [ - { - int32_t x32 = 0; - volatile int32_t *x32p = &x32; - OSAtomicAdd32(1, x32p); - } - { - int64_t x64 = 0; - volatile int64_t *x64p = &x64; - OSAtomicAdd64(1, x64p); - } -], [osatomic]) -if test "x${osatomic}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSATOMIC]) -fi - -dnl ============================================================================ -dnl Check for spinlock(3) operations as provided on Darwin. - -JE_COMPILABLE([Darwin OSSpin*()], [ -#include -#include -], [ - OSSpinLock lock = 0; - OSSpinLockLock(&lock); - OSSpinLockUnlock(&lock); -], [osspin]) -if test "x${osspin}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSSPIN]) -fi - -dnl ============================================================================ -dnl Check for allocator-related functions that should be wrapped. - -AC_CHECK_FUNC([memalign], - [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN])]) -AC_CHECK_FUNC([valloc], - [AC_DEFINE([JEMALLOC_OVERRIDE_VALLOC])]) - -dnl ============================================================================ -dnl Darwin-related configuration. - -if test "x${abi}" = "xmacho" ; then - AC_DEFINE([JEMALLOC_IVSALLOC]) - AC_DEFINE([JEMALLOC_ZONE]) - - dnl The szone version jumped from 3 to 6 between the OS X 10.5.x and 10.6 - dnl releases. malloc_zone_t and malloc_introspection_t have new fields in - dnl 10.6, which is the only source-level indication of the change. - AC_MSG_CHECKING([malloc zone version]) - AC_TRY_COMPILE([#include -#include ], [ - static malloc_zone_t zone; - static struct malloc_introspection_t zone_introspect; - - zone.size = NULL; - zone.malloc = NULL; - zone.calloc = NULL; - zone.valloc = NULL; - zone.free = NULL; - zone.realloc = NULL; - zone.destroy = NULL; - zone.zone_name = "jemalloc_zone"; - zone.batch_malloc = NULL; - zone.batch_free = NULL; - zone.introspect = &zone_introspect; - zone.version = 6; - zone.memalign = NULL; - zone.free_definite_size = NULL; - - zone_introspect.enumerator = NULL; - zone_introspect.good_size = NULL; - zone_introspect.check = NULL; - zone_introspect.print = NULL; - zone_introspect.log = NULL; - zone_introspect.force_lock = NULL; - zone_introspect.force_unlock = NULL; - zone_introspect.statistics = NULL; - zone_introspect.zone_locked = NULL; -], [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [6]) - AC_MSG_RESULT([6])], - [AC_DEFINE_UNQUOTED([JEMALLOC_ZONE_VERSION], [3]) - AC_MSG_RESULT([3])]) -fi - -dnl ============================================================================ -dnl Check for typedefs, structures, and compiler characteristics. -AC_HEADER_STDBOOL - -dnl Process .in files. -AC_SUBST([cfghdrs_in]) -AC_SUBST([cfghdrs_out]) -AC_CONFIG_HEADERS([$cfghdrs_tup]) - -dnl ============================================================================ -dnl Generate outputs. -AC_CONFIG_FILES([$cfgoutputs_tup config.stamp]) -AC_SUBST([cfgoutputs_in]) -AC_SUBST([cfgoutputs_out]) -AC_OUTPUT - -dnl ============================================================================ -dnl Print out the results of configuration. -AC_MSG_RESULT([===============================================================================]) -AC_MSG_RESULT([jemalloc version : $jemalloc_version]) -AC_MSG_RESULT([]) -AC_MSG_RESULT([CC : ${CC}]) -AC_MSG_RESULT([CPPFLAGS : ${CPPFLAGS}]) -AC_MSG_RESULT([CFLAGS : ${CFLAGS}]) -AC_MSG_RESULT([LDFLAGS : ${LDFLAGS}]) -AC_MSG_RESULT([LIBS : ${LIBS}]) -AC_MSG_RESULT([RPATH_EXTRA : ${RPATH_EXTRA}]) -AC_MSG_RESULT([]) -AC_MSG_RESULT([XSLTPROC : ${XSLTPROC}]) -AC_MSG_RESULT([XSLROOT : ${XSLROOT}]) -AC_MSG_RESULT([]) -AC_MSG_RESULT([PREFIX : ${PREFIX}]) -AC_MSG_RESULT([BINDIR : ${BINDIR}]) -AC_MSG_RESULT([INCLUDEDIR : ${INCLUDEDIR}]) -AC_MSG_RESULT([LIBDIR : ${LIBDIR}]) -AC_MSG_RESULT([DATADIR : ${DATADIR}]) -AC_MSG_RESULT([MANDIR : ${MANDIR}]) -AC_MSG_RESULT([]) -AC_MSG_RESULT([srcroot : ${srcroot}]) -AC_MSG_RESULT([abs_srcroot : ${abs_srcroot}]) -AC_MSG_RESULT([objroot : ${objroot}]) -AC_MSG_RESULT([abs_objroot : ${abs_objroot}]) -AC_MSG_RESULT([]) -AC_MSG_RESULT([JEMALLOC_PREFIX : ${JEMALLOC_PREFIX}]) -AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) -AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) -AC_MSG_RESULT([install_suffix : ${install_suffix}]) -AC_MSG_RESULT([autogen : ${enable_autogen}]) -AC_MSG_RESULT([cc-silence : ${enable_cc_silence}]) -AC_MSG_RESULT([debug : ${enable_debug}]) -AC_MSG_RESULT([stats : ${enable_stats}]) -AC_MSG_RESULT([prof : ${enable_prof}]) -AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) -AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) -AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) -AC_MSG_RESULT([tiny : ${enable_tiny}]) -AC_MSG_RESULT([tcache : ${enable_tcache}]) -AC_MSG_RESULT([fill : ${enable_fill}]) -AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) -AC_MSG_RESULT([sysv : ${enable_sysv}]) -AC_MSG_RESULT([swap : ${enable_swap}]) -AC_MSG_RESULT([dss : ${enable_dss}]) -AC_MSG_RESULT([dynamic_page_shift : ${enable_dynamic_page_shift}]) -AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) -AC_MSG_RESULT([tls : ${enable_tls}]) -AC_MSG_RESULT([===============================================================================]) diff --git a/deps/jemalloc.orig/doc/html.xsl.in b/deps/jemalloc.orig/doc/html.xsl.in deleted file mode 100644 index a91d9746f62..00000000000 --- a/deps/jemalloc.orig/doc/html.xsl.in +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/deps/jemalloc.orig/doc/jemalloc.xml.in b/deps/jemalloc.orig/doc/jemalloc.xml.in deleted file mode 100644 index 7a32879abee..00000000000 --- a/deps/jemalloc.orig/doc/jemalloc.xml.in +++ /dev/null @@ -1,2280 +0,0 @@ - - - - - - - User Manual - jemalloc - @jemalloc_version@ - - - Jason - Evans - Author - - - - - JEMALLOC - 3 - - - jemalloc - jemalloc - - general purpose memory allocation functions - - - LIBRARY - This manual describes jemalloc @jemalloc_version@. More information - can be found at the jemalloc website. - - - SYNOPSIS - - #include <stdlib.h> -#include <jemalloc/jemalloc.h> - - Standard API - - void *malloc - size_t size - - - void *calloc - size_t number - size_t size - - - int posix_memalign - void **ptr - size_t alignment - size_t size - - - void *realloc - void *ptr - size_t size - - - void free - void *ptr - - - - Non-standard API - - size_t malloc_usable_size - const void *ptr - - - void malloc_stats_print - void (*write_cb) - void *, const char * - - void *cbopaque - const char *opts - - - int mallctl - const char *name - void *oldp - size_t *oldlenp - void *newp - size_t newlen - - - int mallctlnametomib - const char *name - size_t *mibp - size_t *miblenp - - - int mallctlbymib - const size_t *mib - size_t miblen - void *oldp - size_t *oldlenp - void *newp - size_t newlen - - - void (*malloc_message) - void *cbopaque - const char *s - - const char *malloc_conf; - - - Experimental API - - int allocm - void **ptr - size_t *rsize - size_t size - int flags - - - int rallocm - void **ptr - size_t *rsize - size_t size - size_t extra - int flags - - - int sallocm - const void *ptr - size_t *rsize - int flags - - - int dallocm - void *ptr - int flags - - - - - - DESCRIPTION - - Standard API - - The malloc function allocates - size bytes of uninitialized memory. The allocated - space is suitably aligned (after possible pointer coercion) for storage - of any type of object. - - The calloc function allocates - space for number objects, each - size bytes in length. The result is identical to - calling malloc with an argument of - number * size, with the - exception that the allocated memory is explicitly initialized to zero - bytes. - - The posix_memalign function - allocates size bytes of memory such that the - allocation's base address is an even multiple of - alignment, and returns the allocation in the value - pointed to by ptr. The requested - alignment must be a power of 2 at least as large - as sizeof(void *). - - The realloc function changes the - size of the previously allocated memory referenced by - ptr to size bytes. The - contents of the memory are unchanged up to the lesser of the new and old - sizes. If the new size is larger, the contents of the newly allocated - portion of the memory are undefined. Upon success, the memory referenced - by ptr is freed and a pointer to the newly - allocated memory is returned. Note that - realloc may move the memory allocation, - resulting in a different return value than ptr. - If ptr is NULL, the - realloc function behaves identically to - malloc for the specified size. - - The free function causes the - allocated memory referenced by ptr to be made - available for future allocations. If ptr is - NULL, no action occurs. - - - Non-standard API - - The malloc_usable_size function - returns the usable size of the allocation pointed to by - ptr. The return value may be larger than the size - that was requested during allocation. The - malloc_usable_size function is not a - mechanism for in-place realloc; rather - it is provided solely as a tool for introspection purposes. Any - discrepancy between the requested allocation size and the size reported - by malloc_usable_size should not be - depended on, since such behavior is entirely implementation-dependent. - - - The malloc_stats_print function - writes human-readable summary statistics via the - write_cb callback function pointer and - cbopaque data passed to - write_cb, or - malloc_message if - write_cb is NULL. This - function can be called repeatedly. General information that never - changes during execution can be omitted by specifying "g" as a character - within the opts string. Note that - malloc_message uses the - mallctl* functions internally, so - inconsistent statistics can be reported if multiple threads use these - functions simultaneously. If is - specified during configuration, “m” and “a” can - be specified to omit merged arena and per arena statistics, respectively; - “b” and “l” can be specified to omit per size - class statistics for bins and large objects, respectively. Unrecognized - characters are silently ignored. Note that thread caching may prevent - some statistics from being completely up to date, since extra locking - would be required to merge counters that track thread cache operations. - - - The mallctl function provides a - general interface for introspecting the memory allocator, as well as - setting modifiable parameters and triggering actions. The - period-separated name argument specifies a - location in a tree-structured namespace; see the section for - documentation on the tree contents. To read a value, pass a pointer via - oldp to adequate space to contain the value, and a - pointer to its length via oldlenp; otherwise pass - NULL and NULL. Similarly, to - write a value, pass a pointer to the value via - newp, and its length via - newlen; otherwise pass NULL - and 0. - - The mallctlnametomib function - provides a way to avoid repeated name lookups for applications that - repeatedly query the same portion of the namespace, by translating a name - to a “Management Information Base” (MIB) that can be passed - repeatedly to mallctlbymib. Upon - successful return from mallctlnametomib, - mibp contains an array of - *miblenp integers, where - *miblenp is the lesser of the number of components - in name and the input value of - *miblenp. Thus it is possible to pass a - *miblenp that is smaller than the number of - period-separated name components, which results in a partial MIB that can - be used as the basis for constructing a complete MIB. For name - components that are integers (e.g. the 2 in - arenas.bin.2.size), - the corresponding MIB component will always be that integer. Therefore, - it is legitimate to construct code like the following: - - - Experimental API - The experimental API is subject to change or removal without regard - for backward compatibility. - - The allocm, - rallocm, - sallocm, and - dallocm functions all have a - flags argument that can be used to specify - options. The functions only check the options that are contextually - relevant. Use bitwise or (|) operations to - specify one or more of the following: - - - ALLOCM_LG_ALIGN(la) - - - Align the memory allocation to start at an address - that is a multiple of (1 << - la). This macro does not validate - that la is within the valid - range. - - - ALLOCM_ALIGN(a) - - - Align the memory allocation to start at an address - that is a multiple of a, where - a is a power of two. This macro does not - validate that a is a power of 2. - - - - ALLOCM_ZERO - - Initialize newly allocated memory to contain zero - bytes. In the growing reallocation case, the real size prior to - reallocation defines the boundary between untouched bytes and those - that are initialized to contain zero bytes. If this option is - absent, newly allocated memory is uninitialized. - - - ALLOCM_NO_MOVE - - For reallocation, fail rather than moving the - object. This constraint can apply to both growth and - shrinkage. - - - - - The allocm function allocates at - least size bytes of memory, sets - *ptr to the base address of the allocation, and - sets *rsize to the real size of the allocation if - rsize is not NULL. - - The rallocm function resizes the - allocation at *ptr to be at least - size bytes, sets *ptr to - the base address of the allocation if it moved, and sets - *rsize to the real size of the allocation if - rsize is not NULL. If - extra is non-zero, an attempt is made to resize - the allocation to be at least size + - extra) bytes, though inability to allocate - the extra byte(s) will not by itself result in failure. Behavior is - undefined if (size + - extra > - SIZE_T_MAX). - - The sallocm function sets - *rsize to the real size of the allocation. - - The dallocm function causes the - memory referenced by ptr to be made available for - future allocations. - - - - TUNING - Once, when the first call is made to one of the memory allocation - routines, the allocator initializes its internals based in part on various - options that can be specified at compile- or run-time. - - The string pointed to by the global variable - malloc_conf, the “name” of the file - referenced by the symbolic link named /etc/malloc.conf, and the value of the - environment variable MALLOC_CONF, will be interpreted, in - that order, from left to right as options. - - An options string is a comma-separated list of option:value pairs. - There is one key corresponding to each opt.* mallctl (see the section for options - documentation). For example, abort:true,narenas:1 sets - the opt.abort and opt.narenas options. Some - options have boolean values (true/false), others have integer values (base - 8, 10, or 16, depending on prefix), and yet others have raw string - values. - - - IMPLEMENTATION NOTES - Traditionally, allocators have used - sbrk - 2 to obtain memory, which is - suboptimal for several reasons, including race conditions, increased - fragmentation, and artificial limitations on maximum usable memory. If - is specified during configuration, this - allocator uses both sbrk - 2 and - mmap - 2, in that order of preference; - otherwise only mmap - 2 is used. - - This allocator uses multiple arenas in order to reduce lock - contention for threaded programs on multi-processor systems. This works - well with regard to threading scalability, but incurs some costs. There is - a small fixed per-arena overhead, and additionally, arenas manage memory - completely independently of each other, which means a small fixed increase - in overall memory fragmentation. These overheads are not generally an - issue, given the number of arenas normally used. Note that using - substantially more arenas than the default is not likely to improve - performance, mainly due to reduced cache performance. However, it may make - sense to reduce the number of arenas if an application does not make much - use of the allocation functions. - - In addition to multiple arenas, unless - is specified during configuration, this - allocator supports thread-specific caching for small and large objects, in - order to make it possible to completely avoid synchronization for most - allocation requests. Such caching allows very fast allocation in the - common case, but it increases memory usage and fragmentation, since a - bounded number of objects can remain allocated in each thread cache. - - Memory is conceptually broken into equal-sized chunks, where the - chunk size is a power of two that is greater than the page size. Chunks - are always aligned to multiples of the chunk size. This alignment makes it - possible to find metadata for user objects very quickly. - - User objects are broken into three categories according to size: - small, large, and huge. Small objects are smaller than one page. Large - objects are smaller than the chunk size. Huge objects are a multiple of - the chunk size. Small and large objects are managed by arenas; huge - objects are managed separately in a single data structure that is shared by - all threads. Huge objects are used by applications infrequently enough - that this single data structure is not a scalability issue. - - Each chunk that is managed by an arena tracks its contents as runs of - contiguous pages (unused, backing a set of small objects, or backing one - large object). The combination of chunk alignment and chunk page maps - makes it possible to determine all metadata regarding small and large - allocations in constant time. - - Small objects are managed in groups by page runs. Each run maintains - a frontier and free list to track which regions are in use. Unless - is specified during configuration, - allocation requests that are no more than half the quantum (8 or 16, - depending on architecture) are rounded up to the nearest power of two that - is at least sizeof(void *). - Allocation requests that are more than half the quantum, but no more than - the minimum cacheline-multiple size class (see the opt.lg_qspace_max - option) are rounded up to the nearest multiple of the quantum. Allocation - requests that are more than the minimum cacheline-multiple size class, but - no more than the minimum subpage-multiple size class (see the opt.lg_cspace_max - option) are rounded up to the nearest multiple of the cacheline size (64). - Allocation requests that are more than the minimum subpage-multiple size - class, but no more than the maximum subpage-multiple size class are rounded - up to the nearest multiple of the subpage size (256). Allocation requests - that are more than the maximum subpage-multiple size class, but small - enough to fit in an arena-managed chunk (see the opt.lg_chunk option), are - rounded up to the nearest run size. Allocation requests that are too large - to fit in an arena-managed chunk are rounded up to the nearest multiple of - the chunk size. - - Allocations are packed tightly together, which can be an issue for - multi-threaded applications. If you need to assure that allocations do not - suffer from cacheline sharing, round your allocation requests up to the - nearest multiple of the cacheline size, or specify cacheline alignment when - allocating. - - Assuming 4 MiB chunks, 4 KiB pages, and a 16-byte quantum on a 64-bit - system, the size classes in each category are as shown in . - -
- Size classes - - - - - - - Category - Subcategory - Size - - - - - Small - Tiny - [8] - - - Quantum-spaced - [16, 32, 48, ..., 128] - - - Cacheline-spaced - [192, 256, 320, ..., 512] - - - Subpage-spaced - [768, 1024, 1280, ..., 3840] - - - Large - [4 KiB, 8 KiB, 12 KiB, ..., 4072 KiB] - - - Huge - [4 MiB, 8 MiB, 12 MiB, ...] - - - -
- - - MALLCTL NAMESPACE - The following names are defined in the namespace accessible via the - mallctl* functions. Value types are - specified in parentheses, their readable/writable statuses are encoded as - rw, r-, -w, or - --, and required build configuration flags follow, if - any. A name element encoded as <i> or - <j> indicates an integer component, where the - integer varies from 0 to some upper value that must be determined via - introspection. In the case of stats.arenas.<i>.*, - <i> equal to arenas.narenas can be - used to access the summation of statistics from all arenas. Take special - note of the epoch mallctl, - which controls refreshing of cached dynamic statistics. - - - - - version - (const char *) - r- - - Return the jemalloc version string. - - - - - epoch - (uint64_t) - rw - - If a value is passed in, refresh the data from which - the mallctl* functions report values, - and increment the epoch. Return the current epoch. This is useful for - detecting whether another thread caused a refresh. - - - - - config.debug - (bool) - r- - - was specified during - build configuration. - - - - - config.dss - (bool) - r- - - was specified during - build configuration. - - - - - config.dynamic_page_shift - (bool) - r- - - was - specified during build configuration. - - - - - config.fill - (bool) - r- - - was specified during - build configuration. - - - - - config.lazy_lock - (bool) - r- - - was specified - during build configuration. - - - - - config.prof - (bool) - r- - - was specified during - build configuration. - - - - - config.prof_libgcc - (bool) - r- - - was not - specified during build configuration. - - - - - config.prof_libunwind - (bool) - r- - - was specified - during build configuration. - - - - - config.stats - (bool) - r- - - was specified during - build configuration. - - - - - config.swap - (bool) - r- - - was specified during - build configuration. - - - - - config.sysv - (bool) - r- - - was specified during - build configuration. - - - - - config.tcache - (bool) - r- - - was not specified - during build configuration. - - - - - config.tiny - (bool) - r- - - was not specified - during build configuration. - - - - - config.tls - (bool) - r- - - was not specified during - build configuration. - - - - - config.xmalloc - (bool) - r- - - was specified during - build configuration. - - - - - opt.abort - (bool) - r- - - Abort-on-warning enabled/disabled. If true, most - warnings are fatal. The process will call - abort - 3 in these cases. This option is - disabled by default unless is - specified during configuration, in which case it is enabled by default. - - - - - - opt.lg_qspace_max - (size_t) - r- - - Size (log base 2) of the maximum size class that is a - multiple of the quantum (8 or 16 bytes, depending on architecture). - Above this size, cacheline spacing is used for size classes. The - default value is 128 bytes (2^7). - - - - - opt.lg_cspace_max - (size_t) - r- - - Size (log base 2) of the maximum size class that is a - multiple of the cacheline size (64). Above this size, subpage spacing - (256 bytes) is used for size classes. The default value is 512 bytes - (2^9). - - - - - opt.lg_chunk - (size_t) - r- - - Virtual memory chunk size (log base 2). The default - chunk size is 4 MiB (2^22). - - - - - opt.narenas - (size_t) - r- - - Maximum number of arenas to use. The default maximum - number of arenas is four times the number of CPUs, or one if there is a - single CPU. - - - - - opt.lg_dirty_mult - (ssize_t) - r- - - Per-arena minimum ratio (log base 2) of active to dirty - pages. Some dirty unused pages may be allowed to accumulate, within - the limit set by the ratio (or one chunk worth of dirty pages, - whichever is greater), before informing the kernel about some of those - pages via madvise - 2 or a similar system call. This - provides the kernel with sufficient information to recycle dirty pages - if physical memory becomes scarce and the pages remain unused. The - default minimum ratio is 32:1 (2^5:1); an option value of -1 will - disable dirty page purging. - - - - - opt.stats_print - (bool) - r- - - Enable/disable statistics printing at exit. If - enabled, the malloc_stats_print - function is called at program exit via an - atexit - 3 function. If - is specified during configuration, this - has the potential to cause deadlock for a multi-threaded process that - exits while one or more threads are executing in the memory allocation - functions. Therefore, this option should only be used with care; it is - primarily intended as a performance tuning aid during application - development. This option is disabled by default. - - - - - opt.junk - (bool) - r- - [] - - Junk filling enabled/disabled. If enabled, each byte - of uninitialized allocated memory will be initialized to - 0xa5. All deallocated memory will be initialized to - 0x5a. This is intended for debugging and will - impact performance negatively. This option is disabled by default - unless is specified during - configuration, in which case it is enabled by default. - - - - - opt.zero - (bool) - r- - [] - - Zero filling enabled/disabled. If enabled, each byte - of uninitialized allocated memory will be initialized to 0. Note that - this initialization only happens once for each byte, so - realloc and - rallocm calls do not zero memory that - was previously allocated. This is intended for debugging and will - impact performance negatively. This option is disabled by default. - - - - - - opt.sysv - (bool) - r- - [] - - If enabled, attempting to allocate zero bytes will - return a NULL pointer instead of a valid pointer. - (The default behavior is to make a minimal allocation and return a - pointer to it.) This option is provided for System V compatibility. - This option is incompatible with the opt.xmalloc option. - This option is disabled by default. - - - - - opt.xmalloc - (bool) - r- - [] - - Abort-on-out-of-memory enabled/disabled. If enabled, - rather than returning failure for any allocation function, display a - diagnostic message on STDERR_FILENO and cause the - program to drop core (using - abort - 3). If an application is - designed to depend on this behavior, set the option at compile time by - including the following in the source code: - - This option is disabled by default. - - - - - opt.tcache - (bool) - r- - [] - - Thread-specific caching enabled/disabled. When there - are multiple threads, each thread uses a thread-specific cache for - objects up to a certain size. Thread-specific caching allows many - allocations to be satisfied without performing any thread - synchronization, at the cost of increased memory use. See the - opt.lg_tcache_gc_sweep - and opt.lg_tcache_max - options for related tuning information. This option is enabled by - default. - - - - - opt.lg_tcache_gc_sweep - (ssize_t) - r- - [] - - Approximate interval (log base 2) between full - thread-specific cache garbage collection sweeps, counted in terms of - thread-specific cache allocation/deallocation events. Garbage - collection is actually performed incrementally, one size class at a - time, in order to avoid large collection pauses. The default sweep - interval is 8192 (2^13); setting this option to -1 will disable garbage - collection. - - - - - opt.lg_tcache_max - (size_t) - r- - [] - - Maximum size class (log base 2) to cache in the - thread-specific cache. At a minimum, all small size classes are - cached, and at a maximum all large size classes are cached. The - default maximum is 32 KiB (2^15). - - - - - opt.prof - (bool) - r- - [] - - Memory profiling enabled/disabled. If enabled, profile - memory allocation activity, and use an - atexit - 3 function to dump final memory - usage to a file named according to the pattern - <prefix>.<pid>.<seq>.f.heap, - where <prefix> is controlled by the opt.prof_prefix - option. See the opt.lg_prof_bt_max - option for backtrace depth control. See the opt.prof_active - option for on-the-fly activation/deactivation. See the opt.lg_prof_sample - option for probabilistic sampling control. See the opt.prof_accum - option for control of cumulative sample reporting. See the opt.lg_prof_tcmax - option for control of per thread backtrace caching. See the opt.lg_prof_interval - option for information on interval-triggered profile dumping, and the - opt.prof_gdump - option for information on high-water-triggered profile dumping. - Profile output is compatible with the included pprof - Perl script, which originates from the google-perftools - package. - - - - - opt.prof_prefix - (const char *) - r- - [] - - Filename prefix for profile dumps. If the prefix is - set to the empty string, no automatic dumps will occur; this is - primarily useful for disabling the automatic final heap dump (which - also disables leak reporting, if enabled). The default prefix is - jeprof. - - - - - opt.lg_prof_bt_max - (size_t) - r- - [] - - Maximum backtrace depth (log base 2) when profiling - memory allocation activity. The default is 128 (2^7). - - - - - opt.prof_active - (bool) - r- - [] - - Profiling activated/deactivated. This is a secondary - control mechanism that makes it possible to start the application with - profiling enabled (see the opt.prof option) but - inactive, then toggle profiling at any time during program execution - with the prof.active mallctl. - This option is enabled by default. - - - - - opt.lg_prof_sample - (ssize_t) - r- - [] - - Average interval (log base 2) between allocation - samples, as measured in bytes of allocation activity. Increasing the - sampling interval decreases profile fidelity, but also decreases the - computational overhead. The default sample interval is 1 (2^0) (i.e. - all allocations are sampled). - - - - - opt.prof_accum - (bool) - r- - [] - - Reporting of cumulative object/byte counts in profile - dumps enabled/disabled. If this option is enabled, every unique - backtrace must be stored for the duration of execution. Depending on - the application, this can impose a large memory overhead, and the - cumulative counts are not always of interest. See the - opt.lg_prof_tcmax - option for control of per thread backtrace caching, which has important - interactions. This option is enabled by default. - - - - - opt.lg_prof_tcmax - (ssize_t) - r- - [] - - Maximum per thread backtrace cache (log base 2) used - for heap profiling. A backtrace can only be discarded if the - opt.prof_accum - option is disabled, and no thread caches currently refer to the - backtrace. Therefore, a backtrace cache limit should be imposed if the - intention is to limit how much memory is used by backtraces. By - default, no limit is imposed (encoded as -1). - - - - - - opt.lg_prof_interval - (ssize_t) - r- - [] - - Average interval (log base 2) between memory profile - dumps, as measured in bytes of allocation activity. The actual - interval between dumps may be sporadic because decentralized allocation - counters are used to avoid synchronization bottlenecks. Profiles are - dumped to files named according to the pattern - <prefix>.<pid>.<seq>.i<iseq>.heap, - where <prefix> is controlled by the - opt.prof_prefix - option. By default, interval-triggered profile dumping is disabled - (encoded as -1). - - - - - - opt.prof_gdump - (bool) - r- - [] - - Trigger a memory profile dump every time the total - virtual memory exceeds the previous maximum. Profiles are dumped to - files named according to the pattern - <prefix>.<pid>.<seq>.u<useq>.heap, - where <prefix> is controlled by the opt.prof_prefix - option. This option is disabled by default. - - - - - opt.prof_leak - (bool) - r- - [] - - Leak reporting enabled/disabled. If enabled, use an - atexit - 3 function to report memory leaks - detected by allocation sampling. See the - opt.lg_prof_bt_max - option for backtrace depth control. See the - opt.prof option for - information on analyzing heap profile output. This option is disabled - by default. - - - - - opt.overcommit - (bool) - r- - [] - - Over-commit enabled/disabled. If enabled, over-commit - memory as a side effect of using anonymous - mmap - 2 or - sbrk - 2 for virtual memory allocation. - In order for overcommit to be disabled, the swap.fds mallctl must have - been successfully written to. This option is enabled by - default. - - - - - tcache.flush - (void) - -- - [] - - Flush calling thread's tcache. This interface releases - all cached objects and internal data structures associated with the - calling thread's thread-specific cache. Ordinarily, this interface - need not be called, since automatic periodic incremental garbage - collection occurs, and the thread cache is automatically discarded when - a thread exits. However, garbage collection is triggered by allocation - activity, so it is possible for a thread that stops - allocating/deallocating to retain its cache indefinitely, in which case - the developer may find manual flushing useful. - - - - - thread.arena - (unsigned) - rw - - Get or set the arena associated with the calling - thread. The arena index must be less than the maximum number of arenas - (see the arenas.narenas - mallctl). If the specified arena was not initialized beforehand (see - the arenas.initialized - mallctl), it will be automatically initialized as a side effect of - calling this interface. - - - - - thread.allocated - (uint64_t) - r- - [] - - Get the total number of bytes ever allocated by the - calling thread. This counter has the potential to wrap around; it is - up to the application to appropriately interpret the counter in such - cases. - - - - - thread.allocatedp - (uint64_t *) - r- - [] - - Get a pointer to the the value that is returned by the - thread.allocated - mallctl. This is useful for avoiding the overhead of repeated - mallctl* calls. - - - - - thread.deallocated - (uint64_t) - r- - [] - - Get the total number of bytes ever deallocated by the - calling thread. This counter has the potential to wrap around; it is - up to the application to appropriately interpret the counter in such - cases. - - - - - thread.deallocatedp - (uint64_t *) - r- - [] - - Get a pointer to the the value that is returned by the - thread.deallocated - mallctl. This is useful for avoiding the overhead of repeated - mallctl* calls. - - - - - arenas.narenas - (unsigned) - r- - - Maximum number of arenas. - - - - - arenas.initialized - (bool *) - r- - - An array of arenas.narenas - booleans. Each boolean indicates whether the corresponding arena is - initialized. - - - - - arenas.quantum - (size_t) - r- - - Quantum size. - - - - - arenas.cacheline - (size_t) - r- - - Assumed cacheline size. - - - - - arenas.subpage - (size_t) - r- - - Subpage size class interval. - - - - - arenas.pagesize - (size_t) - r- - - Page size. - - - - - arenas.chunksize - (size_t) - r- - - Chunk size. - - - - - arenas.tspace_min - (size_t) - r- - - Minimum tiny size class. Tiny size classes are powers - of two. - - - - - arenas.tspace_max - (size_t) - r- - - Maximum tiny size class. Tiny size classes are powers - of two. - - - - - arenas.qspace_min - (size_t) - r- - - Minimum quantum-spaced size class. - - - - - arenas.qspace_max - (size_t) - r- - - Maximum quantum-spaced size class. - - - - - arenas.cspace_min - (size_t) - r- - - Minimum cacheline-spaced size class. - - - - - arenas.cspace_max - (size_t) - r- - - Maximum cacheline-spaced size class. - - - - - arenas.sspace_min - (size_t) - r- - - Minimum subpage-spaced size class. - - - - - arenas.sspace_max - (size_t) - r- - - Maximum subpage-spaced size class. - - - - - arenas.tcache_max - (size_t) - r- - [] - - Maximum thread-cached size class. - - - - - arenas.ntbins - (unsigned) - r- - - Number of tiny bin size classes. - - - - - arenas.nqbins - (unsigned) - r- - - Number of quantum-spaced bin size - classes. - - - - - arenas.ncbins - (unsigned) - r- - - Number of cacheline-spaced bin size - classes. - - - - - arenas.nsbins - (unsigned) - r- - - Number of subpage-spaced bin size - classes. - - - - - arenas.nbins - (unsigned) - r- - - Total number of bin size classes. - - - - - arenas.nhbins - (unsigned) - r- - [] - - Total number of thread cache bin size - classes. - - - - - arenas.bin.<i>.size - (size_t) - r- - - Maximum size supported by size class. - - - - - arenas.bin.<i>.nregs - (uint32_t) - r- - - Number of regions per page run. - - - - - arenas.bin.<i>.run_size - (size_t) - r- - - Number of bytes per page run. - - - - - arenas.nlruns - (size_t) - r- - - Total number of large size classes. - - - - - arenas.lrun.<i>.size - (size_t) - r- - - Maximum size supported by this large size - class. - - - - - arenas.purge - (unsigned) - -w - - Purge unused dirty pages for the specified arena, or - for all arenas if none is specified. - - - - - prof.active - (bool) - rw - [] - - Control whether sampling is currently active. See the - opt.prof_active - option for additional information. - - - - - - prof.dump - (const char *) - -w - [] - - Dump a memory profile to the specified file, or if NULL - is specified, to a file according to the pattern - <prefix>.<pid>.<seq>.m<mseq>.heap, - where <prefix> is controlled by the - opt.prof_prefix - option. - - - - - prof.interval - (uint64_t) - r- - [] - - Average number of bytes allocated between - inverval-based profile dumps. See the - opt.lg_prof_interval - option for additional information. - - - - - stats.cactive - (size_t *) - r- - [] - - Pointer to a counter that contains an approximate count - of the current number of bytes in active pages. The estimate may be - high, but never low, because each arena rounds up to the nearest - multiple of the chunk size when computing its contribution to the - counter. Note that the epoch mallctl has no bearing - on this counter. Furthermore, counter consistency is maintained via - atomic operations, so it is necessary to use an atomic operation in - order to guarantee a consistent read when dereferencing the pointer. - - - - - - stats.allocated - (size_t) - r- - [] - - Total number of bytes allocated by the - application. - - - - - stats.active - (size_t) - r- - [] - - Total number of bytes in active pages allocated by the - application. This is a multiple of the page size, and greater than or - equal to stats.allocated. - - - - - - stats.mapped - (size_t) - r- - [] - - Total number of bytes in chunks mapped on behalf of the - application. This is a multiple of the chunk size, and is at least as - large as stats.active. This - does not include inactive chunks backed by swap files. his does not - include inactive chunks embedded in the DSS. - - - - - stats.chunks.current - (size_t) - r- - [] - - Total number of chunks actively mapped on behalf of the - application. This does not include inactive chunks backed by swap - files. This does not include inactive chunks embedded in the DSS. - - - - - - stats.chunks.total - (uint64_t) - r- - [] - - Cumulative number of chunks allocated. - - - - - stats.chunks.high - (size_t) - r- - [] - - Maximum number of active chunks at any time thus far. - - - - - - stats.huge.allocated - (size_t) - r- - [] - - Number of bytes currently allocated by huge objects. - - - - - - stats.huge.nmalloc - (uint64_t) - r- - [] - - Cumulative number of huge allocation requests. - - - - - - stats.huge.ndalloc - (uint64_t) - r- - [] - - Cumulative number of huge deallocation requests. - - - - - - stats.arenas.<i>.nthreads - (unsigned) - r- - - Number of threads currently assigned to - arena. - - - - - stats.arenas.<i>.pactive - (size_t) - r- - - Number of pages in active runs. - - - - - stats.arenas.<i>.pdirty - (size_t) - r- - - Number of pages within unused runs that are potentially - dirty, and for which madvise... - MADV_DONTNEED or - similar has not been called. - - - - - stats.arenas.<i>.mapped - (size_t) - r- - [] - - Number of mapped bytes. - - - - - stats.arenas.<i>.npurge - (uint64_t) - r- - [] - - Number of dirty page purge sweeps performed. - - - - - - stats.arenas.<i>.nmadvise - (uint64_t) - r- - [] - - Number of madvise... - MADV_DONTNEED or - similar calls made to purge dirty pages. - - - - - stats.arenas.<i>.npurged - (uint64_t) - r- - [] - - Number of pages purged. - - - - - stats.arenas.<i>.small.allocated - (size_t) - r- - [] - - Number of bytes currently allocated by small objects. - - - - - - stats.arenas.<i>.small.nmalloc - (uint64_t) - r- - [] - - Cumulative number of allocation requests served by - small bins. - - - - - stats.arenas.<i>.small.ndalloc - (uint64_t) - r- - [] - - Cumulative number of small objects returned to bins. - - - - - - stats.arenas.<i>.small.nrequests - (uint64_t) - r- - [] - - Cumulative number of small allocation requests. - - - - - - stats.arenas.<i>.large.allocated - (size_t) - r- - [] - - Number of bytes currently allocated by large objects. - - - - - - stats.arenas.<i>.large.nmalloc - (uint64_t) - r- - [] - - Cumulative number of large allocation requests served - directly by the arena. - - - - - stats.arenas.<i>.large.ndalloc - (uint64_t) - r- - [] - - Cumulative number of large deallocation requests served - directly by the arena. - - - - - stats.arenas.<i>.large.nrequests - (uint64_t) - r- - [] - - Cumulative number of large allocation requests. - - - - - - stats.arenas.<i>.bins.<j>.allocated - (size_t) - r- - [] - - Current number of bytes allocated by - bin. - - - - - stats.arenas.<i>.bins.<j>.nmalloc - (uint64_t) - r- - [] - - Cumulative number of allocations served by bin. - - - - - - stats.arenas.<i>.bins.<j>.ndalloc - (uint64_t) - r- - [] - - Cumulative number of allocations returned to bin. - - - - - - stats.arenas.<i>.bins.<j>.nrequests - (uint64_t) - r- - [] - - Cumulative number of allocation - requests. - - - - - stats.arenas.<i>.bins.<j>.nfills - (uint64_t) - r- - [ ] - - Cumulative number of tcache fills. - - - - - stats.arenas.<i>.bins.<j>.nflushes - (uint64_t) - r- - [ ] - - Cumulative number of tcache flushes. - - - - - stats.arenas.<i>.bins.<j>.nruns - (uint64_t) - r- - [] - - Cumulative number of runs created. - - - - - stats.arenas.<i>.bins.<j>.nreruns - (uint64_t) - r- - [] - - Cumulative number of times the current run from which - to allocate changed. - - - - - stats.arenas.<i>.bins.<j>.highruns - (size_t) - r- - [] - - Maximum number of runs at any time thus far. - - - - - - stats.arenas.<i>.bins.<j>.curruns - (size_t) - r- - [] - - Current number of runs. - - - - - stats.arenas.<i>.lruns.<j>.nmalloc - (uint64_t) - r- - [] - - Cumulative number of allocation requests for this size - class served directly by the arena. - - - - - stats.arenas.<i>.lruns.<j>.ndalloc - (uint64_t) - r- - [] - - Cumulative number of deallocation requests for this - size class served directly by the arena. - - - - - stats.arenas.<i>.lruns.<j>.nrequests - (uint64_t) - r- - [] - - Cumulative number of allocation requests for this size - class. - - - - - stats.arenas.<i>.lruns.<j>.highruns - (size_t) - r- - [] - - Maximum number of runs at any time thus far for this - size class. - - - - - stats.arenas.<i>.lruns.<j>.curruns - (size_t) - r- - [] - - Current number of runs for this size class. - - - - - - swap.avail - (size_t) - r- - [] - - Number of swap file bytes that are currently not - associated with any chunk (i.e. mapped, but otherwise completely - unmanaged). - - - - - swap.prezeroed - (bool) - rw - [] - - If true, the allocator assumes that the swap file(s) - contain nothing but nil bytes. If this assumption is violated, - allocator behavior is undefined. This value becomes read-only after - swap.fds is - successfully written to. - - - - - swap.nfds - (size_t) - r- - [] - - Number of file descriptors in use for swap. - - - - - - swap.fds - (int *) - rw - [] - - When written to, the files associated with the - specified file descriptors are contiguously mapped via - mmap - 2. The resulting virtual memory - region is preferred over anonymous - mmap - 2 and - sbrk - 2 memory. Note that if a file's - size is not a multiple of the page size, it is automatically truncated - to the nearest page size multiple. See the - swap.prezeroed - mallctl for specifying that the files are pre-zeroed. - - - - - DEBUGGING MALLOC PROBLEMS - When debugging, it is a good idea to configure/build jemalloc with - the and - options, and recompile the program with suitable options and symbols for - debugger support. When so configured, jemalloc incorporates a wide variety - of run-time assertions that catch application errors such as double-free, - write-after-free, etc. - - Programs often accidentally depend on “uninitialized” - memory actually being filled with zero bytes. Junk filling - (see the opt.junk - option) tends to expose such bugs in the form of obviously incorrect - results and/or coredumps. Conversely, zero - filling (see the opt.zero option) eliminates - the symptoms of such bugs. Between these two options, it is usually - possible to quickly detect, diagnose, and eliminate such bugs. - - This implementation does not provide much detail about the problems - it detects, because the performance impact for storing such information - would be prohibitive. There are a number of allocator implementations - available on the Internet which focus on detecting and pinpointing problems - by trading performance for extra sanity checks and detailed - diagnostics. - - - DIAGNOSTIC MESSAGES - If any of the memory allocation/deallocation functions detect an - error or warning condition, a message will be printed to file descriptor - STDERR_FILENO. Errors will result in the process - dumping core. If the opt.abort option is set, most - warnings are treated as errors. - - The malloc_message variable allows the programmer - to override the function which emits the text strings forming the errors - and warnings if for some reason the STDERR_FILENO file - descriptor is not suitable for this. - malloc_message takes the - cbopaque pointer argument that is - NULL unless overridden by the arguments in a call to - malloc_stats_print, followed by a string - pointer. Please note that doing anything which tries to allocate memory in - this function is likely to result in a crash or deadlock. - - All messages are prefixed by - “<jemalloc>: ”. - - - RETURN VALUES - - Standard API - The malloc and - calloc functions return a pointer to the - allocated memory if successful; otherwise a NULL - pointer is returned and errno is set to - ENOMEM. - - The posix_memalign function - returns the value 0 if successful; otherwise it returns an error value. - The posix_memalign function will fail - if: - - - EINVAL - - The alignment parameter is - not a power of 2 at least as large as - sizeof(void *). - - - - ENOMEM - - Memory allocation error. - - - - - The realloc function returns a - pointer, possibly identical to ptr, to the - allocated memory if successful; otherwise a NULL - pointer is returned, and errno is set to - ENOMEM if the error was the result of an - allocation failure. The realloc - function always leaves the original buffer intact when an error occurs. - - - The free function returns no - value. - - - Non-standard API - The malloc_usable_size function - returns the usable size of the allocation pointed to by - ptr. - - The mallctl, - mallctlnametomib, and - mallctlbymib functions return 0 on - success; otherwise they return an error value. The functions will fail - if: - - - EINVAL - - newp is not - NULL, and newlen is too - large or too small. Alternatively, *oldlenp - is too large or too small; in this case as much data as possible - are read despite the error. - - - ENOMEM - - *oldlenp is too short to - hold the requested value. - - - ENOENT - - name or - mib specifies an unknown/invalid - value. - - - EPERM - - Attempt to read or write void value, or attempt to - write read-only value. - - - EAGAIN - - A memory allocation failure - occurred. - - - EFAULT - - An interface with side effects failed in some way - not directly related to mallctl* - read/write processing. - - - - - - Experimental API - The allocm, - rallocm, - sallocm, and - dallocm functions return - ALLOCM_SUCCESS on success; otherwise they return an - error value. The allocm and - rallocm functions will fail if: - - - ALLOCM_ERR_OOM - - Out of memory. Insufficient contiguous memory was - available to service the allocation request. The - allocm function additionally sets - *ptr to NULL, whereas - the rallocm function leaves - *ptr unmodified. - - - The rallocm function will also - fail if: - - - ALLOCM_ERR_NOT_MOVED - - ALLOCM_NO_MOVE was specified, - but the reallocation request could not be serviced without moving - the object. - - - - - - - ENVIRONMENT - The following environment variable affects the execution of the - allocation functions: - - - MALLOC_CONF - - If the environment variable - MALLOC_CONF is set, the characters it contains - will be interpreted as options. - - - - - - EXAMPLES - To dump core whenever a problem occurs: - ln -s 'abort:true' /etc/malloc.conf - - To specify in the source a chunk size that is 16 MiB: - - - - SEE ALSO - madvise - 2, - mmap - 2, - sbrk - 2, - alloca - 3, - atexit - 3, - getpagesize - 3 - - - STANDARDS - The malloc, - calloc, - realloc, and - free functions conform to ISO/IEC - 9899:1990 (“ISO C90”). - - The posix_memalign function conforms - to IEEE Std 1003.1-2001 (“POSIX.1”). - - diff --git a/deps/jemalloc.orig/doc/manpages.xsl.in b/deps/jemalloc.orig/doc/manpages.xsl.in deleted file mode 100644 index 88b2626b958..00000000000 --- a/deps/jemalloc.orig/doc/manpages.xsl.in +++ /dev/null @@ -1,4 +0,0 @@ - - - - diff --git a/deps/jemalloc.orig/doc/stylesheet.xsl b/deps/jemalloc.orig/doc/stylesheet.xsl deleted file mode 100644 index 4e334a86f87..00000000000 --- a/deps/jemalloc.orig/doc/stylesheet.xsl +++ /dev/null @@ -1,7 +0,0 @@ - - ansi - - - "" - - diff --git a/deps/jemalloc.orig/include/jemalloc/internal/arena.h b/deps/jemalloc.orig/include/jemalloc/internal/arena.h deleted file mode 100644 index b80c118d811..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/arena.h +++ /dev/null @@ -1,743 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -/* - * Subpages are an artificially designated partitioning of pages. Their only - * purpose is to support subpage-spaced size classes. - * - * There must be at least 4 subpages per page, due to the way size classes are - * handled. - */ -#define LG_SUBPAGE 8 -#define SUBPAGE ((size_t)(1U << LG_SUBPAGE)) -#define SUBPAGE_MASK (SUBPAGE - 1) - -/* Return the smallest subpage multiple that is >= s. */ -#define SUBPAGE_CEILING(s) \ - (((s) + SUBPAGE_MASK) & ~SUBPAGE_MASK) - -#ifdef JEMALLOC_TINY - /* Smallest size class to support. */ -# define LG_TINY_MIN LG_SIZEOF_PTR -# define TINY_MIN (1U << LG_TINY_MIN) -#endif - -/* - * Maximum size class that is a multiple of the quantum, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ -#define LG_QSPACE_MAX_DEFAULT 7 - -/* - * Maximum size class that is a multiple of the cacheline, but not (necessarily) - * a power of 2. Above this size, allocations are rounded up to the nearest - * power of 2. - */ -#define LG_CSPACE_MAX_DEFAULT 9 - -/* - * RUN_MAX_OVRHD indicates maximum desired run header overhead. Runs are sized - * as small as possible such that this setting is still honored, without - * violating other constraints. The goal is to make runs as small as possible - * without exceeding a per run external fragmentation threshold. - * - * We use binary fixed point math for overhead computations, where the binary - * point is implicitly RUN_BFP bits to the left. - * - * Note that it is possible to set RUN_MAX_OVRHD low enough that it cannot be - * honored for some/all object sizes, since when heap profiling is enabled - * there is one pointer of header overhead per object (plus a constant). This - * constraint is relaxed (ignored) for runs that are so small that the - * per-region overhead is greater than: - * - * (RUN_MAX_OVRHD / (reg_size << (3+RUN_BFP)) - */ -#define RUN_BFP 12 -/* \/ Implicit binary fixed point. */ -#define RUN_MAX_OVRHD 0x0000003dU -#define RUN_MAX_OVRHD_RELAX 0x00001800U - -/* Maximum number of regions in one run. */ -#define LG_RUN_MAXREGS 11 -#define RUN_MAXREGS (1U << LG_RUN_MAXREGS) - -/* - * The minimum ratio of active:dirty pages per arena is computed as: - * - * (nactive >> opt_lg_dirty_mult) >= ndirty - * - * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 - * times as many active pages as dirty pages. - */ -#define LG_DIRTY_MULT_DEFAULT 5 - -typedef struct arena_chunk_map_s arena_chunk_map_t; -typedef struct arena_chunk_s arena_chunk_t; -typedef struct arena_run_s arena_run_t; -typedef struct arena_bin_info_s arena_bin_info_t; -typedef struct arena_bin_s arena_bin_t; -typedef struct arena_s arena_t; - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -/* Each element of the chunk map corresponds to one page within the chunk. */ -struct arena_chunk_map_s { - union { - /* - * Linkage for run trees. There are two disjoint uses: - * - * 1) arena_t's runs_avail_{clean,dirty} trees. - * 2) arena_run_t conceptually uses this linkage for in-use - * non-full runs, rather than directly embedding linkage. - */ - rb_node(arena_chunk_map_t) rb_link; - /* - * List of runs currently in purgatory. arena_chunk_purge() - * temporarily allocates runs that contain dirty pages while - * purging, so that other threads cannot use the runs while the - * purging thread is operating without the arena lock held. - */ - ql_elm(arena_chunk_map_t) ql_link; - } u; - -#ifdef JEMALLOC_PROF - /* Profile counters, used for large object runs. */ - prof_ctx_t *prof_ctx; -#endif - - /* - * Run address (or size) and various flags are stored together. The bit - * layout looks like (assuming 32-bit system): - * - * ???????? ???????? ????---- ----dula - * - * ? : Unallocated: Run address for first/last pages, unset for internal - * pages. - * Small: Run page offset. - * Large: Run size for first page, unset for trailing pages. - * - : Unused. - * d : dirty? - * u : unzeroed? - * l : large? - * a : allocated? - * - * Following are example bit patterns for the three types of runs. - * - * p : run page offset - * s : run size - * c : (binind+1) for size class (used only if prof_promote is true) - * x : don't care - * - : 0 - * + : 1 - * [DULA] : bit set - * [dula] : bit unset - * - * Unallocated (clean): - * ssssssss ssssssss ssss---- ----du-a - * xxxxxxxx xxxxxxxx xxxx---- -----Uxx - * ssssssss ssssssss ssss---- ----dU-a - * - * Unallocated (dirty): - * ssssssss ssssssss ssss---- ----D--a - * xxxxxxxx xxxxxxxx xxxx---- ----xxxx - * ssssssss ssssssss ssss---- ----D--a - * - * Small: - * pppppppp pppppppp pppp---- ----d--A - * pppppppp pppppppp pppp---- -------A - * pppppppp pppppppp pppp---- ----d--A - * - * Large: - * ssssssss ssssssss ssss---- ----D-LA - * xxxxxxxx xxxxxxxx xxxx---- ----xxxx - * -------- -------- -------- ----D-LA - * - * Large (sampled, size <= PAGE_SIZE): - * ssssssss ssssssss sssscccc ccccD-LA - * - * Large (not sampled, size == PAGE_SIZE): - * ssssssss ssssssss ssss---- ----D-LA - */ - size_t bits; -#ifdef JEMALLOC_PROF -#define CHUNK_MAP_CLASS_SHIFT 4 -#define CHUNK_MAP_CLASS_MASK ((size_t)0xff0U) -#endif -#define CHUNK_MAP_FLAGS_MASK ((size_t)0xfU) -#define CHUNK_MAP_DIRTY ((size_t)0x8U) -#define CHUNK_MAP_UNZEROED ((size_t)0x4U) -#define CHUNK_MAP_LARGE ((size_t)0x2U) -#define CHUNK_MAP_ALLOCATED ((size_t)0x1U) -#define CHUNK_MAP_KEY CHUNK_MAP_ALLOCATED -}; -typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; -typedef rb_tree(arena_chunk_map_t) arena_run_tree_t; - -/* Arena chunk header. */ -struct arena_chunk_s { - /* Arena that owns the chunk. */ - arena_t *arena; - - /* Linkage for the arena's chunks_dirty list. */ - ql_elm(arena_chunk_t) link_dirty; - - /* - * True if the chunk is currently in the chunks_dirty list, due to - * having at some point contained one or more dirty pages. Removal - * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. - */ - bool dirtied; - - /* Number of dirty pages. */ - size_t ndirty; - - /* - * Map of pages within chunk that keeps track of free/large/small. The - * first map_bias entries are omitted, since the chunk header does not - * need to be tracked in the map. This omission saves a header page - * for common chunk sizes (e.g. 4 MiB). - */ - arena_chunk_map_t map[1]; /* Dynamically sized. */ -}; -typedef rb_tree(arena_chunk_t) arena_chunk_tree_t; - -struct arena_run_s { -#ifdef JEMALLOC_DEBUG - uint32_t magic; -# define ARENA_RUN_MAGIC 0x384adf93 -#endif - - /* Bin this run is associated with. */ - arena_bin_t *bin; - - /* Index of next region that has never been allocated, or nregs. */ - uint32_t nextind; - - /* Number of free regions in run. */ - unsigned nfree; -}; - -/* - * Read-only information associated with each element of arena_t's bins array - * is stored separately, partly to reduce memory usage (only one copy, rather - * than one per arena), but mainly to avoid false cacheline sharing. - */ -struct arena_bin_info_s { - /* Size of regions in a run for this bin's size class. */ - size_t reg_size; - - /* Total size of a run for this bin's size class. */ - size_t run_size; - - /* Total number of regions in a run for this bin's size class. */ - uint32_t nregs; - - /* - * Offset of first bitmap_t element in a run header for this bin's size - * class. - */ - uint32_t bitmap_offset; - - /* - * Metadata used to manipulate bitmaps for runs associated with this - * bin. - */ - bitmap_info_t bitmap_info; - -#ifdef JEMALLOC_PROF - /* - * Offset of first (prof_ctx_t *) in a run header for this bin's size - * class, or 0 if (opt_prof == false). - */ - uint32_t ctx0_offset; -#endif - - /* Offset of first region in a run for this bin's size class. */ - uint32_t reg0_offset; -}; - -struct arena_bin_s { - /* - * All operations on runcur, runs, and stats require that lock be - * locked. Run allocation/deallocation are protected by the arena lock, - * which may be acquired while holding one or more bin locks, but not - * vise versa. - */ - malloc_mutex_t lock; - - /* - * Current run being used to service allocations of this bin's size - * class. - */ - arena_run_t *runcur; - - /* - * Tree of non-full runs. This tree is used when looking for an - * existing run when runcur is no longer usable. We choose the - * non-full run that is lowest in memory; this policy tends to keep - * objects packed well, and it can also help reduce the number of - * almost-empty chunks. - */ - arena_run_tree_t runs; - -#ifdef JEMALLOC_STATS - /* Bin statistics. */ - malloc_bin_stats_t stats; -#endif -}; - -struct arena_s { -#ifdef JEMALLOC_DEBUG - uint32_t magic; -# define ARENA_MAGIC 0x947d3d24 -#endif - - /* This arena's index within the arenas array. */ - unsigned ind; - - /* - * Number of threads currently assigned to this arena. This field is - * protected by arenas_lock. - */ - unsigned nthreads; - - /* - * There are three classes of arena operations from a locking - * perspective: - * 1) Thread asssignment (modifies nthreads) is protected by - * arenas_lock. - * 2) Bin-related operations are protected by bin locks. - * 3) Chunk- and run-related operations are protected by this mutex. - */ - malloc_mutex_t lock; - -#ifdef JEMALLOC_STATS - arena_stats_t stats; -# ifdef JEMALLOC_TCACHE - /* - * List of tcaches for extant threads associated with this arena. - * Stats from these are merged incrementally, and at exit. - */ - ql_head(tcache_t) tcache_ql; -# endif -#endif - -#ifdef JEMALLOC_PROF - uint64_t prof_accumbytes; -#endif - - /* List of dirty-page-containing chunks this arena manages. */ - ql_head(arena_chunk_t) chunks_dirty; - - /* - * In order to avoid rapid chunk allocation/deallocation when an arena - * oscillates right on the cusp of needing a new chunk, cache the most - * recently freed chunk. The spare is left in the arena's chunk trees - * until it is deleted. - * - * There is one spare chunk per arena, rather than one spare total, in - * order to avoid interactions between multiple threads that could make - * a single spare inadequate. - */ - arena_chunk_t *spare; - - /* Number of pages in active runs. */ - size_t nactive; - - /* - * Current count of pages within unused runs that are potentially - * dirty, and for which madvise(... MADV_DONTNEED) has not been called. - * By tracking this, we can institute a limit on how much dirty unused - * memory is mapped for each arena. - */ - size_t ndirty; - - /* - * Approximate number of pages being purged. It is possible for - * multiple threads to purge dirty pages concurrently, and they use - * npurgatory to indicate the total number of pages all threads are - * attempting to purge. - */ - size_t npurgatory; - - /* - * Size/address-ordered trees of this arena's available runs. The trees - * are used for first-best-fit run allocation. The dirty tree contains - * runs with dirty pages (i.e. very likely to have been touched and - * therefore have associated physical pages), whereas the clean tree - * contains runs with pages that either have no associated physical - * pages, or have pages that the kernel may recycle at any time due to - * previous madvise(2) calls. The dirty tree is used in preference to - * the clean tree for allocations, because using dirty pages reduces - * the amount of dirty purging necessary to keep the active:dirty page - * ratio below the purge threshold. - */ - arena_avail_tree_t runs_avail_clean; - arena_avail_tree_t runs_avail_dirty; - - /* - * bins is used to store trees of free regions of the following sizes, - * assuming a 64-bit system with 16-byte quantum, 4 KiB page size, and - * default MALLOC_CONF. - * - * bins[i] | size | - * --------+--------+ - * 0 | 8 | - * --------+--------+ - * 1 | 16 | - * 2 | 32 | - * 3 | 48 | - * : : - * 6 | 96 | - * 7 | 112 | - * 8 | 128 | - * --------+--------+ - * 9 | 192 | - * 10 | 256 | - * 11 | 320 | - * 12 | 384 | - * 13 | 448 | - * 14 | 512 | - * --------+--------+ - * 15 | 768 | - * 16 | 1024 | - * 17 | 1280 | - * : : - * 25 | 3328 | - * 26 | 3584 | - * 27 | 3840 | - * --------+--------+ - */ - arena_bin_t bins[1]; /* Dynamically sized. */ -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern size_t opt_lg_qspace_max; -extern size_t opt_lg_cspace_max; -extern ssize_t opt_lg_dirty_mult; -/* - * small_size2bin is a compact lookup table that rounds request sizes up to - * size classes. In order to reduce cache footprint, the table is compressed, - * and all accesses are via the SMALL_SIZE2BIN macro. - */ -extern uint8_t const *small_size2bin; -#define SMALL_SIZE2BIN(s) (small_size2bin[(s-1) >> LG_TINY_MIN]) - -extern arena_bin_info_t *arena_bin_info; - -/* Various bin-related settings. */ -#ifdef JEMALLOC_TINY /* Number of (2^n)-spaced tiny bins. */ -# define ntbins ((unsigned)(LG_QUANTUM - LG_TINY_MIN)) -#else -# define ntbins 0 -#endif -extern unsigned nqbins; /* Number of quantum-spaced bins. */ -extern unsigned ncbins; /* Number of cacheline-spaced bins. */ -extern unsigned nsbins; /* Number of subpage-spaced bins. */ -extern unsigned nbins; -#ifdef JEMALLOC_TINY -# define tspace_max ((size_t)(QUANTUM >> 1)) -#endif -#define qspace_min QUANTUM -extern size_t qspace_max; -extern size_t cspace_min; -extern size_t cspace_max; -extern size_t sspace_min; -extern size_t sspace_max; -#define small_maxclass sspace_max - -#define nlclasses (chunk_npages - map_bias) - -void arena_purge_all(arena_t *arena); -#ifdef JEMALLOC_PROF -void arena_prof_accum(arena_t *arena, uint64_t accumbytes); -#endif -#ifdef JEMALLOC_TCACHE -void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, - size_t binind -# ifdef JEMALLOC_PROF - , uint64_t prof_accumbytes -# endif - ); -#endif -void *arena_malloc_small(arena_t *arena, size_t size, bool zero); -void *arena_malloc_large(arena_t *arena, size_t size, bool zero); -void *arena_malloc(size_t size, bool zero); -void *arena_palloc(arena_t *arena, size_t size, size_t alloc_size, - size_t alignment, bool zero); -size_t arena_salloc(const void *ptr); -#ifdef JEMALLOC_PROF -void arena_prof_promoted(const void *ptr, size_t size); -size_t arena_salloc_demote(const void *ptr); -#endif -void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, - arena_chunk_map_t *mapelm); -void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); -#ifdef JEMALLOC_STATS -void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats); -#endif -void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); -void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); -bool arena_new(arena_t *arena, unsigned ind); -bool arena_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); -unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, - const void *ptr); -# ifdef JEMALLOC_PROF -prof_ctx_t *arena_prof_ctx_get(const void *ptr); -void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -# endif -void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) -JEMALLOC_INLINE size_t -arena_bin_index(arena_t *arena, arena_bin_t *bin) -{ - size_t binind = bin - arena->bins; - assert(binind < nbins); - return (binind); -} - -JEMALLOC_INLINE unsigned -arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr) -{ - unsigned shift, diff, regind; - size_t size; - - dassert(run->magic == ARENA_RUN_MAGIC); - /* - * Freeing a pointer lower than region zero can cause assertion - * failure. - */ - assert((uintptr_t)ptr >= (uintptr_t)run + - (uintptr_t)bin_info->reg0_offset); - - /* - * Avoid doing division with a variable divisor if possible. Using - * actual division here can reduce allocator throughput by over 20%! - */ - diff = (unsigned)((uintptr_t)ptr - (uintptr_t)run - - bin_info->reg0_offset); - - /* Rescale (factor powers of 2 out of the numerator and denominator). */ - size = bin_info->reg_size; - shift = ffs(size) - 1; - diff >>= shift; - size >>= shift; - - if (size == 1) { - /* The divisor was a power of 2. */ - regind = diff; - } else { - /* - * To divide by a number D that is not a power of two we - * multiply by (2^21 / D) and then right shift by 21 positions. - * - * X / D - * - * becomes - * - * (X * size_invs[D - 3]) >> SIZE_INV_SHIFT - * - * We can omit the first three elements, because we never - * divide by 0, and 1 and 2 are both powers of two, which are - * handled above. - */ -#define SIZE_INV_SHIFT ((sizeof(unsigned) << 3) - LG_RUN_MAXREGS) -#define SIZE_INV(s) (((1U << SIZE_INV_SHIFT) / (s)) + 1) - static const unsigned size_invs[] = { - SIZE_INV(3), - SIZE_INV(4), SIZE_INV(5), SIZE_INV(6), SIZE_INV(7), - SIZE_INV(8), SIZE_INV(9), SIZE_INV(10), SIZE_INV(11), - SIZE_INV(12), SIZE_INV(13), SIZE_INV(14), SIZE_INV(15), - SIZE_INV(16), SIZE_INV(17), SIZE_INV(18), SIZE_INV(19), - SIZE_INV(20), SIZE_INV(21), SIZE_INV(22), SIZE_INV(23), - SIZE_INV(24), SIZE_INV(25), SIZE_INV(26), SIZE_INV(27), - SIZE_INV(28), SIZE_INV(29), SIZE_INV(30), SIZE_INV(31) - }; - - if (size <= ((sizeof(size_invs) / sizeof(unsigned)) + 2)) - regind = (diff * size_invs[size - 3]) >> SIZE_INV_SHIFT; - else - regind = diff / size; -#undef SIZE_INV -#undef SIZE_INV_SHIFT - } - assert(diff == regind * size); - assert(regind < bin_info->nregs); - - return (regind); -} - -#ifdef JEMALLOC_PROF -JEMALLOC_INLINE prof_ctx_t * -arena_prof_ctx_get(const void *ptr) -{ - prof_ctx_t *ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - if (prof_promote) - ret = (prof_ctx_t *)(uintptr_t)1U; - else { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - unsigned regind; - - dassert(run->magic == ARENA_RUN_MAGIC); - regind = arena_run_regind(run, bin_info, ptr); - ret = *(prof_ctx_t **)((uintptr_t)run + - bin_info->ctx0_offset + (regind * - sizeof(prof_ctx_t *))); - } - } else - ret = chunk->map[pageind-map_bias].prof_ctx; - - return (ret); -} - -JEMALLOC_INLINE void -arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) -{ - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - if (prof_promote == false) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - arena_bin_t *bin = run->bin; - size_t binind; - arena_bin_info_t *bin_info; - unsigned regind; - - dassert(run->magic == ARENA_RUN_MAGIC); - binind = arena_bin_index(chunk->arena, bin); - bin_info = &arena_bin_info[binind]; - regind = arena_run_regind(run, bin_info, ptr); - - *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset - + (regind * sizeof(prof_ctx_t *)))) = ctx; - } else - assert((uintptr_t)ctx == (uintptr_t)1U); - } else - chunk->map[pageind-map_bias].prof_ctx = ctx; -} -#endif - -JEMALLOC_INLINE void -arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr) -{ - size_t pageind; - arena_chunk_map_t *mapelm; - - assert(arena != NULL); - dassert(arena->magic == ARENA_MAGIC); - assert(chunk->arena == arena); - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapelm = &chunk->map[pageind-map_bias]; - assert((mapelm->bits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapelm->bits & CHUNK_MAP_LARGE) == 0) { - /* Small allocation. */ -#ifdef JEMALLOC_TCACHE - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - tcache_dalloc_small(tcache, ptr); - else { -#endif - arena_run_t *run; - arena_bin_t *bin; - - run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapelm->bits >> - PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; -#ifdef JEMALLOC_DEBUG - { - size_t binind = arena_bin_index(arena, bin); - arena_bin_info_t *bin_info = - &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % - bin_info->reg_size == 0); - } -#endif - malloc_mutex_lock(&bin->lock); - arena_dalloc_bin(arena, chunk, ptr, mapelm); - malloc_mutex_unlock(&bin->lock); -#ifdef JEMALLOC_TCACHE - } -#endif - } else { -#ifdef JEMALLOC_TCACHE - size_t size = mapelm->bits & ~PAGE_MASK; - - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - if (size <= tcache_maxclass) { - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - tcache_dalloc_large(tcache, ptr, size); - else { - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); - } - } else { - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); - } -#else - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, ptr); - malloc_mutex_unlock(&arena->lock); -#endif - } -} -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/atomic.h b/deps/jemalloc.orig/include/jemalloc/internal/atomic.h deleted file mode 100644 index 9a298623f8a..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/atomic.h +++ /dev/null @@ -1,169 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#define atomic_read_uint64(p) atomic_add_uint64(p, 0) -#define atomic_read_uint32(p) atomic_add_uint32(p, 0) - -#if (LG_SIZEOF_PTR == 3) -# define atomic_read_z(p) \ - (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)0) -# define atomic_add_z(p, x) \ - (size_t)atomic_add_uint64((uint64_t *)p, (uint64_t)x) -# define atomic_sub_z(p, x) \ - (size_t)atomic_sub_uint64((uint64_t *)p, (uint64_t)x) -#elif (LG_SIZEOF_PTR == 2) -# define atomic_read_z(p) \ - (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)0) -# define atomic_add_z(p, x) \ - (size_t)atomic_add_uint32((uint32_t *)p, (uint32_t)x) -# define atomic_sub_z(p, x) \ - (size_t)atomic_sub_uint32((uint32_t *)p, (uint32_t)x) -#endif - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -uint64_t atomic_add_uint64(uint64_t *p, uint64_t x); -uint64_t atomic_sub_uint64(uint64_t *p, uint64_t x); -uint32_t atomic_add_uint32(uint32_t *p, uint32_t x); -uint32_t atomic_sub_uint32(uint32_t *p, uint32_t x); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ATOMIC_C_)) -/******************************************************************************/ -/* 64-bit operations. */ -#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - - return (__sync_add_and_fetch(p, x)); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - - return (__sync_sub_and_fetch(p, x)); -} -#elif (defined(JEMALLOC_OSATOMIC)) -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - - return (OSAtomicAdd64((int64_t)x, (int64_t *)p)); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - - return (OSAtomicAdd64(-((int64_t)x), (int64_t *)p)); -} -#elif (defined(__amd64_) || defined(__x86_64__)) -JEMALLOC_INLINE uint64_t -atomic_add_uint64(uint64_t *p, uint64_t x) -{ - - asm volatile ( - "lock; xaddq %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ - : "m" (*p) /* Inputs. */ - ); - - return (x); -} - -JEMALLOC_INLINE uint64_t -atomic_sub_uint64(uint64_t *p, uint64_t x) -{ - - x = (uint64_t)(-(int64_t)x); - asm volatile ( - "lock; xaddq %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ - : "m" (*p) /* Inputs. */ - ); - - return (x); -} -#else -# if (LG_SIZEOF_PTR == 3) -# error "Missing implementation for 64-bit atomic operations" -# endif -#endif - -/******************************************************************************/ -/* 32-bit operations. */ -#ifdef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - - return (__sync_add_and_fetch(p, x)); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - - return (__sync_sub_and_fetch(p, x)); -} -#elif (defined(JEMALLOC_OSATOMIC)) -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - - return (OSAtomicAdd32((int32_t)x, (int32_t *)p)); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - - return (OSAtomicAdd32(-((int32_t)x), (int32_t *)p)); -} -#elif (defined(__i386__) || defined(__amd64_) || defined(__x86_64__)) -JEMALLOC_INLINE uint32_t -atomic_add_uint32(uint32_t *p, uint32_t x) -{ - - asm volatile ( - "lock; xaddl %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ - : "m" (*p) /* Inputs. */ - ); - - return (x); -} - -JEMALLOC_INLINE uint32_t -atomic_sub_uint32(uint32_t *p, uint32_t x) -{ - - x = (uint32_t)(-(int32_t)x); - asm volatile ( - "lock; xaddl %0, %1;" - : "+r" (x), "=m" (*p) /* Outputs. */ - : "m" (*p) /* Inputs. */ - ); - - return (x); -} -#else -# error "Missing implementation for 32-bit atomic operations" -#endif -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/base.h b/deps/jemalloc.orig/include/jemalloc/internal/base.h deleted file mode 100644 index e353f309bd2..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/base.h +++ /dev/null @@ -1,24 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern malloc_mutex_t base_mtx; - -void *base_alloc(size_t size); -extent_node_t *base_node_alloc(void); -void base_node_dealloc(extent_node_t *node); -bool base_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/bitmap.h b/deps/jemalloc.orig/include/jemalloc/internal/bitmap.h deleted file mode 100644 index 605ebac58c1..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/bitmap.h +++ /dev/null @@ -1,184 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -/* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#define LG_BITMAP_MAXBITS LG_RUN_MAXREGS - -typedef struct bitmap_level_s bitmap_level_t; -typedef struct bitmap_info_s bitmap_info_t; -typedef unsigned long bitmap_t; -#define LG_SIZEOF_BITMAP LG_SIZEOF_LONG - -/* Number of bits per group. */ -#define LG_BITMAP_GROUP_NBITS (LG_SIZEOF_BITMAP + 3) -#define BITMAP_GROUP_NBITS (ZU(1) << LG_BITMAP_GROUP_NBITS) -#define BITMAP_GROUP_NBITS_MASK (BITMAP_GROUP_NBITS-1) - -/* Maximum number of levels possible. */ -#define BITMAP_MAX_LEVELS \ - (LG_BITMAP_MAXBITS / LG_SIZEOF_BITMAP) \ - + !!(LG_BITMAP_MAXBITS % LG_SIZEOF_BITMAP) - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -struct bitmap_level_s { - /* Offset of this level's groups within the array of groups. */ - size_t group_offset; -}; - -struct bitmap_info_s { - /* Logical number of bits in bitmap (stored at bottom level). */ - size_t nbits; - - /* Number of levels necessary for nbits. */ - unsigned nlevels; - - /* - * Only the first (nlevels+1) elements are used, and levels are ordered - * bottom to top (e.g. the bottom level is stored in levels[0]). - */ - bitmap_level_t levels[BITMAP_MAX_LEVELS+1]; -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -void bitmap_info_init(bitmap_info_t *binfo, size_t nbits); -size_t bitmap_info_ngroups(const bitmap_info_t *binfo); -size_t bitmap_size(size_t nbits); -void bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -bool bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo); -bool bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); -void bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); -size_t bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo); -void bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_BITMAP_C_)) -JEMALLOC_INLINE bool -bitmap_full(bitmap_t *bitmap, const bitmap_info_t *binfo) -{ - unsigned rgoff = binfo->levels[binfo->nlevels].group_offset - 1; - bitmap_t rg = bitmap[rgoff]; - /* The bitmap is full iff the root group is 0. */ - return (rg == 0); -} - -JEMALLOC_INLINE bool -bitmap_get(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) -{ - size_t goff; - bitmap_t g; - - assert(bit < binfo->nbits); - goff = bit >> LG_BITMAP_GROUP_NBITS; - g = bitmap[goff]; - return (!(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK)))); -} - -JEMALLOC_INLINE void -bitmap_set(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) -{ - size_t goff; - bitmap_t *gp; - bitmap_t g; - - assert(bit < binfo->nbits); - assert(bitmap_get(bitmap, binfo, bit) == false); - goff = bit >> LG_BITMAP_GROUP_NBITS; - gp = &bitmap[goff]; - g = *gp; - assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))); - g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); - *gp = g; - assert(bitmap_get(bitmap, binfo, bit)); - /* Propagate group state transitions up the tree. */ - if (g == 0) { - unsigned i; - for (i = 1; i < binfo->nlevels; i++) { - bit = goff; - goff = bit >> LG_BITMAP_GROUP_NBITS; - gp = &bitmap[binfo->levels[i].group_offset + goff]; - g = *gp; - assert(g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))); - g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); - *gp = g; - if (g != 0) - break; - } - } -} - -/* sfu: set first unset. */ -JEMALLOC_INLINE size_t -bitmap_sfu(bitmap_t *bitmap, const bitmap_info_t *binfo) -{ - size_t bit; - bitmap_t g; - unsigned i; - - assert(bitmap_full(bitmap, binfo) == false); - - i = binfo->nlevels - 1; - g = bitmap[binfo->levels[i].group_offset]; - bit = ffsl(g) - 1; - while (i > 0) { - i--; - g = bitmap[binfo->levels[i].group_offset + bit]; - bit = (bit << LG_BITMAP_GROUP_NBITS) + (ffsl(g) - 1); - } - - bitmap_set(bitmap, binfo, bit); - return (bit); -} - -JEMALLOC_INLINE void -bitmap_unset(bitmap_t *bitmap, const bitmap_info_t *binfo, size_t bit) -{ - size_t goff; - bitmap_t *gp; - bitmap_t g; - bool propagate; - - assert(bit < binfo->nbits); - assert(bitmap_get(bitmap, binfo, bit)); - goff = bit >> LG_BITMAP_GROUP_NBITS; - gp = &bitmap[goff]; - g = *gp; - propagate = (g == 0); - assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) == 0); - g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); - *gp = g; - assert(bitmap_get(bitmap, binfo, bit) == false); - /* Propagate group state transitions up the tree. */ - if (propagate) { - unsigned i; - for (i = 1; i < binfo->nlevels; i++) { - bit = goff; - goff = bit >> LG_BITMAP_GROUP_NBITS; - gp = &bitmap[binfo->levels[i].group_offset + goff]; - g = *gp; - propagate = (g == 0); - assert((g & (1LU << (bit & BITMAP_GROUP_NBITS_MASK))) - == 0); - g ^= 1LU << (bit & BITMAP_GROUP_NBITS_MASK); - *gp = g; - if (propagate == false) - break; - } - } -} - -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk.h deleted file mode 100644 index 54b6a3ec886..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/chunk.h +++ /dev/null @@ -1,65 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -/* - * Size and alignment of memory chunks that are allocated by the OS's virtual - * memory system. - */ -#define LG_CHUNK_DEFAULT 22 - -/* Return the chunk address for allocation address a. */ -#define CHUNK_ADDR2BASE(a) \ - ((void *)((uintptr_t)(a) & ~chunksize_mask)) - -/* Return the chunk offset of address a. */ -#define CHUNK_ADDR2OFFSET(a) \ - ((size_t)((uintptr_t)(a) & chunksize_mask)) - -/* Return the smallest chunk multiple that is >= s. */ -#define CHUNK_CEILING(s) \ - (((s) + chunksize_mask) & ~chunksize_mask) - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern size_t opt_lg_chunk; -#ifdef JEMALLOC_SWAP -extern bool opt_overcommit; -#endif - -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) -/* Protects stats_chunks; currently not used for any other purpose. */ -extern malloc_mutex_t chunks_mtx; -/* Chunk statistics. */ -extern chunk_stats_t stats_chunks; -#endif - -#ifdef JEMALLOC_IVSALLOC -extern rtree_t *chunks_rtree; -#endif - -extern size_t chunksize; -extern size_t chunksize_mask; /* (chunksize - 1). */ -extern size_t chunk_npages; -extern size_t map_bias; /* Number of arena chunk header pages. */ -extern size_t arena_maxclass; /* Max size class for arenas. */ - -void *chunk_alloc(size_t size, bool base, bool *zero); -void chunk_dealloc(void *chunk, size_t size, bool unmap); -bool chunk_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ - -#include "jemalloc/internal/chunk_swap.h" -#include "jemalloc/internal/chunk_dss.h" -#include "jemalloc/internal/chunk_mmap.h" diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h deleted file mode 100644 index 6f005222181..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/chunk_dss.h +++ /dev/null @@ -1,30 +0,0 @@ -#ifdef JEMALLOC_DSS -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -/* - * Protects sbrk() calls. This avoids malloc races among threads, though it - * does not protect against races with threads that call sbrk() directly. - */ -extern malloc_mutex_t dss_mtx; - -void *chunk_alloc_dss(size_t size, bool *zero); -bool chunk_in_dss(void *chunk); -bool chunk_dealloc_dss(void *chunk, size_t size); -bool chunk_dss_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ -#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h deleted file mode 100644 index 07b50a4dc37..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/chunk_mmap.h +++ /dev/null @@ -1,23 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -void *chunk_alloc_mmap(size_t size); -void *chunk_alloc_mmap_noreserve(size_t size); -void chunk_dealloc_mmap(void *chunk, size_t size); - -bool chunk_mmap_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/chunk_swap.h b/deps/jemalloc.orig/include/jemalloc/internal/chunk_swap.h deleted file mode 100644 index 9faa739f713..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/chunk_swap.h +++ /dev/null @@ -1,34 +0,0 @@ -#ifdef JEMALLOC_SWAP -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern malloc_mutex_t swap_mtx; -extern bool swap_enabled; -extern bool swap_prezeroed; -extern size_t swap_nfds; -extern int *swap_fds; -#ifdef JEMALLOC_STATS -extern size_t swap_avail; -#endif - -void *chunk_alloc_swap(size_t size, bool *zero); -bool chunk_in_swap(void *chunk); -bool chunk_dealloc_swap(void *chunk, size_t size); -bool chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed); -bool chunk_swap_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ -#endif /* JEMALLOC_SWAP */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/ckh.h b/deps/jemalloc.orig/include/jemalloc/internal/ckh.h deleted file mode 100644 index 3e4ad4c85f9..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/ckh.h +++ /dev/null @@ -1,95 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -typedef struct ckh_s ckh_t; -typedef struct ckhc_s ckhc_t; - -/* Typedefs to allow easy function pointer passing. */ -typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *); -typedef bool ckh_keycomp_t (const void *, const void *); - -/* Maintain counters used to get an idea of performance. */ -/* #define CKH_COUNT */ -/* Print counter values in ckh_delete() (requires CKH_COUNT). */ -/* #define CKH_VERBOSE */ - -/* - * There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket. Try to fit - * one bucket per L1 cache line. - */ -#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -/* Hash table cell. */ -struct ckhc_s { - const void *key; - const void *data; -}; - -struct ckh_s { -#ifdef JEMALLOC_DEBUG -#define CKH_MAGIC 0x3af2489d - uint32_t magic; -#endif - -#ifdef CKH_COUNT - /* Counters used to get an idea of performance. */ - uint64_t ngrows; - uint64_t nshrinks; - uint64_t nshrinkfails; - uint64_t ninserts; - uint64_t nrelocs; -#endif - - /* Used for pseudo-random number generation. */ -#define CKH_A 1103515241 -#define CKH_C 12347 - uint32_t prn_state; - - /* Total number of items. */ - size_t count; - - /* - * Minimum and current number of hash table buckets. There are - * 2^LG_CKH_BUCKET_CELLS cells per bucket. - */ - unsigned lg_minbuckets; - unsigned lg_curbuckets; - - /* Hash and comparison functions. */ - ckh_hash_t *hash; - ckh_keycomp_t *keycomp; - - /* Hash table with 2^lg_curbuckets buckets. */ - ckhc_t *tab; -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -bool ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, - ckh_keycomp_t *keycomp); -void ckh_delete(ckh_t *ckh); -size_t ckh_count(ckh_t *ckh); -bool ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data); -bool ckh_insert(ckh_t *ckh, const void *key, const void *data); -bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key, - void **data); -bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data); -void ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); -bool ckh_string_keycomp(const void *k1, const void *k2); -void ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); -bool ckh_pointer_keycomp(const void *k1, const void *k2); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/ctl.h b/deps/jemalloc.orig/include/jemalloc/internal/ctl.h deleted file mode 100644 index f1f5eb70a2a..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/ctl.h +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -typedef struct ctl_node_s ctl_node_t; -typedef struct ctl_arena_stats_s ctl_arena_stats_t; -typedef struct ctl_stats_s ctl_stats_t; - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -struct ctl_node_s { - bool named; - union { - struct { - const char *name; - /* If (nchildren == 0), this is a terminal node. */ - unsigned nchildren; - const ctl_node_t *children; - } named; - struct { - const ctl_node_t *(*index)(const size_t *, size_t, - size_t); - } indexed; - } u; - int (*ctl)(const size_t *, size_t, void *, size_t *, void *, - size_t); -}; - -struct ctl_arena_stats_s { - bool initialized; - unsigned nthreads; - size_t pactive; - size_t pdirty; -#ifdef JEMALLOC_STATS - arena_stats_t astats; - - /* Aggregate stats for small size classes, based on bin stats. */ - size_t allocated_small; - uint64_t nmalloc_small; - uint64_t ndalloc_small; - uint64_t nrequests_small; - - malloc_bin_stats_t *bstats; /* nbins elements. */ - malloc_large_stats_t *lstats; /* nlclasses elements. */ -#endif -}; - -struct ctl_stats_s { -#ifdef JEMALLOC_STATS - size_t allocated; - size_t active; - size_t mapped; - struct { - size_t current; /* stats_chunks.curchunks */ - uint64_t total; /* stats_chunks.nchunks */ - size_t high; /* stats_chunks.highchunks */ - } chunks; - struct { - size_t allocated; /* huge_allocated */ - uint64_t nmalloc; /* huge_nmalloc */ - uint64_t ndalloc; /* huge_ndalloc */ - } huge; -#endif - ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ -#ifdef JEMALLOC_SWAP - size_t swap_avail; -#endif -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -int ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, - size_t newlen); -int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp); - -int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen); -bool ctl_boot(void); - -#define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ - if (JEMALLOC_P(mallctl)(name, oldp, oldlenp, newp, newlen) \ - != 0) { \ - malloc_write(": Failure in xmallctl(\""); \ - malloc_write(name); \ - malloc_write("\", ...)\n"); \ - abort(); \ - } \ -} while (0) - -#define xmallctlnametomib(name, mibp, miblenp) do { \ - if (JEMALLOC_P(mallctlnametomib)(name, mibp, miblenp) != 0) { \ - malloc_write( \ - ": Failure in xmallctlnametomib(\""); \ - malloc_write(name); \ - malloc_write("\", ...)\n"); \ - abort(); \ - } \ -} while (0) - -#define xmallctlbymib(mib, miblen, oldp, oldlenp, newp, newlen) do { \ - if (JEMALLOC_P(mallctlbymib)(mib, miblen, oldp, oldlenp, newp, \ - newlen) != 0) { \ - malloc_write( \ - ": Failure in xmallctlbymib()\n"); \ - abort(); \ - } \ -} while (0) - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ - diff --git a/deps/jemalloc.orig/include/jemalloc/internal/extent.h b/deps/jemalloc.orig/include/jemalloc/internal/extent.h deleted file mode 100644 index 6fe9702b5f6..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/extent.h +++ /dev/null @@ -1,49 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -typedef struct extent_node_s extent_node_t; - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -/* Tree of extents. */ -struct extent_node_s { -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) - /* Linkage for the size/address-ordered tree. */ - rb_node(extent_node_t) link_szad; -#endif - - /* Linkage for the address-ordered tree. */ - rb_node(extent_node_t) link_ad; - -#ifdef JEMALLOC_PROF - /* Profile counters, used for huge objects. */ - prof_ctx_t *prof_ctx; -#endif - - /* Pointer to the extent that this tree node is responsible for. */ - void *addr; - - /* Total region size. */ - size_t size; -}; -typedef rb_tree(extent_node_t) extent_tree_t; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) -rb_proto(, extent_tree_szad_, extent_tree_t, extent_node_t) -#endif - -rb_proto(, extent_tree_ad_, extent_tree_t, extent_node_t) - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ - diff --git a/deps/jemalloc.orig/include/jemalloc/internal/hash.h b/deps/jemalloc.orig/include/jemalloc/internal/hash.h deleted file mode 100644 index 8a46ce30803..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/hash.h +++ /dev/null @@ -1,70 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -uint64_t hash(const void *key, size_t len, uint64_t seed); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_)) -/* - * The following hash function is based on MurmurHash64A(), placed into the - * public domain by Austin Appleby. See http://murmurhash.googlepages.com/ for - * details. - */ -JEMALLOC_INLINE uint64_t -hash(const void *key, size_t len, uint64_t seed) -{ - const uint64_t m = 0xc6a4a7935bd1e995LLU; - const int r = 47; - uint64_t h = seed ^ (len * m); - const uint64_t *data = (const uint64_t *)key; - const uint64_t *end = data + (len/8); - const unsigned char *data2; - - assert(((uintptr_t)key & 0x7) == 0); - - while(data != end) { - uint64_t k = *data++; - - k *= m; - k ^= k >> r; - k *= m; - - h ^= k; - h *= m; - } - - data2 = (const unsigned char *)data; - switch(len & 7) { - case 7: h ^= ((uint64_t)(data2[6])) << 48; - case 6: h ^= ((uint64_t)(data2[5])) << 40; - case 5: h ^= ((uint64_t)(data2[4])) << 32; - case 4: h ^= ((uint64_t)(data2[3])) << 24; - case 3: h ^= ((uint64_t)(data2[2])) << 16; - case 2: h ^= ((uint64_t)(data2[1])) << 8; - case 1: h ^= ((uint64_t)(data2[0])); - h *= m; - } - - h ^= h >> r; - h *= m; - h ^= h >> r; - - return (h); -} -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/huge.h b/deps/jemalloc.orig/include/jemalloc/internal/huge.h deleted file mode 100644 index 66544cf8d97..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/huge.h +++ /dev/null @@ -1,41 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#ifdef JEMALLOC_STATS -/* Huge allocation statistics. */ -extern uint64_t huge_nmalloc; -extern uint64_t huge_ndalloc; -extern size_t huge_allocated; -#endif - -/* Protects chunk-related data structures. */ -extern malloc_mutex_t huge_mtx; - -void *huge_malloc(size_t size, bool zero); -void *huge_palloc(size_t size, size_t alignment, bool zero); -void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, - size_t extra); -void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); -void huge_dalloc(void *ptr, bool unmap); -size_t huge_salloc(const void *ptr); -#ifdef JEMALLOC_PROF -prof_ctx_t *huge_prof_ctx_get(const void *ptr); -void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -#endif -bool huge_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in b/deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in deleted file mode 100644 index a44f0978ae4..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/jemalloc_internal.h.in +++ /dev/null @@ -1,788 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include -#include -#ifndef SIZE_T_MAX -# define SIZE_T_MAX SIZE_MAX -#endif -#include -#include -#include -#include -#include -#include -#include -#include -#ifndef offsetof -# define offsetof(type, member) ((size_t)&(((type *)NULL)->member)) -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "../jemalloc@install_suffix@.h" - -#include "jemalloc/internal/private_namespace.h" - -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) -#include -#endif - -#ifdef JEMALLOC_ZONE -#include -#include -#include -#include -#endif - -#ifdef JEMALLOC_LAZY_LOCK -#include -#endif - -#define RB_COMPACT -#include "jemalloc/internal/rb.h" -#include "jemalloc/internal/qr.h" -#include "jemalloc/internal/ql.h" - -extern void (*JEMALLOC_P(malloc_message))(void *wcbopaque, const char *s); - -/* - * Define a custom assert() in order to reduce the chances of deadlock during - * assertion failure. - */ -#ifndef assert -# ifdef JEMALLOC_DEBUG -# define assert(e) do { \ - if (!(e)) { \ - char line_buf[UMAX2S_BUFSIZE]; \ - malloc_write(": "); \ - malloc_write(__FILE__); \ - malloc_write(":"); \ - malloc_write(u2s(__LINE__, 10, line_buf)); \ - malloc_write(": Failed assertion: "); \ - malloc_write("\""); \ - malloc_write(#e); \ - malloc_write("\"\n"); \ - abort(); \ - } \ -} while (0) -# else -# define assert(e) -# endif -#endif - -#ifdef JEMALLOC_DEBUG -# define dassert(e) assert(e) -#else -# define dassert(e) -#endif - -/* - * jemalloc can conceptually be broken into components (arena, tcache, etc.), - * but there are circular dependencies that cannot be broken without - * substantial performance degradation. In order to reduce the effect on - * visual code flow, read the header files in multiple passes, with one of the - * following cpp variables defined during each pass: - * - * JEMALLOC_H_TYPES : Preprocessor-defined constants and psuedo-opaque data - * types. - * JEMALLOC_H_STRUCTS : Data structures. - * JEMALLOC_H_EXTERNS : Extern data declarations and function prototypes. - * JEMALLOC_H_INLINES : Inline functions. - */ -/******************************************************************************/ -#define JEMALLOC_H_TYPES - -#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) - -#define ZU(z) ((size_t)z) - -#ifndef __DECONST -# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) -#endif - -#ifdef JEMALLOC_DEBUG - /* Disable inlining to make debugging easier. */ -# define JEMALLOC_INLINE -# define inline -#else -# define JEMALLOC_ENABLE_INLINE -# define JEMALLOC_INLINE static inline -#endif - -/* Size of stack-allocated buffer passed to buferror(). */ -#define BUFERROR_BUF 64 - -/* Minimum alignment of allocations is 2^LG_QUANTUM bytes. */ -#ifdef __i386__ -# define LG_QUANTUM 4 -#endif -#ifdef __ia64__ -# define LG_QUANTUM 4 -#endif -#ifdef __alpha__ -# define LG_QUANTUM 4 -#endif -#ifdef __sparc64__ -# define LG_QUANTUM 4 -#endif -#if (defined(__amd64__) || defined(__x86_64__)) -# define LG_QUANTUM 4 -#endif -#ifdef __arm__ -# define LG_QUANTUM 3 -#endif -#ifdef __mips__ -# define LG_QUANTUM 3 -#endif -#ifdef __powerpc__ -# define LG_QUANTUM 4 -#endif -#ifdef __s390x__ -# define LG_QUANTUM 4 -#endif - -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) - -/* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) - -#define LONG ((size_t)(1U << LG_SIZEOF_LONG)) -#define LONG_MASK (LONG - 1) - -/* Return the smallest long multiple that is >= a. */ -#define LONG_CEILING(a) \ - (((a) + LONG_MASK) & ~LONG_MASK) - -#define SIZEOF_PTR (1U << LG_SIZEOF_PTR) -#define PTR_MASK (SIZEOF_PTR - 1) - -/* Return the smallest (void *) multiple that is >= a. */ -#define PTR_CEILING(a) \ - (((a) + PTR_MASK) & ~PTR_MASK) - -/* - * Maximum size of L1 cache line. This is used to avoid cache line aliasing. - * In addition, this controls the spacing of cacheline-spaced size classes. - */ -#define LG_CACHELINE 6 -#define CACHELINE ((size_t)(1U << LG_CACHELINE)) -#define CACHELINE_MASK (CACHELINE - 1) - -/* Return the smallest cacheline multiple that is >= s. */ -#define CACHELINE_CEILING(s) \ - (((s) + CACHELINE_MASK) & ~CACHELINE_MASK) - -/* - * Page size. STATIC_PAGE_SHIFT is determined by the configure script. If - * DYNAMIC_PAGE_SHIFT is enabled, only use the STATIC_PAGE_* macros where - * compile-time values are required for the purposes of defining data - * structures. - */ -#define STATIC_PAGE_SIZE ((size_t)(1U << STATIC_PAGE_SHIFT)) -#define STATIC_PAGE_MASK ((size_t)(STATIC_PAGE_SIZE - 1)) - -#ifdef PAGE_SHIFT -# undef PAGE_SHIFT -#endif -#ifdef PAGE_SIZE -# undef PAGE_SIZE -#endif -#ifdef PAGE_MASK -# undef PAGE_MASK -#endif - -#ifdef DYNAMIC_PAGE_SHIFT -# define PAGE_SHIFT lg_pagesize -# define PAGE_SIZE pagesize -# define PAGE_MASK pagesize_mask -#else -# define PAGE_SHIFT STATIC_PAGE_SHIFT -# define PAGE_SIZE STATIC_PAGE_SIZE -# define PAGE_MASK STATIC_PAGE_MASK -#endif - -/* Return the smallest pagesize multiple that is >= s. */ -#define PAGE_CEILING(s) \ - (((s) + PAGE_MASK) & ~PAGE_MASK) - -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" -#include "jemalloc/internal/ckh.h" -#include "jemalloc/internal/stats.h" -#include "jemalloc/internal/ctl.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/mb.h" -#include "jemalloc/internal/extent.h" -#include "jemalloc/internal/arena.h" -#include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/base.h" -#include "jemalloc/internal/chunk.h" -#include "jemalloc/internal/huge.h" -#include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/tcache.h" -#include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif -#include "jemalloc/internal/prof.h" - -#undef JEMALLOC_H_TYPES -/******************************************************************************/ -#define JEMALLOC_H_STRUCTS - -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" -#include "jemalloc/internal/ckh.h" -#include "jemalloc/internal/stats.h" -#include "jemalloc/internal/ctl.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/mb.h" -#include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/extent.h" -#include "jemalloc/internal/arena.h" -#include "jemalloc/internal/base.h" -#include "jemalloc/internal/chunk.h" -#include "jemalloc/internal/huge.h" -#include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/tcache.h" -#include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif -#include "jemalloc/internal/prof.h" - -#ifdef JEMALLOC_STATS -typedef struct { - uint64_t allocated; - uint64_t deallocated; -} thread_allocated_t; -#endif - -#undef JEMALLOC_H_STRUCTS -/******************************************************************************/ -#define JEMALLOC_H_EXTERNS - -extern bool opt_abort; -#ifdef JEMALLOC_FILL -extern bool opt_junk; -#endif -#ifdef JEMALLOC_SYSV -extern bool opt_sysv; -#endif -#ifdef JEMALLOC_XMALLOC -extern bool opt_xmalloc; -#endif -#ifdef JEMALLOC_FILL -extern bool opt_zero; -#endif -extern size_t opt_narenas; - -#ifdef DYNAMIC_PAGE_SHIFT -extern size_t pagesize; -extern size_t pagesize_mask; -extern size_t lg_pagesize; -#endif - -/* Number of CPUs. */ -extern unsigned ncpus; - -extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ -extern pthread_key_t arenas_tsd; -#ifndef NO_TLS -/* - * Map of pthread_self() --> arenas[???], used for selecting an arena to use - * for allocations. - */ -extern __thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); -# define ARENA_GET() arenas_tls -# define ARENA_SET(v) do { \ - arenas_tls = (v); \ - pthread_setspecific(arenas_tsd, (void *)(v)); \ -} while (0) -#else -# define ARENA_GET() ((arena_t *)pthread_getspecific(arenas_tsd)) -# define ARENA_SET(v) do { \ - pthread_setspecific(arenas_tsd, (void *)(v)); \ -} while (0) -#endif - -/* - * Arenas that are used to service external requests. Not all elements of the - * arenas array are necessarily used; arenas are created lazily as needed. - */ -extern arena_t **arenas; -extern unsigned narenas; - -#ifdef JEMALLOC_STATS -# ifndef NO_TLS -extern __thread thread_allocated_t thread_allocated_tls; -# define ALLOCATED_GET() (thread_allocated_tls.allocated) -# define ALLOCATEDP_GET() (&thread_allocated_tls.allocated) -# define DEALLOCATED_GET() (thread_allocated_tls.deallocated) -# define DEALLOCATEDP_GET() (&thread_allocated_tls.deallocated) -# define ALLOCATED_ADD(a, d) do { \ - thread_allocated_tls.allocated += a; \ - thread_allocated_tls.deallocated += d; \ -} while (0) -# else -extern pthread_key_t thread_allocated_tsd; -thread_allocated_t *thread_allocated_get_hard(void); - -# define ALLOCATED_GET() (thread_allocated_get()->allocated) -# define ALLOCATEDP_GET() (&thread_allocated_get()->allocated) -# define DEALLOCATED_GET() (thread_allocated_get()->deallocated) -# define DEALLOCATEDP_GET() (&thread_allocated_get()->deallocated) -# define ALLOCATED_ADD(a, d) do { \ - thread_allocated_t *thread_allocated = thread_allocated_get(); \ - thread_allocated->allocated += (a); \ - thread_allocated->deallocated += (d); \ -} while (0) -# endif -#endif - -arena_t *arenas_extend(unsigned ind); -arena_t *choose_arena_hard(void); -int buferror(int errnum, char *buf, size_t buflen); -void jemalloc_prefork(void); -void jemalloc_postfork(void); - -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" -#include "jemalloc/internal/ckh.h" -#include "jemalloc/internal/stats.h" -#include "jemalloc/internal/ctl.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/mb.h" -#include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/extent.h" -#include "jemalloc/internal/arena.h" -#include "jemalloc/internal/base.h" -#include "jemalloc/internal/chunk.h" -#include "jemalloc/internal/huge.h" -#include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/tcache.h" -#include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif -#include "jemalloc/internal/prof.h" - -#undef JEMALLOC_H_EXTERNS -/******************************************************************************/ -#define JEMALLOC_H_INLINES - -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/prn.h" -#include "jemalloc/internal/ckh.h" -#include "jemalloc/internal/stats.h" -#include "jemalloc/internal/ctl.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/mb.h" -#include "jemalloc/internal/extent.h" -#include "jemalloc/internal/base.h" -#include "jemalloc/internal/chunk.h" -#include "jemalloc/internal/huge.h" - -#ifndef JEMALLOC_ENABLE_INLINE -size_t pow2_ceil(size_t x); -size_t s2u(size_t size); -size_t sa2u(size_t size, size_t alignment, size_t *run_size_p); -void malloc_write(const char *s); -arena_t *choose_arena(void); -# if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -thread_allocated_t *thread_allocated_get(void); -# endif -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -/* Compute the smallest power of 2 that is >= x. */ -JEMALLOC_INLINE size_t -pow2_ceil(size_t x) -{ - - x--; - x |= x >> 1; - x |= x >> 2; - x |= x >> 4; - x |= x >> 8; - x |= x >> 16; -#if (LG_SIZEOF_PTR == 3) - x |= x >> 32; -#endif - x++; - return (x); -} - -/* - * Compute usable size that would result from allocating an object with the - * specified size. - */ -JEMALLOC_INLINE size_t -s2u(size_t size) -{ - - if (size <= small_maxclass) - return (arena_bin_info[SMALL_SIZE2BIN(size)].reg_size); - if (size <= arena_maxclass) - return (PAGE_CEILING(size)); - return (CHUNK_CEILING(size)); -} - -/* - * Compute usable size that would result from allocating an object with the - * specified size and alignment. - */ -JEMALLOC_INLINE size_t -sa2u(size_t size, size_t alignment, size_t *run_size_p) -{ - size_t usize; - - /* - * Round size up to the nearest multiple of alignment. - * - * This done, we can take advantage of the fact that for each small - * size class, every object is aligned at the smallest power of two - * that is non-zero in the base two representation of the size. For - * example: - * - * Size | Base 2 | Minimum alignment - * -----+----------+------------------ - * 96 | 1100000 | 32 - * 144 | 10100000 | 32 - * 192 | 11000000 | 64 - * - * Depending on runtime settings, it is possible that arena_malloc() - * will further round up to a power of two, but that never causes - * correctness issues. - */ - usize = (size + (alignment - 1)) & (-alignment); - /* - * (usize < size) protects against the combination of maximal - * alignment and size greater than maximal alignment. - */ - if (usize < size) { - /* size_t overflow. */ - return (0); - } - - if (usize <= arena_maxclass && alignment <= PAGE_SIZE) { - if (usize <= small_maxclass) - return (arena_bin_info[SMALL_SIZE2BIN(usize)].reg_size); - return (PAGE_CEILING(usize)); - } else { - size_t run_size; - - /* - * We can't achieve subpage alignment, so round up alignment - * permanently; it makes later calculations simpler. - */ - alignment = PAGE_CEILING(alignment); - usize = PAGE_CEILING(size); - /* - * (usize < size) protects against very large sizes within - * PAGE_SIZE of SIZE_T_MAX. - * - * (usize + alignment < usize) protects against the - * combination of maximal alignment and usize large enough - * to cause overflow. This is similar to the first overflow - * check above, but it needs to be repeated due to the new - * usize value, which may now be *equal* to maximal - * alignment, whereas before we only detected overflow if the - * original size was *greater* than maximal alignment. - */ - if (usize < size || usize + alignment < usize) { - /* size_t overflow. */ - return (0); - } - - /* - * Calculate the size of the over-size run that arena_palloc() - * would need to allocate in order to guarantee the alignment. - */ - if (usize >= alignment) - run_size = usize + alignment - PAGE_SIZE; - else { - /* - * It is possible that (alignment << 1) will cause - * overflow, but it doesn't matter because we also - * subtract PAGE_SIZE, which in the case of overflow - * leaves us with a very large run_size. That causes - * the first conditional below to fail, which means - * that the bogus run_size value never gets used for - * anything important. - */ - run_size = (alignment << 1) - PAGE_SIZE; - } - if (run_size_p != NULL) - *run_size_p = run_size; - - if (run_size <= arena_maxclass) - return (PAGE_CEILING(usize)); - return (CHUNK_CEILING(usize)); - } -} - -/* - * Wrapper around malloc_message() that avoids the need for - * JEMALLOC_P(malloc_message)(...) throughout the code. - */ -JEMALLOC_INLINE void -malloc_write(const char *s) -{ - - JEMALLOC_P(malloc_message)(NULL, s); -} - -/* - * Choose an arena based on a per-thread value (fast-path code, calls slow-path - * code if necessary). - */ -JEMALLOC_INLINE arena_t * -choose_arena(void) -{ - arena_t *ret; - - ret = ARENA_GET(); - if (ret == NULL) { - ret = choose_arena_hard(); - assert(ret != NULL); - } - - return (ret); -} - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -JEMALLOC_INLINE thread_allocated_t * -thread_allocated_get(void) -{ - thread_allocated_t *thread_allocated = (thread_allocated_t *) - pthread_getspecific(thread_allocated_tsd); - - if (thread_allocated == NULL) - return (thread_allocated_get_hard()); - return (thread_allocated); -} -#endif -#endif - -#include "jemalloc/internal/bitmap.h" -#include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/tcache.h" -#include "jemalloc/internal/arena.h" -#include "jemalloc/internal/hash.h" -#ifdef JEMALLOC_ZONE -#include "jemalloc/internal/zone.h" -#endif - -#ifndef JEMALLOC_ENABLE_INLINE -void *imalloc(size_t size); -void *icalloc(size_t size); -void *ipalloc(size_t usize, size_t alignment, bool zero); -size_t isalloc(const void *ptr); -# ifdef JEMALLOC_IVSALLOC -size_t ivsalloc(const void *ptr); -# endif -void idalloc(void *ptr); -void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero, bool no_move); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_INLINE void * -imalloc(size_t size) -{ - - assert(size != 0); - - if (size <= arena_maxclass) - return (arena_malloc(size, false)); - else - return (huge_malloc(size, false)); -} - -JEMALLOC_INLINE void * -icalloc(size_t size) -{ - - if (size <= arena_maxclass) - return (arena_malloc(size, true)); - else - return (huge_malloc(size, true)); -} - -JEMALLOC_INLINE void * -ipalloc(size_t usize, size_t alignment, bool zero) -{ - void *ret; - - assert(usize != 0); - assert(usize == sa2u(usize, alignment, NULL)); - - if (usize <= arena_maxclass && alignment <= PAGE_SIZE) - ret = arena_malloc(usize, zero); - else { - size_t run_size -#ifdef JEMALLOC_CC_SILENCE - = 0 -#endif - ; - - /* - * Ideally we would only ever call sa2u() once per aligned - * allocation request, and the caller of this function has - * already done so once. However, it's rather burdensome to - * require every caller to pass in run_size, especially given - * that it's only relevant to large allocations. Therefore, - * just call it again here in order to get run_size. - */ - sa2u(usize, alignment, &run_size); - if (run_size <= arena_maxclass) { - ret = arena_palloc(choose_arena(), usize, run_size, - alignment, zero); - } else if (alignment <= chunksize) - ret = huge_malloc(usize, zero); - else - ret = huge_palloc(usize, alignment, zero); - } - - assert(((uintptr_t)ret & (alignment - 1)) == 0); - return (ret); -} - -JEMALLOC_INLINE size_t -isalloc(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - - assert(ptr != NULL); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { - /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - -#ifdef JEMALLOC_PROF - ret = arena_salloc_demote(ptr); -#else - ret = arena_salloc(ptr); -#endif - } else - ret = huge_salloc(ptr); - - return (ret); -} - -#ifdef JEMALLOC_IVSALLOC -JEMALLOC_INLINE size_t -ivsalloc(const void *ptr) -{ - - /* Return 0 if ptr is not within a chunk managed by jemalloc. */ - if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) - return (0); - - return (isalloc(ptr)); -} -#endif - -JEMALLOC_INLINE void -idalloc(void *ptr) -{ - arena_chunk_t *chunk; - - assert(ptr != NULL); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) - arena_dalloc(chunk->arena, chunk, ptr); - else - huge_dalloc(ptr, true); -} - -JEMALLOC_INLINE void * -iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, - bool no_move) -{ - void *ret; - size_t oldsize; - - assert(ptr != NULL); - assert(size != 0); - - oldsize = isalloc(ptr); - - if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) - != 0) { - size_t usize, copysize; - - /* - * Existing object alignment is inadquate; allocate new space - * and copy. - */ - if (no_move) - return (NULL); - usize = sa2u(size + extra, alignment, NULL); - if (usize == 0) - return (NULL); - ret = ipalloc(usize, alignment, zero); - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, without extra this time. */ - usize = sa2u(size, alignment, NULL); - if (usize == 0) - return (NULL); - ret = ipalloc(usize, alignment, zero); - if (ret == NULL) - return (NULL); - } - /* - * Copy at most size bytes (not size+extra), since the caller - * has no expectation that the extra bytes will be reliably - * preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - memcpy(ret, ptr, copysize); - idalloc(ptr); - return (ret); - } - - if (no_move) { - if (size <= arena_maxclass) { - return (arena_ralloc_no_move(ptr, oldsize, size, - extra, zero)); - } else { - return (huge_ralloc_no_move(ptr, oldsize, size, - extra)); - } - } else { - if (size + extra <= arena_maxclass) { - return (arena_ralloc(ptr, oldsize, size, extra, - alignment, zero)); - } else { - return (huge_ralloc(ptr, oldsize, size, extra, - alignment, zero)); - } - } -} -#endif - -#include "jemalloc/internal/prof.h" - -#undef JEMALLOC_H_INLINES -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/mb.h b/deps/jemalloc.orig/include/jemalloc/internal/mb.h deleted file mode 100644 index dc9f2a54262..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/mb.h +++ /dev/null @@ -1,108 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -void mb_write(void); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MB_C_)) -#ifdef __i386__ -/* - * According to the Intel Architecture Software Developer's Manual, current - * processors execute instructions in order from the perspective of other - * processors in a multiprocessor system, but 1) Intel reserves the right to - * change that, and 2) the compiler's optimizer could re-order instructions if - * there weren't some form of barrier. Therefore, even if running on an - * architecture that does not need memory barriers (everything through at least - * i686), an "optimizer barrier" is necessary. - */ -JEMALLOC_INLINE void -mb_write(void) -{ - -# if 0 - /* This is a true memory barrier. */ - asm volatile ("pusha;" - "xor %%eax,%%eax;" - "cpuid;" - "popa;" - : /* Outputs. */ - : /* Inputs. */ - : "memory" /* Clobbers. */ - ); -#else - /* - * This is hopefully enough to keep the compiler from reordering - * instructions around this one. - */ - asm volatile ("nop;" - : /* Outputs. */ - : /* Inputs. */ - : "memory" /* Clobbers. */ - ); -#endif -} -#elif (defined(__amd64_) || defined(__x86_64__)) -JEMALLOC_INLINE void -mb_write(void) -{ - - asm volatile ("sfence" - : /* Outputs. */ - : /* Inputs. */ - : "memory" /* Clobbers. */ - ); -} -#elif defined(__powerpc__) -JEMALLOC_INLINE void -mb_write(void) -{ - - asm volatile ("eieio" - : /* Outputs. */ - : /* Inputs. */ - : "memory" /* Clobbers. */ - ); -} -#elif defined(__sparc64__) -JEMALLOC_INLINE void -mb_write(void) -{ - - asm volatile ("membar #StoreStore" - : /* Outputs. */ - : /* Inputs. */ - : "memory" /* Clobbers. */ - ); -} -#else -/* - * This is much slower than a simple memory barrier, but the semantics of mutex - * unlock make this work. - */ -JEMALLOC_INLINE void -mb_write(void) -{ - malloc_mutex_t mtx; - - malloc_mutex_init(&mtx); - malloc_mutex_lock(&mtx); - malloc_mutex_unlock(&mtx); -} -#endif -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/mutex.h b/deps/jemalloc.orig/include/jemalloc/internal/mutex.h deleted file mode 100644 index 62947ced55e..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/mutex.h +++ /dev/null @@ -1,86 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#ifdef JEMALLOC_OSSPIN -typedef OSSpinLock malloc_mutex_t; -#else -typedef pthread_mutex_t malloc_mutex_t; -#endif - -#ifdef PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP -# define MALLOC_MUTEX_INITIALIZER PTHREAD_ADAPTIVE_MUTEX_INITIALIZER_NP -#else -# define MALLOC_MUTEX_INITIALIZER PTHREAD_MUTEX_INITIALIZER -#endif - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#ifdef JEMALLOC_LAZY_LOCK -extern bool isthreaded; -#else -# define isthreaded true -#endif - -bool malloc_mutex_init(malloc_mutex_t *mutex); -void malloc_mutex_destroy(malloc_mutex_t *mutex); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -void malloc_mutex_lock(malloc_mutex_t *mutex); -bool malloc_mutex_trylock(malloc_mutex_t *mutex); -void malloc_mutex_unlock(malloc_mutex_t *mutex); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) -JEMALLOC_INLINE void -malloc_mutex_lock(malloc_mutex_t *mutex) -{ - - if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - OSSpinLockLock(mutex); -#else - pthread_mutex_lock(mutex); -#endif - } -} - -JEMALLOC_INLINE bool -malloc_mutex_trylock(malloc_mutex_t *mutex) -{ - - if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - return (OSSpinLockTry(mutex) == false); -#else - return (pthread_mutex_trylock(mutex) != 0); -#endif - } else - return (false); -} - -JEMALLOC_INLINE void -malloc_mutex_unlock(malloc_mutex_t *mutex) -{ - - if (isthreaded) { -#ifdef JEMALLOC_OSSPIN - OSSpinLockUnlock(mutex); -#else - pthread_mutex_unlock(mutex); -#endif - } -} -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h b/deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h deleted file mode 100644 index d4f5f96d7b2..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/private_namespace.h +++ /dev/null @@ -1,195 +0,0 @@ -#define arena_bin_index JEMALLOC_N(arena_bin_index) -#define arena_boot JEMALLOC_N(arena_boot) -#define arena_dalloc JEMALLOC_N(arena_dalloc) -#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) -#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) -#define arena_malloc JEMALLOC_N(arena_malloc) -#define arena_malloc_large JEMALLOC_N(arena_malloc_large) -#define arena_malloc_small JEMALLOC_N(arena_malloc_small) -#define arena_new JEMALLOC_N(arena_new) -#define arena_palloc JEMALLOC_N(arena_palloc) -#define arena_prof_accum JEMALLOC_N(arena_prof_accum) -#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) -#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) -#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) -#define arena_purge_all JEMALLOC_N(arena_purge_all) -#define arena_ralloc JEMALLOC_N(arena_ralloc) -#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) -#define arena_run_regind JEMALLOC_N(arena_run_regind) -#define arena_salloc JEMALLOC_N(arena_salloc) -#define arena_salloc_demote JEMALLOC_N(arena_salloc_demote) -#define arena_stats_merge JEMALLOC_N(arena_stats_merge) -#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) -#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index) -#define arenas_extend JEMALLOC_N(arenas_extend) -#define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index) -#define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32) -#define atomic_add_uint64 JEMALLOC_N(atomic_add_uint64) -#define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) -#define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) -#define base_alloc JEMALLOC_N(base_alloc) -#define base_boot JEMALLOC_N(base_boot) -#define base_node_alloc JEMALLOC_N(base_node_alloc) -#define base_node_dealloc JEMALLOC_N(base_node_dealloc) -#define bitmap_full JEMALLOC_N(bitmap_full) -#define bitmap_get JEMALLOC_N(bitmap_get) -#define bitmap_info_init JEMALLOC_N(bitmap_info_init) -#define bitmap_info_ngroups JEMALLOC_N(bitmap_info_ngroups) -#define bitmap_init JEMALLOC_N(bitmap_init) -#define bitmap_set JEMALLOC_N(bitmap_set) -#define bitmap_sfu JEMALLOC_N(bitmap_sfu) -#define bitmap_size JEMALLOC_N(bitmap_size) -#define bitmap_unset JEMALLOC_N(bitmap_unset) -#define bt_init JEMALLOC_N(bt_init) -#define buferror JEMALLOC_N(buferror) -#define choose_arena JEMALLOC_N(choose_arena) -#define choose_arena_hard JEMALLOC_N(choose_arena_hard) -#define chunk_alloc JEMALLOC_N(chunk_alloc) -#define chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss) -#define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) -#define chunk_alloc_mmap_noreserve JEMALLOC_N(chunk_alloc_mmap_noreserve) -#define chunk_alloc_swap JEMALLOC_N(chunk_alloc_swap) -#define chunk_boot JEMALLOC_N(chunk_boot) -#define chunk_dealloc JEMALLOC_N(chunk_dealloc) -#define chunk_dealloc_dss JEMALLOC_N(chunk_dealloc_dss) -#define chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap) -#define chunk_dealloc_swap JEMALLOC_N(chunk_dealloc_swap) -#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) -#define chunk_in_dss JEMALLOC_N(chunk_in_dss) -#define chunk_in_swap JEMALLOC_N(chunk_in_swap) -#define chunk_mmap_boot JEMALLOC_N(chunk_mmap_boot) -#define chunk_swap_boot JEMALLOC_N(chunk_swap_boot) -#define chunk_swap_enable JEMALLOC_N(chunk_swap_enable) -#define ckh_bucket_search JEMALLOC_N(ckh_bucket_search) -#define ckh_count JEMALLOC_N(ckh_count) -#define ckh_delete JEMALLOC_N(ckh_delete) -#define ckh_evict_reloc_insert JEMALLOC_N(ckh_evict_reloc_insert) -#define ckh_insert JEMALLOC_N(ckh_insert) -#define ckh_isearch JEMALLOC_N(ckh_isearch) -#define ckh_iter JEMALLOC_N(ckh_iter) -#define ckh_new JEMALLOC_N(ckh_new) -#define ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash) -#define ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp) -#define ckh_rebuild JEMALLOC_N(ckh_rebuild) -#define ckh_remove JEMALLOC_N(ckh_remove) -#define ckh_search JEMALLOC_N(ckh_search) -#define ckh_string_hash JEMALLOC_N(ckh_string_hash) -#define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) -#define ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert) -#define ckh_try_insert JEMALLOC_N(ckh_try_insert) -#define create_zone JEMALLOC_N(create_zone) -#define ctl_boot JEMALLOC_N(ctl_boot) -#define ctl_bymib JEMALLOC_N(ctl_bymib) -#define ctl_byname JEMALLOC_N(ctl_byname) -#define ctl_nametomib JEMALLOC_N(ctl_nametomib) -#define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) -#define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) -#define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) -#define extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse) -#define extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start) -#define extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last) -#define extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new) -#define extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next) -#define extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch) -#define extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev) -#define extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch) -#define extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove) -#define extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter) -#define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) -#define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) -#define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) -#define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) -#define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) -#define extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter) -#define extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse) -#define extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start) -#define extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last) -#define extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new) -#define extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next) -#define extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch) -#define extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev) -#define extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch) -#define extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove) -#define extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter) -#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) -#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) -#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) -#define hash JEMALLOC_N(hash) -#define huge_boot JEMALLOC_N(huge_boot) -#define huge_dalloc JEMALLOC_N(huge_dalloc) -#define huge_malloc JEMALLOC_N(huge_malloc) -#define huge_palloc JEMALLOC_N(huge_palloc) -#define huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get) -#define huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set) -#define huge_ralloc JEMALLOC_N(huge_ralloc) -#define huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move) -#define huge_salloc JEMALLOC_N(huge_salloc) -#define iallocm JEMALLOC_N(iallocm) -#define icalloc JEMALLOC_N(icalloc) -#define idalloc JEMALLOC_N(idalloc) -#define imalloc JEMALLOC_N(imalloc) -#define ipalloc JEMALLOC_N(ipalloc) -#define iralloc JEMALLOC_N(iralloc) -#define isalloc JEMALLOC_N(isalloc) -#define ivsalloc JEMALLOC_N(ivsalloc) -#define jemalloc_darwin_init JEMALLOC_N(jemalloc_darwin_init) -#define jemalloc_postfork JEMALLOC_N(jemalloc_postfork) -#define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) -#define malloc_cprintf JEMALLOC_N(malloc_cprintf) -#define malloc_mutex_destroy JEMALLOC_N(malloc_mutex_destroy) -#define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) -#define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) -#define malloc_mutex_trylock JEMALLOC_N(malloc_mutex_trylock) -#define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) -#define malloc_printf JEMALLOC_N(malloc_printf) -#define malloc_write JEMALLOC_N(malloc_write) -#define mb_write JEMALLOC_N(mb_write) -#define pow2_ceil JEMALLOC_N(pow2_ceil) -#define prof_backtrace JEMALLOC_N(prof_backtrace) -#define prof_boot0 JEMALLOC_N(prof_boot0) -#define prof_boot1 JEMALLOC_N(prof_boot1) -#define prof_boot2 JEMALLOC_N(prof_boot2) -#define prof_ctx_get JEMALLOC_N(prof_ctx_get) -#define prof_ctx_set JEMALLOC_N(prof_ctx_set) -#define prof_free JEMALLOC_N(prof_free) -#define prof_gdump JEMALLOC_N(prof_gdump) -#define prof_idump JEMALLOC_N(prof_idump) -#define prof_lookup JEMALLOC_N(prof_lookup) -#define prof_malloc JEMALLOC_N(prof_malloc) -#define prof_mdump JEMALLOC_N(prof_mdump) -#define prof_realloc JEMALLOC_N(prof_realloc) -#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) -#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) -#define prof_tdata_init JEMALLOC_N(prof_tdata_init) -#define pthread_create JEMALLOC_N(pthread_create) -#define rtree_get JEMALLOC_N(rtree_get) -#define rtree_get_locked JEMALLOC_N(rtree_get_locked) -#define rtree_new JEMALLOC_N(rtree_new) -#define rtree_set JEMALLOC_N(rtree_set) -#define s2u JEMALLOC_N(s2u) -#define sa2u JEMALLOC_N(sa2u) -#define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index) -#define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index) -#define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index) -#define stats_cactive_add JEMALLOC_N(stats_cactive_add) -#define stats_cactive_get JEMALLOC_N(stats_cactive_get) -#define stats_cactive_sub JEMALLOC_N(stats_cactive_sub) -#define stats_print JEMALLOC_N(stats_print) -#define szone2ozone JEMALLOC_N(szone2ozone) -#define tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy) -#define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) -#define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) -#define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) -#define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) -#define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) -#define tcache_boot JEMALLOC_N(tcache_boot) -#define tcache_create JEMALLOC_N(tcache_create) -#define tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large) -#define tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small) -#define tcache_destroy JEMALLOC_N(tcache_destroy) -#define tcache_event JEMALLOC_N(tcache_event) -#define tcache_get JEMALLOC_N(tcache_get) -#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) -#define thread_allocated_get JEMALLOC_N(thread_allocated_get) -#define thread_allocated_get_hard JEMALLOC_N(thread_allocated_get_hard) -#define u2s JEMALLOC_N(u2s) diff --git a/deps/jemalloc.orig/include/jemalloc/internal/prn.h b/deps/jemalloc.orig/include/jemalloc/internal/prn.h deleted file mode 100644 index 0709d708012..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/prn.h +++ /dev/null @@ -1,60 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -/* - * Simple linear congruential pseudo-random number generator: - * - * prn(y) = (a*x + c) % m - * - * where the following constants ensure maximal period: - * - * a == Odd number (relatively prime to 2^n), and (a-1) is a multiple of 4. - * c == Odd number (relatively prime to 2^n). - * m == 2^32 - * - * See Knuth's TAOCP 3rd Ed., Vol. 2, pg. 17 for details on these constraints. - * - * This choice of m has the disadvantage that the quality of the bits is - * proportional to bit position. For example. the lowest bit has a cycle of 2, - * the next has a cycle of 4, etc. For this reason, we prefer to use the upper - * bits. - * - * Macro parameters: - * uint32_t r : Result. - * unsigned lg_range : (0..32], number of least significant bits to return. - * uint32_t state : Seed value. - * const uint32_t a, c : See above discussion. - */ -#define prn32(r, lg_range, state, a, c) do { \ - assert(lg_range > 0); \ - assert(lg_range <= 32); \ - \ - r = (state * (a)) + (c); \ - state = r; \ - r >>= (32 - lg_range); \ -} while (false) - -/* Same as prn32(), but 64 bits of pseudo-randomness, using uint64_t. */ -#define prn64(r, lg_range, state, a, c) do { \ - assert(lg_range > 0); \ - assert(lg_range <= 64); \ - \ - r = (state * (a)) + (c); \ - state = r; \ - r >>= (64 - lg_range); \ -} while (false) - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/prof.h b/deps/jemalloc.orig/include/jemalloc/internal/prof.h deleted file mode 100644 index e9064ba6e73..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/prof.h +++ /dev/null @@ -1,547 +0,0 @@ -#ifdef JEMALLOC_PROF -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -typedef struct prof_bt_s prof_bt_t; -typedef struct prof_cnt_s prof_cnt_t; -typedef struct prof_thr_cnt_s prof_thr_cnt_t; -typedef struct prof_ctx_s prof_ctx_t; -typedef struct prof_tdata_s prof_tdata_t; - -/* Option defaults. */ -#define PROF_PREFIX_DEFAULT "jeprof" -#define LG_PROF_BT_MAX_DEFAULT 7 -#define LG_PROF_SAMPLE_DEFAULT 0 -#define LG_PROF_INTERVAL_DEFAULT -1 -#define LG_PROF_TCMAX_DEFAULT -1 - -/* - * Hard limit on stack backtrace depth. Note that the version of - * prof_backtrace() that is based on __builtin_return_address() necessarily has - * a hard-coded number of backtrace frame handlers. - */ -#if (defined(JEMALLOC_PROF_LIBGCC) || defined(JEMALLOC_PROF_LIBUNWIND)) -# define LG_PROF_BT_MAX ((ZU(1) << (LG_SIZEOF_PTR+3)) - 1) -#else -# define LG_PROF_BT_MAX 7 /* >= LG_PROF_BT_MAX_DEFAULT */ -#endif -#define PROF_BT_MAX (1U << LG_PROF_BT_MAX) - -/* Initial hash table size. */ -#define PROF_CKH_MINITEMS 64 - -/* Size of memory buffer to use when writing dump files. */ -#define PROF_DUMP_BUF_SIZE 65536 - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -struct prof_bt_s { - /* Backtrace, stored as len program counters. */ - void **vec; - unsigned len; -}; - -#ifdef JEMALLOC_PROF_LIBGCC -/* Data structure passed to libgcc _Unwind_Backtrace() callback functions. */ -typedef struct { - prof_bt_t *bt; - unsigned nignore; - unsigned max; -} prof_unwind_data_t; -#endif - -struct prof_cnt_s { - /* - * Profiling counters. An allocation/deallocation pair can operate on - * different prof_thr_cnt_t objects that are linked into the same - * prof_ctx_t cnts_ql, so it is possible for the cur* counters to go - * negative. In principle it is possible for the *bytes counters to - * overflow/underflow, but a general solution would require something - * like 128-bit counters; this implementation doesn't bother to solve - * that problem. - */ - int64_t curobjs; - int64_t curbytes; - uint64_t accumobjs; - uint64_t accumbytes; -}; - -struct prof_thr_cnt_s { - /* Linkage into prof_ctx_t's cnts_ql. */ - ql_elm(prof_thr_cnt_t) cnts_link; - - /* Linkage into thread's LRU. */ - ql_elm(prof_thr_cnt_t) lru_link; - - /* - * Associated context. If a thread frees an object that it did not - * allocate, it is possible that the context is not cached in the - * thread's hash table, in which case it must be able to look up the - * context, insert a new prof_thr_cnt_t into the thread's hash table, - * and link it into the prof_ctx_t's cnts_ql. - */ - prof_ctx_t *ctx; - - /* - * Threads use memory barriers to update the counters. Since there is - * only ever one writer, the only challenge is for the reader to get a - * consistent read of the counters. - * - * The writer uses this series of operations: - * - * 1) Increment epoch to an odd number. - * 2) Update counters. - * 3) Increment epoch to an even number. - * - * The reader must assure 1) that the epoch is even while it reads the - * counters, and 2) that the epoch doesn't change between the time it - * starts and finishes reading the counters. - */ - unsigned epoch; - - /* Profiling counters. */ - prof_cnt_t cnts; -}; - -struct prof_ctx_s { - /* Associated backtrace. */ - prof_bt_t *bt; - - /* Protects cnt_merged and cnts_ql. */ - malloc_mutex_t lock; - - /* Temporary storage for summation during dump. */ - prof_cnt_t cnt_summed; - - /* When threads exit, they merge their stats into cnt_merged. */ - prof_cnt_t cnt_merged; - - /* - * List of profile counters, one for each thread that has allocated in - * this context. - */ - ql_head(prof_thr_cnt_t) cnts_ql; -}; - -struct prof_tdata_s { - /* - * Hash of (prof_bt_t *)-->(prof_thr_cnt_t *). Each thread keeps a - * cache of backtraces, with associated thread-specific prof_thr_cnt_t - * objects. Other threads may read the prof_thr_cnt_t contents, but no - * others will ever write them. - * - * Upon thread exit, the thread must merge all the prof_thr_cnt_t - * counter data into the associated prof_ctx_t objects, and unlink/free - * the prof_thr_cnt_t objects. - */ - ckh_t bt2cnt; - - /* LRU for contents of bt2cnt. */ - ql_head(prof_thr_cnt_t) lru_ql; - - /* Backtrace vector, used for calls to prof_backtrace(). */ - void **vec; - - /* Sampling state. */ - uint64_t prn_state; - uint64_t threshold; - uint64_t accum; -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern bool opt_prof; -/* - * Even if opt_prof is true, sampling can be temporarily disabled by setting - * opt_prof_active to false. No locking is used when updating opt_prof_active, - * so there are no guarantees regarding how long it will take for all threads - * to notice state changes. - */ -extern bool opt_prof_active; -extern size_t opt_lg_prof_bt_max; /* Maximum backtrace depth. */ -extern size_t opt_lg_prof_sample; /* Mean bytes between samples. */ -extern ssize_t opt_lg_prof_interval; /* lg(prof_interval). */ -extern bool opt_prof_gdump; /* High-water memory dumping. */ -extern bool opt_prof_leak; /* Dump leak summary at exit. */ -extern bool opt_prof_accum; /* Report cumulative bytes. */ -extern ssize_t opt_lg_prof_tcmax; /* lg(max per thread bactrace cache) */ -extern char opt_prof_prefix[PATH_MAX + 1]; - -/* - * Profile dump interval, measured in bytes allocated. Each arena triggers a - * profile dump when it reaches this threshold. The effect is that the - * interval between profile dumps averages prof_interval, though the actual - * interval between dumps will tend to be sporadic, and the interval will be a - * maximum of approximately (prof_interval * narenas). - */ -extern uint64_t prof_interval; - -/* - * If true, promote small sampled objects to large objects, since small run - * headers do not have embedded profile context pointers. - */ -extern bool prof_promote; - -/* (1U << opt_lg_prof_bt_max). */ -extern unsigned prof_bt_max; - -/* Thread-specific backtrace cache, used to reduce bt2ctx contention. */ -#ifndef NO_TLS -extern __thread prof_tdata_t *prof_tdata_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -# define PROF_TCACHE_GET() prof_tdata_tls -# define PROF_TCACHE_SET(v) do { \ - prof_tdata_tls = (v); \ - pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ -} while (0) -#else -# define PROF_TCACHE_GET() \ - ((prof_tdata_t *)pthread_getspecific(prof_tdata_tsd)) -# define PROF_TCACHE_SET(v) do { \ - pthread_setspecific(prof_tdata_tsd, (void *)(v)); \ -} while (0) -#endif -/* - * Same contents as b2cnt_tls, but initialized such that the TSD destructor is - * called when a thread exits, so that prof_tdata_tls contents can be merged, - * unlinked, and deallocated. - */ -extern pthread_key_t prof_tdata_tsd; - -void bt_init(prof_bt_t *bt, void **vec); -void prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max); -prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); -void prof_idump(void); -bool prof_mdump(const char *filename); -void prof_gdump(void); -prof_tdata_t *prof_tdata_init(void); -void prof_boot0(void); -void prof_boot1(void); -bool prof_boot2(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#define PROF_ALLOC_PREP(nignore, size, ret) do { \ - prof_tdata_t *prof_tdata; \ - prof_bt_t bt; \ - \ - assert(size == s2u(size)); \ - \ - prof_tdata = PROF_TCACHE_GET(); \ - if (prof_tdata == NULL) { \ - prof_tdata = prof_tdata_init(); \ - if (prof_tdata == NULL) { \ - ret = NULL; \ - break; \ - } \ - } \ - \ - if (opt_prof_active == false) { \ - /* Sampling is currently inactive, so avoid sampling. */\ - ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ - } else if (opt_lg_prof_sample == 0) { \ - /* Don't bother with sampling logic, since sampling */\ - /* interval is 1. */\ - bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore, prof_bt_max); \ - ret = prof_lookup(&bt); \ - } else { \ - if (prof_tdata->threshold == 0) { \ - /* Initialize. Seed the prng differently for */\ - /* each thread. */\ - prof_tdata->prn_state = \ - (uint64_t)(uintptr_t)&size; \ - prof_sample_threshold_update(prof_tdata); \ - } \ - \ - /* Determine whether to capture a backtrace based on */\ - /* whether size is enough for prof_accum to reach */\ - /* prof_tdata->threshold. However, delay updating */\ - /* these variables until prof_{m,re}alloc(), because */\ - /* we don't know for sure that the allocation will */\ - /* succeed. */\ - /* */\ - /* Use subtraction rather than addition to avoid */\ - /* potential integer overflow. */\ - if (size >= prof_tdata->threshold - \ - prof_tdata->accum) { \ - bt_init(&bt, prof_tdata->vec); \ - prof_backtrace(&bt, nignore, prof_bt_max); \ - ret = prof_lookup(&bt); \ - } else \ - ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ - } \ -} while (0) - -#ifndef JEMALLOC_ENABLE_INLINE -void prof_sample_threshold_update(prof_tdata_t *prof_tdata); -prof_ctx_t *prof_ctx_get(const void *ptr); -void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); -bool prof_sample_accum_update(size_t size); -void prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt); -void prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, - size_t old_size, prof_ctx_t *old_ctx); -void prof_free(const void *ptr, size_t size); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_PROF_C_)) -JEMALLOC_INLINE void -prof_sample_threshold_update(prof_tdata_t *prof_tdata) -{ - uint64_t r; - double u; - - /* - * Compute sample threshold as a geometrically distributed random - * variable with mean (2^opt_lg_prof_sample). - * - * __ __ - * | log(u) | 1 - * prof_tdata->threshold = | -------- |, where p = ------------------- - * | log(1-p) | opt_lg_prof_sample - * 2 - * - * For more information on the math, see: - * - * Non-Uniform Random Variate Generation - * Luc Devroye - * Springer-Verlag, New York, 1986 - * pp 500 - * (http://cg.scs.carleton.ca/~luc/rnbookindex.html) - */ - prn64(r, 53, prof_tdata->prn_state, - (uint64_t)6364136223846793005LLU, (uint64_t)1442695040888963407LLU); - u = (double)r * (1.0/9007199254740992.0L); - prof_tdata->threshold = (uint64_t)(log(u) / - log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) - + (uint64_t)1U; -} - -JEMALLOC_INLINE prof_ctx_t * -prof_ctx_get(const void *ptr) -{ - prof_ctx_t *ret; - arena_chunk_t *chunk; - - assert(ptr != NULL); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { - /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - - ret = arena_prof_ctx_get(ptr); - } else - ret = huge_prof_ctx_get(ptr); - - return (ret); -} - -JEMALLOC_INLINE void -prof_ctx_set(const void *ptr, prof_ctx_t *ctx) -{ - arena_chunk_t *chunk; - - assert(ptr != NULL); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk != ptr) { - /* Region. */ - dassert(chunk->arena->magic == ARENA_MAGIC); - - arena_prof_ctx_set(ptr, ctx); - } else - huge_prof_ctx_set(ptr, ctx); -} - -JEMALLOC_INLINE bool -prof_sample_accum_update(size_t size) -{ - prof_tdata_t *prof_tdata; - - /* Sampling logic is unnecessary if the interval is 1. */ - assert(opt_lg_prof_sample != 0); - - prof_tdata = PROF_TCACHE_GET(); - assert(prof_tdata != NULL); - - /* Take care to avoid integer overflow. */ - if (size >= prof_tdata->threshold - prof_tdata->accum) { - prof_tdata->accum -= (prof_tdata->threshold - size); - /* Compute new sample threshold. */ - prof_sample_threshold_update(prof_tdata); - while (prof_tdata->accum >= prof_tdata->threshold) { - prof_tdata->accum -= prof_tdata->threshold; - prof_sample_threshold_update(prof_tdata); - } - return (false); - } else { - prof_tdata->accum += size; - return (true); - } -} - -JEMALLOC_INLINE void -prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) -{ - - assert(ptr != NULL); - assert(size == isalloc(ptr)); - - if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(size)) { - /* - * Don't sample. For malloc()-like allocation, it is - * always possible to tell in advance how large an - * object's usable size will be, so there should never - * be a difference between the size passed to - * PROF_ALLOC_PREP() and prof_malloc(). - */ - assert((uintptr_t)cnt == (uintptr_t)1U); - } - } - - if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); - - cnt->epoch++; - /*********/ - mb_write(); - /*********/ - cnt->cnts.curobjs++; - cnt->cnts.curbytes += size; - if (opt_prof_accum) { - cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += size; - } - /*********/ - mb_write(); - /*********/ - cnt->epoch++; - /*********/ - mb_write(); - /*********/ - } else - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); -} - -JEMALLOC_INLINE void -prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, - size_t old_size, prof_ctx_t *old_ctx) -{ - prof_thr_cnt_t *told_cnt; - - assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); - - if (ptr != NULL) { - assert(size == isalloc(ptr)); - if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(size)) { - /* - * Don't sample. The size passed to - * PROF_ALLOC_PREP() was larger than what - * actually got allocated, so a backtrace was - * captured for this allocation, even though - * its actual size was insufficient to cross - * the sample threshold. - */ - cnt = (prof_thr_cnt_t *)(uintptr_t)1U; - } - } - } - - if ((uintptr_t)old_ctx > (uintptr_t)1U) { - told_cnt = prof_lookup(old_ctx->bt); - if (told_cnt == NULL) { - /* - * It's too late to propagate OOM for this realloc(), - * so operate directly on old_cnt->ctx->cnt_merged. - */ - malloc_mutex_lock(&old_ctx->lock); - old_ctx->cnt_merged.curobjs--; - old_ctx->cnt_merged.curbytes -= old_size; - malloc_mutex_unlock(&old_ctx->lock); - told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; - } - } else - told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; - - if ((uintptr_t)told_cnt > (uintptr_t)1U) - told_cnt->epoch++; - if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); - cnt->epoch++; - } else - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); - /*********/ - mb_write(); - /*********/ - if ((uintptr_t)told_cnt > (uintptr_t)1U) { - told_cnt->cnts.curobjs--; - told_cnt->cnts.curbytes -= old_size; - } - if ((uintptr_t)cnt > (uintptr_t)1U) { - cnt->cnts.curobjs++; - cnt->cnts.curbytes += size; - if (opt_prof_accum) { - cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += size; - } - } - /*********/ - mb_write(); - /*********/ - if ((uintptr_t)told_cnt > (uintptr_t)1U) - told_cnt->epoch++; - if ((uintptr_t)cnt > (uintptr_t)1U) - cnt->epoch++; - /*********/ - mb_write(); /* Not strictly necessary. */ -} - -JEMALLOC_INLINE void -prof_free(const void *ptr, size_t size) -{ - prof_ctx_t *ctx = prof_ctx_get(ptr); - - if ((uintptr_t)ctx > (uintptr_t)1) { - assert(size == isalloc(ptr)); - prof_thr_cnt_t *tcnt = prof_lookup(ctx->bt); - - if (tcnt != NULL) { - tcnt->epoch++; - /*********/ - mb_write(); - /*********/ - tcnt->cnts.curobjs--; - tcnt->cnts.curbytes -= size; - /*********/ - mb_write(); - /*********/ - tcnt->epoch++; - /*********/ - mb_write(); - /*********/ - } else { - /* - * OOM during free() cannot be propagated, so operate - * directly on cnt->ctx->cnt_merged. - */ - malloc_mutex_lock(&ctx->lock); - ctx->cnt_merged.curobjs--; - ctx->cnt_merged.curbytes -= size; - malloc_mutex_unlock(&ctx->lock); - } - } -} -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ -#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/ql.h b/deps/jemalloc.orig/include/jemalloc/internal/ql.h deleted file mode 100644 index a9ed2393f0c..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/ql.h +++ /dev/null @@ -1,83 +0,0 @@ -/* - * List definitions. - */ -#define ql_head(a_type) \ -struct { \ - a_type *qlh_first; \ -} - -#define ql_head_initializer(a_head) {NULL} - -#define ql_elm(a_type) qr(a_type) - -/* List functions. */ -#define ql_new(a_head) do { \ - (a_head)->qlh_first = NULL; \ -} while (0) - -#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field) - -#define ql_first(a_head) ((a_head)->qlh_first) - -#define ql_last(a_head, a_field) \ - ((ql_first(a_head) != NULL) \ - ? qr_prev(ql_first(a_head), a_field) : NULL) - -#define ql_next(a_head, a_elm, a_field) \ - ((ql_last(a_head, a_field) != (a_elm)) \ - ? qr_next((a_elm), a_field) : NULL) - -#define ql_prev(a_head, a_elm, a_field) \ - ((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) \ - : NULL) - -#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ - qr_before_insert((a_qlelm), (a_elm), a_field); \ - if (ql_first(a_head) == (a_qlelm)) { \ - ql_first(a_head) = (a_elm); \ - } \ -} while (0) - -#define ql_after_insert(a_qlelm, a_elm, a_field) \ - qr_after_insert((a_qlelm), (a_elm), a_field) - -#define ql_head_insert(a_head, a_elm, a_field) do { \ - if (ql_first(a_head) != NULL) { \ - qr_before_insert(ql_first(a_head), (a_elm), a_field); \ - } \ - ql_first(a_head) = (a_elm); \ -} while (0) - -#define ql_tail_insert(a_head, a_elm, a_field) do { \ - if (ql_first(a_head) != NULL) { \ - qr_before_insert(ql_first(a_head), (a_elm), a_field); \ - } \ - ql_first(a_head) = qr_next((a_elm), a_field); \ -} while (0) - -#define ql_remove(a_head, a_elm, a_field) do { \ - if (ql_first(a_head) == (a_elm)) { \ - ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ - } \ - if (ql_first(a_head) != (a_elm)) { \ - qr_remove((a_elm), a_field); \ - } else { \ - ql_first(a_head) = NULL; \ - } \ -} while (0) - -#define ql_head_remove(a_head, a_type, a_field) do { \ - a_type *t = ql_first(a_head); \ - ql_remove((a_head), t, a_field); \ -} while (0) - -#define ql_tail_remove(a_head, a_type, a_field) do { \ - a_type *t = ql_last(a_head, a_field); \ - ql_remove((a_head), t, a_field); \ -} while (0) - -#define ql_foreach(a_var, a_head, a_field) \ - qr_foreach((a_var), ql_first(a_head), a_field) - -#define ql_reverse_foreach(a_var, a_head, a_field) \ - qr_reverse_foreach((a_var), ql_first(a_head), a_field) diff --git a/deps/jemalloc.orig/include/jemalloc/internal/qr.h b/deps/jemalloc.orig/include/jemalloc/internal/qr.h deleted file mode 100644 index fe22352fedd..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/qr.h +++ /dev/null @@ -1,67 +0,0 @@ -/* Ring definitions. */ -#define qr(a_type) \ -struct { \ - a_type *qre_next; \ - a_type *qre_prev; \ -} - -/* Ring functions. */ -#define qr_new(a_qr, a_field) do { \ - (a_qr)->a_field.qre_next = (a_qr); \ - (a_qr)->a_field.qre_prev = (a_qr); \ -} while (0) - -#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next) - -#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev) - -#define qr_before_insert(a_qrelm, a_qr, a_field) do { \ - (a_qr)->a_field.qre_prev = (a_qrelm)->a_field.qre_prev; \ - (a_qr)->a_field.qre_next = (a_qrelm); \ - (a_qr)->a_field.qre_prev->a_field.qre_next = (a_qr); \ - (a_qrelm)->a_field.qre_prev = (a_qr); \ -} while (0) - -#define qr_after_insert(a_qrelm, a_qr, a_field) \ - do \ - { \ - (a_qr)->a_field.qre_next = (a_qrelm)->a_field.qre_next; \ - (a_qr)->a_field.qre_prev = (a_qrelm); \ - (a_qr)->a_field.qre_next->a_field.qre_prev = (a_qr); \ - (a_qrelm)->a_field.qre_next = (a_qr); \ - } while (0) - -#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ - void *t; \ - (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ - (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ - t = (a_qr_a)->a_field.qre_prev; \ - (a_qr_a)->a_field.qre_prev = (a_qr_b)->a_field.qre_prev; \ - (a_qr_b)->a_field.qre_prev = t; \ -} while (0) - -/* qr_meld() and qr_split() are functionally equivalent, so there's no need to - * have two copies of the code. */ -#define qr_split(a_qr_a, a_qr_b, a_field) \ - qr_meld((a_qr_a), (a_qr_b), a_field) - -#define qr_remove(a_qr, a_field) do { \ - (a_qr)->a_field.qre_prev->a_field.qre_next \ - = (a_qr)->a_field.qre_next; \ - (a_qr)->a_field.qre_next->a_field.qre_prev \ - = (a_qr)->a_field.qre_prev; \ - (a_qr)->a_field.qre_next = (a_qr); \ - (a_qr)->a_field.qre_prev = (a_qr); \ -} while (0) - -#define qr_foreach(var, a_qr, a_field) \ - for ((var) = (a_qr); \ - (var) != NULL; \ - (var) = (((var)->a_field.qre_next != (a_qr)) \ - ? (var)->a_field.qre_next : NULL)) - -#define qr_reverse_foreach(var, a_qr, a_field) \ - for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ - (var) != NULL; \ - (var) = (((var) != (a_qr)) \ - ? (var)->a_field.qre_prev : NULL)) diff --git a/deps/jemalloc.orig/include/jemalloc/internal/rb.h b/deps/jemalloc.orig/include/jemalloc/internal/rb.h deleted file mode 100644 index ee9b009d235..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/rb.h +++ /dev/null @@ -1,973 +0,0 @@ -/*- - ******************************************************************************* - * - * cpp macro implementation of left-leaning 2-3 red-black trees. Parent - * pointers are not used, and color bits are stored in the least significant - * bit of right-child pointers (if RB_COMPACT is defined), thus making node - * linkage as compact as is possible for red-black trees. - * - * Usage: - * - * #include - * #include - * #define NDEBUG // (Optional, see assert(3).) - * #include - * #define RB_COMPACT // (Optional, embed color bits in right-child pointers.) - * #include - * ... - * - ******************************************************************************* - */ - -#ifndef RB_H_ -#define RB_H_ - -#if 0 -__FBSDID("$FreeBSD: head/lib/libc/stdlib/rb.h 204493 2010-02-28 22:57:13Z jasone $"); -#endif - -#ifdef RB_COMPACT -/* Node structure. */ -#define rb_node(a_type) \ -struct { \ - a_type *rbn_left; \ - a_type *rbn_right_red; \ -} -#else -#define rb_node(a_type) \ -struct { \ - a_type *rbn_left; \ - a_type *rbn_right; \ - bool rbn_red; \ -} -#endif - -/* Root structure. */ -#define rb_tree(a_type) \ -struct { \ - a_type *rbt_root; \ - a_type rbt_nil; \ -} - -/* Left accessors. */ -#define rbtn_left_get(a_type, a_field, a_node) \ - ((a_node)->a_field.rbn_left) -#define rbtn_left_set(a_type, a_field, a_node, a_left) do { \ - (a_node)->a_field.rbn_left = a_left; \ -} while (0) - -#ifdef RB_COMPACT -/* Right accessors. */ -#define rbtn_right_get(a_type, a_field, a_node) \ - ((a_type *) (((intptr_t) (a_node)->a_field.rbn_right_red) \ - & ((ssize_t)-2))) -#define rbtn_right_set(a_type, a_field, a_node, a_right) do { \ - (a_node)->a_field.rbn_right_red = (a_type *) (((uintptr_t) a_right) \ - | (((uintptr_t) (a_node)->a_field.rbn_right_red) & ((size_t)1))); \ -} while (0) - -/* Color accessors. */ -#define rbtn_red_get(a_type, a_field, a_node) \ - ((bool) (((uintptr_t) (a_node)->a_field.rbn_right_red) \ - & ((size_t)1))) -#define rbtn_color_set(a_type, a_field, a_node, a_red) do { \ - (a_node)->a_field.rbn_right_red = (a_type *) ((((intptr_t) \ - (a_node)->a_field.rbn_right_red) & ((ssize_t)-2)) \ - | ((ssize_t)a_red)); \ -} while (0) -#define rbtn_red_set(a_type, a_field, a_node) do { \ - (a_node)->a_field.rbn_right_red = (a_type *) (((uintptr_t) \ - (a_node)->a_field.rbn_right_red) | ((size_t)1)); \ -} while (0) -#define rbtn_black_set(a_type, a_field, a_node) do { \ - (a_node)->a_field.rbn_right_red = (a_type *) (((intptr_t) \ - (a_node)->a_field.rbn_right_red) & ((ssize_t)-2)); \ -} while (0) -#else -/* Right accessors. */ -#define rbtn_right_get(a_type, a_field, a_node) \ - ((a_node)->a_field.rbn_right) -#define rbtn_right_set(a_type, a_field, a_node, a_right) do { \ - (a_node)->a_field.rbn_right = a_right; \ -} while (0) - -/* Color accessors. */ -#define rbtn_red_get(a_type, a_field, a_node) \ - ((a_node)->a_field.rbn_red) -#define rbtn_color_set(a_type, a_field, a_node, a_red) do { \ - (a_node)->a_field.rbn_red = (a_red); \ -} while (0) -#define rbtn_red_set(a_type, a_field, a_node) do { \ - (a_node)->a_field.rbn_red = true; \ -} while (0) -#define rbtn_black_set(a_type, a_field, a_node) do { \ - (a_node)->a_field.rbn_red = false; \ -} while (0) -#endif - -/* Node initializer. */ -#define rbt_node_new(a_type, a_field, a_rbt, a_node) do { \ - rbtn_left_set(a_type, a_field, (a_node), &(a_rbt)->rbt_nil); \ - rbtn_right_set(a_type, a_field, (a_node), &(a_rbt)->rbt_nil); \ - rbtn_red_set(a_type, a_field, (a_node)); \ -} while (0) - -/* Tree initializer. */ -#define rb_new(a_type, a_field, a_rbt) do { \ - (a_rbt)->rbt_root = &(a_rbt)->rbt_nil; \ - rbt_node_new(a_type, a_field, a_rbt, &(a_rbt)->rbt_nil); \ - rbtn_black_set(a_type, a_field, &(a_rbt)->rbt_nil); \ -} while (0) - -/* Internal utility macros. */ -#define rbtn_first(a_type, a_field, a_rbt, a_root, r_node) do { \ - (r_node) = (a_root); \ - if ((r_node) != &(a_rbt)->rbt_nil) { \ - for (; \ - rbtn_left_get(a_type, a_field, (r_node)) != &(a_rbt)->rbt_nil;\ - (r_node) = rbtn_left_get(a_type, a_field, (r_node))) { \ - } \ - } \ -} while (0) - -#define rbtn_last(a_type, a_field, a_rbt, a_root, r_node) do { \ - (r_node) = (a_root); \ - if ((r_node) != &(a_rbt)->rbt_nil) { \ - for (; rbtn_right_get(a_type, a_field, (r_node)) != \ - &(a_rbt)->rbt_nil; (r_node) = rbtn_right_get(a_type, a_field, \ - (r_node))) { \ - } \ - } \ -} while (0) - -#define rbtn_rotate_left(a_type, a_field, a_node, r_node) do { \ - (r_node) = rbtn_right_get(a_type, a_field, (a_node)); \ - rbtn_right_set(a_type, a_field, (a_node), \ - rbtn_left_get(a_type, a_field, (r_node))); \ - rbtn_left_set(a_type, a_field, (r_node), (a_node)); \ -} while (0) - -#define rbtn_rotate_right(a_type, a_field, a_node, r_node) do { \ - (r_node) = rbtn_left_get(a_type, a_field, (a_node)); \ - rbtn_left_set(a_type, a_field, (a_node), \ - rbtn_right_get(a_type, a_field, (r_node))); \ - rbtn_right_set(a_type, a_field, (r_node), (a_node)); \ -} while (0) - -/* - * The rb_proto() macro generates function prototypes that correspond to the - * functions generated by an equivalently parameterized call to rb_gen(). - */ - -#define rb_proto(a_attr, a_prefix, a_rbt_type, a_type) \ -a_attr void \ -a_prefix##new(a_rbt_type *rbtree); \ -a_attr a_type * \ -a_prefix##first(a_rbt_type *rbtree); \ -a_attr a_type * \ -a_prefix##last(a_rbt_type *rbtree); \ -a_attr a_type * \ -a_prefix##next(a_rbt_type *rbtree, a_type *node); \ -a_attr a_type * \ -a_prefix##prev(a_rbt_type *rbtree, a_type *node); \ -a_attr a_type * \ -a_prefix##search(a_rbt_type *rbtree, a_type *key); \ -a_attr a_type * \ -a_prefix##nsearch(a_rbt_type *rbtree, a_type *key); \ -a_attr a_type * \ -a_prefix##psearch(a_rbt_type *rbtree, a_type *key); \ -a_attr void \ -a_prefix##insert(a_rbt_type *rbtree, a_type *node); \ -a_attr void \ -a_prefix##remove(a_rbt_type *rbtree, a_type *node); \ -a_attr a_type * \ -a_prefix##iter(a_rbt_type *rbtree, a_type *start, a_type *(*cb)( \ - a_rbt_type *, a_type *, void *), void *arg); \ -a_attr a_type * \ -a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ - a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg); - -/* - * The rb_gen() macro generates a type-specific red-black tree implementation, - * based on the above cpp macros. - * - * Arguments: - * - * a_attr : Function attribute for generated functions (ex: static). - * a_prefix : Prefix for generated functions (ex: ex_). - * a_rb_type : Type for red-black tree data structure (ex: ex_t). - * a_type : Type for red-black tree node data structure (ex: ex_node_t). - * a_field : Name of red-black tree node linkage (ex: ex_link). - * a_cmp : Node comparison function name, with the following prototype: - * int (a_cmp *)(a_type *a_node, a_type *a_other); - * ^^^^^^ - * or a_key - * Interpretation of comparision function return values: - * -1 : a_node < a_other - * 0 : a_node == a_other - * 1 : a_node > a_other - * In all cases, the a_node or a_key macro argument is the first - * argument to the comparison function, which makes it possible - * to write comparison functions that treat the first argument - * specially. - * - * Assuming the following setup: - * - * typedef struct ex_node_s ex_node_t; - * struct ex_node_s { - * rb_node(ex_node_t) ex_link; - * }; - * typedef rb_tree(ex_node_t) ex_t; - * rb_gen(static, ex_, ex_t, ex_node_t, ex_link, ex_cmp) - * - * The following API is generated: - * - * static void - * ex_new(ex_t *extree); - * Description: Initialize a red-black tree structure. - * Args: - * extree: Pointer to an uninitialized red-black tree object. - * - * static ex_node_t * - * ex_first(ex_t *extree); - * static ex_node_t * - * ex_last(ex_t *extree); - * Description: Get the first/last node in extree. - * Args: - * extree: Pointer to an initialized red-black tree object. - * Ret: First/last node in extree, or NULL if extree is empty. - * - * static ex_node_t * - * ex_next(ex_t *extree, ex_node_t *node); - * static ex_node_t * - * ex_prev(ex_t *extree, ex_node_t *node); - * Description: Get node's successor/predecessor. - * Args: - * extree: Pointer to an initialized red-black tree object. - * node : A node in extree. - * Ret: node's successor/predecessor in extree, or NULL if node is - * last/first. - * - * static ex_node_t * - * ex_search(ex_t *extree, ex_node_t *key); - * Description: Search for node that matches key. - * Args: - * extree: Pointer to an initialized red-black tree object. - * key : Search key. - * Ret: Node in extree that matches key, or NULL if no match. - * - * static ex_node_t * - * ex_nsearch(ex_t *extree, ex_node_t *key); - * static ex_node_t * - * ex_psearch(ex_t *extree, ex_node_t *key); - * Description: Search for node that matches key. If no match is found, - * return what would be key's successor/predecessor, were - * key in extree. - * Args: - * extree: Pointer to an initialized red-black tree object. - * key : Search key. - * Ret: Node in extree that matches key, or if no match, hypothetical - * node's successor/predecessor (NULL if no successor/predecessor). - * - * static void - * ex_insert(ex_t *extree, ex_node_t *node); - * Description: Insert node into extree. - * Args: - * extree: Pointer to an initialized red-black tree object. - * node : Node to be inserted into extree. - * - * static void - * ex_remove(ex_t *extree, ex_node_t *node); - * Description: Remove node from extree. - * Args: - * extree: Pointer to an initialized red-black tree object. - * node : Node in extree to be removed. - * - * static ex_node_t * - * ex_iter(ex_t *extree, ex_node_t *start, ex_node_t *(*cb)(ex_t *, - * ex_node_t *, void *), void *arg); - * static ex_node_t * - * ex_reverse_iter(ex_t *extree, ex_node_t *start, ex_node *(*cb)(ex_t *, - * ex_node_t *, void *), void *arg); - * Description: Iterate forward/backward over extree, starting at node. - * If extree is modified, iteration must be immediately - * terminated by the callback function that causes the - * modification. - * Args: - * extree: Pointer to an initialized red-black tree object. - * start : Node at which to start iteration, or NULL to start at - * first/last node. - * cb : Callback function, which is called for each node during - * iteration. Under normal circumstances the callback function - * should return NULL, which causes iteration to continue. If a - * callback function returns non-NULL, iteration is immediately - * terminated and the non-NULL return value is returned by the - * iterator. This is useful for re-starting iteration after - * modifying extree. - * arg : Opaque pointer passed to cb(). - * Ret: NULL if iteration completed, or the non-NULL callback return value - * that caused termination of the iteration. - */ -#define rb_gen(a_attr, a_prefix, a_rbt_type, a_type, a_field, a_cmp) \ -a_attr void \ -a_prefix##new(a_rbt_type *rbtree) { \ - rb_new(a_type, a_field, rbtree); \ -} \ -a_attr a_type * \ -a_prefix##first(a_rbt_type *rbtree) { \ - a_type *ret; \ - rbtn_first(a_type, a_field, rbtree, rbtree->rbt_root, ret); \ - if (ret == &rbtree->rbt_nil) { \ - ret = NULL; \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##last(a_rbt_type *rbtree) { \ - a_type *ret; \ - rbtn_last(a_type, a_field, rbtree, rbtree->rbt_root, ret); \ - if (ret == &rbtree->rbt_nil) { \ - ret = NULL; \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##next(a_rbt_type *rbtree, a_type *node) { \ - a_type *ret; \ - if (rbtn_right_get(a_type, a_field, node) != &rbtree->rbt_nil) { \ - rbtn_first(a_type, a_field, rbtree, rbtn_right_get(a_type, \ - a_field, node), ret); \ - } else { \ - a_type *tnode = rbtree->rbt_root; \ - assert(tnode != &rbtree->rbt_nil); \ - ret = &rbtree->rbt_nil; \ - while (true) { \ - int cmp = (a_cmp)(node, tnode); \ - if (cmp < 0) { \ - ret = tnode; \ - tnode = rbtn_left_get(a_type, a_field, tnode); \ - } else if (cmp > 0) { \ - tnode = rbtn_right_get(a_type, a_field, tnode); \ - } else { \ - break; \ - } \ - assert(tnode != &rbtree->rbt_nil); \ - } \ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = (NULL); \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##prev(a_rbt_type *rbtree, a_type *node) { \ - a_type *ret; \ - if (rbtn_left_get(a_type, a_field, node) != &rbtree->rbt_nil) { \ - rbtn_last(a_type, a_field, rbtree, rbtn_left_get(a_type, \ - a_field, node), ret); \ - } else { \ - a_type *tnode = rbtree->rbt_root; \ - assert(tnode != &rbtree->rbt_nil); \ - ret = &rbtree->rbt_nil; \ - while (true) { \ - int cmp = (a_cmp)(node, tnode); \ - if (cmp < 0) { \ - tnode = rbtn_left_get(a_type, a_field, tnode); \ - } else if (cmp > 0) { \ - ret = tnode; \ - tnode = rbtn_right_get(a_type, a_field, tnode); \ - } else { \ - break; \ - } \ - assert(tnode != &rbtree->rbt_nil); \ - } \ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = (NULL); \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##search(a_rbt_type *rbtree, a_type *key) { \ - a_type *ret; \ - int cmp; \ - ret = rbtree->rbt_root; \ - while (ret != &rbtree->rbt_nil \ - && (cmp = (a_cmp)(key, ret)) != 0) { \ - if (cmp < 0) { \ - ret = rbtn_left_get(a_type, a_field, ret); \ - } else { \ - ret = rbtn_right_get(a_type, a_field, ret); \ - } \ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = (NULL); \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##nsearch(a_rbt_type *rbtree, a_type *key) { \ - a_type *ret; \ - a_type *tnode = rbtree->rbt_root; \ - ret = &rbtree->rbt_nil; \ - while (tnode != &rbtree->rbt_nil) { \ - int cmp = (a_cmp)(key, tnode); \ - if (cmp < 0) { \ - ret = tnode; \ - tnode = rbtn_left_get(a_type, a_field, tnode); \ - } else if (cmp > 0) { \ - tnode = rbtn_right_get(a_type, a_field, tnode); \ - } else { \ - ret = tnode; \ - break; \ - } \ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = (NULL); \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##psearch(a_rbt_type *rbtree, a_type *key) { \ - a_type *ret; \ - a_type *tnode = rbtree->rbt_root; \ - ret = &rbtree->rbt_nil; \ - while (tnode != &rbtree->rbt_nil) { \ - int cmp = (a_cmp)(key, tnode); \ - if (cmp < 0) { \ - tnode = rbtn_left_get(a_type, a_field, tnode); \ - } else if (cmp > 0) { \ - ret = tnode; \ - tnode = rbtn_right_get(a_type, a_field, tnode); \ - } else { \ - ret = tnode; \ - break; \ - } \ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = (NULL); \ - } \ - return (ret); \ -} \ -a_attr void \ -a_prefix##insert(a_rbt_type *rbtree, a_type *node) { \ - struct { \ - a_type *node; \ - int cmp; \ - } path[sizeof(void *) << 4], *pathp; \ - rbt_node_new(a_type, a_field, rbtree, node); \ - /* Wind. */ \ - path->node = rbtree->rbt_root; \ - for (pathp = path; pathp->node != &rbtree->rbt_nil; pathp++) { \ - int cmp = pathp->cmp = a_cmp(node, pathp->node); \ - assert(cmp != 0); \ - if (cmp < 0) { \ - pathp[1].node = rbtn_left_get(a_type, a_field, \ - pathp->node); \ - } else { \ - pathp[1].node = rbtn_right_get(a_type, a_field, \ - pathp->node); \ - } \ - } \ - pathp->node = node; \ - /* Unwind. */ \ - for (pathp--; (uintptr_t)pathp >= (uintptr_t)path; pathp--) { \ - a_type *cnode = pathp->node; \ - if (pathp->cmp < 0) { \ - a_type *left = pathp[1].node; \ - rbtn_left_set(a_type, a_field, cnode, left); \ - if (rbtn_red_get(a_type, a_field, left)) { \ - a_type *leftleft = rbtn_left_get(a_type, a_field, left);\ - if (rbtn_red_get(a_type, a_field, leftleft)) { \ - /* Fix up 4-node. */ \ - a_type *tnode; \ - rbtn_black_set(a_type, a_field, leftleft); \ - rbtn_rotate_right(a_type, a_field, cnode, tnode); \ - cnode = tnode; \ - } \ - } else { \ - return; \ - } \ - } else { \ - a_type *right = pathp[1].node; \ - rbtn_right_set(a_type, a_field, cnode, right); \ - if (rbtn_red_get(a_type, a_field, right)) { \ - a_type *left = rbtn_left_get(a_type, a_field, cnode); \ - if (rbtn_red_get(a_type, a_field, left)) { \ - /* Split 4-node. */ \ - rbtn_black_set(a_type, a_field, left); \ - rbtn_black_set(a_type, a_field, right); \ - rbtn_red_set(a_type, a_field, cnode); \ - } else { \ - /* Lean left. */ \ - a_type *tnode; \ - bool tred = rbtn_red_get(a_type, a_field, cnode); \ - rbtn_rotate_left(a_type, a_field, cnode, tnode); \ - rbtn_color_set(a_type, a_field, tnode, tred); \ - rbtn_red_set(a_type, a_field, cnode); \ - cnode = tnode; \ - } \ - } else { \ - return; \ - } \ - } \ - pathp->node = cnode; \ - } \ - /* Set root, and make it black. */ \ - rbtree->rbt_root = path->node; \ - rbtn_black_set(a_type, a_field, rbtree->rbt_root); \ -} \ -a_attr void \ -a_prefix##remove(a_rbt_type *rbtree, a_type *node) { \ - struct { \ - a_type *node; \ - int cmp; \ - } *pathp, *nodep, path[sizeof(void *) << 4]; \ - /* Wind. */ \ - nodep = NULL; /* Silence compiler warning. */ \ - path->node = rbtree->rbt_root; \ - for (pathp = path; pathp->node != &rbtree->rbt_nil; pathp++) { \ - int cmp = pathp->cmp = a_cmp(node, pathp->node); \ - if (cmp < 0) { \ - pathp[1].node = rbtn_left_get(a_type, a_field, \ - pathp->node); \ - } else { \ - pathp[1].node = rbtn_right_get(a_type, a_field, \ - pathp->node); \ - if (cmp == 0) { \ - /* Find node's successor, in preparation for swap. */ \ - pathp->cmp = 1; \ - nodep = pathp; \ - for (pathp++; pathp->node != &rbtree->rbt_nil; \ - pathp++) { \ - pathp->cmp = -1; \ - pathp[1].node = rbtn_left_get(a_type, a_field, \ - pathp->node); \ - } \ - break; \ - } \ - } \ - } \ - assert(nodep->node == node); \ - pathp--; \ - if (pathp->node != node) { \ - /* Swap node with its successor. */ \ - bool tred = rbtn_red_get(a_type, a_field, pathp->node); \ - rbtn_color_set(a_type, a_field, pathp->node, \ - rbtn_red_get(a_type, a_field, node)); \ - rbtn_left_set(a_type, a_field, pathp->node, \ - rbtn_left_get(a_type, a_field, node)); \ - /* If node's successor is its right child, the following code */\ - /* will do the wrong thing for the right child pointer. */\ - /* However, it doesn't matter, because the pointer will be */\ - /* properly set when the successor is pruned. */\ - rbtn_right_set(a_type, a_field, pathp->node, \ - rbtn_right_get(a_type, a_field, node)); \ - rbtn_color_set(a_type, a_field, node, tred); \ - /* The pruned leaf node's child pointers are never accessed */\ - /* again, so don't bother setting them to nil. */\ - nodep->node = pathp->node; \ - pathp->node = node; \ - if (nodep == path) { \ - rbtree->rbt_root = nodep->node; \ - } else { \ - if (nodep[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, nodep[-1].node, \ - nodep->node); \ - } else { \ - rbtn_right_set(a_type, a_field, nodep[-1].node, \ - nodep->node); \ - } \ - } \ - } else { \ - a_type *left = rbtn_left_get(a_type, a_field, node); \ - if (left != &rbtree->rbt_nil) { \ - /* node has no successor, but it has a left child. */\ - /* Splice node out, without losing the left child. */\ - assert(rbtn_red_get(a_type, a_field, node) == false); \ - assert(rbtn_red_get(a_type, a_field, left)); \ - rbtn_black_set(a_type, a_field, left); \ - if (pathp == path) { \ - rbtree->rbt_root = left; \ - } else { \ - if (pathp[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, pathp[-1].node, \ - left); \ - } else { \ - rbtn_right_set(a_type, a_field, pathp[-1].node, \ - left); \ - } \ - } \ - return; \ - } else if (pathp == path) { \ - /* The tree only contained one node. */ \ - rbtree->rbt_root = &rbtree->rbt_nil; \ - return; \ - } \ - } \ - if (rbtn_red_get(a_type, a_field, pathp->node)) { \ - /* Prune red node, which requires no fixup. */ \ - assert(pathp[-1].cmp < 0); \ - rbtn_left_set(a_type, a_field, pathp[-1].node, \ - &rbtree->rbt_nil); \ - return; \ - } \ - /* The node to be pruned is black, so unwind until balance is */\ - /* restored. */\ - pathp->node = &rbtree->rbt_nil; \ - for (pathp--; (uintptr_t)pathp >= (uintptr_t)path; pathp--) { \ - assert(pathp->cmp != 0); \ - if (pathp->cmp < 0) { \ - rbtn_left_set(a_type, a_field, pathp->node, \ - pathp[1].node); \ - assert(rbtn_red_get(a_type, a_field, pathp[1].node) \ - == false); \ - if (rbtn_red_get(a_type, a_field, pathp->node)) { \ - a_type *right = rbtn_right_get(a_type, a_field, \ - pathp->node); \ - a_type *rightleft = rbtn_left_get(a_type, a_field, \ - right); \ - a_type *tnode; \ - if (rbtn_red_get(a_type, a_field, rightleft)) { \ - /* In the following diagrams, ||, //, and \\ */\ - /* indicate the path to the removed node. */\ - /* */\ - /* || */\ - /* pathp(r) */\ - /* // \ */\ - /* (b) (b) */\ - /* / */\ - /* (r) */\ - /* */\ - rbtn_black_set(a_type, a_field, pathp->node); \ - rbtn_rotate_right(a_type, a_field, right, tnode); \ - rbtn_right_set(a_type, a_field, pathp->node, tnode);\ - rbtn_rotate_left(a_type, a_field, pathp->node, \ - tnode); \ - } else { \ - /* || */\ - /* pathp(r) */\ - /* // \ */\ - /* (b) (b) */\ - /* / */\ - /* (b) */\ - /* */\ - rbtn_rotate_left(a_type, a_field, pathp->node, \ - tnode); \ - } \ - /* Balance restored, but rotation modified subtree */\ - /* root. */\ - assert((uintptr_t)pathp > (uintptr_t)path); \ - if (pathp[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, pathp[-1].node, \ - tnode); \ - } else { \ - rbtn_right_set(a_type, a_field, pathp[-1].node, \ - tnode); \ - } \ - return; \ - } else { \ - a_type *right = rbtn_right_get(a_type, a_field, \ - pathp->node); \ - a_type *rightleft = rbtn_left_get(a_type, a_field, \ - right); \ - if (rbtn_red_get(a_type, a_field, rightleft)) { \ - /* || */\ - /* pathp(b) */\ - /* // \ */\ - /* (b) (b) */\ - /* / */\ - /* (r) */\ - a_type *tnode; \ - rbtn_black_set(a_type, a_field, rightleft); \ - rbtn_rotate_right(a_type, a_field, right, tnode); \ - rbtn_right_set(a_type, a_field, pathp->node, tnode);\ - rbtn_rotate_left(a_type, a_field, pathp->node, \ - tnode); \ - /* Balance restored, but rotation modified */\ - /* subree root, which may actually be the tree */\ - /* root. */\ - if (pathp == path) { \ - /* Set root. */ \ - rbtree->rbt_root = tnode; \ - } else { \ - if (pathp[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, \ - pathp[-1].node, tnode); \ - } else { \ - rbtn_right_set(a_type, a_field, \ - pathp[-1].node, tnode); \ - } \ - } \ - return; \ - } else { \ - /* || */\ - /* pathp(b) */\ - /* // \ */\ - /* (b) (b) */\ - /* / */\ - /* (b) */\ - a_type *tnode; \ - rbtn_red_set(a_type, a_field, pathp->node); \ - rbtn_rotate_left(a_type, a_field, pathp->node, \ - tnode); \ - pathp->node = tnode; \ - } \ - } \ - } else { \ - a_type *left; \ - rbtn_right_set(a_type, a_field, pathp->node, \ - pathp[1].node); \ - left = rbtn_left_get(a_type, a_field, pathp->node); \ - if (rbtn_red_get(a_type, a_field, left)) { \ - a_type *tnode; \ - a_type *leftright = rbtn_right_get(a_type, a_field, \ - left); \ - a_type *leftrightleft = rbtn_left_get(a_type, a_field, \ - leftright); \ - if (rbtn_red_get(a_type, a_field, leftrightleft)) { \ - /* || */\ - /* pathp(b) */\ - /* / \\ */\ - /* (r) (b) */\ - /* \ */\ - /* (b) */\ - /* / */\ - /* (r) */\ - a_type *unode; \ - rbtn_black_set(a_type, a_field, leftrightleft); \ - rbtn_rotate_right(a_type, a_field, pathp->node, \ - unode); \ - rbtn_rotate_right(a_type, a_field, pathp->node, \ - tnode); \ - rbtn_right_set(a_type, a_field, unode, tnode); \ - rbtn_rotate_left(a_type, a_field, unode, tnode); \ - } else { \ - /* || */\ - /* pathp(b) */\ - /* / \\ */\ - /* (r) (b) */\ - /* \ */\ - /* (b) */\ - /* / */\ - /* (b) */\ - assert(leftright != &rbtree->rbt_nil); \ - rbtn_red_set(a_type, a_field, leftright); \ - rbtn_rotate_right(a_type, a_field, pathp->node, \ - tnode); \ - rbtn_black_set(a_type, a_field, tnode); \ - } \ - /* Balance restored, but rotation modified subtree */\ - /* root, which may actually be the tree root. */\ - if (pathp == path) { \ - /* Set root. */ \ - rbtree->rbt_root = tnode; \ - } else { \ - if (pathp[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, pathp[-1].node, \ - tnode); \ - } else { \ - rbtn_right_set(a_type, a_field, pathp[-1].node, \ - tnode); \ - } \ - } \ - return; \ - } else if (rbtn_red_get(a_type, a_field, pathp->node)) { \ - a_type *leftleft = rbtn_left_get(a_type, a_field, left);\ - if (rbtn_red_get(a_type, a_field, leftleft)) { \ - /* || */\ - /* pathp(r) */\ - /* / \\ */\ - /* (b) (b) */\ - /* / */\ - /* (r) */\ - a_type *tnode; \ - rbtn_black_set(a_type, a_field, pathp->node); \ - rbtn_red_set(a_type, a_field, left); \ - rbtn_black_set(a_type, a_field, leftleft); \ - rbtn_rotate_right(a_type, a_field, pathp->node, \ - tnode); \ - /* Balance restored, but rotation modified */\ - /* subtree root. */\ - assert((uintptr_t)pathp > (uintptr_t)path); \ - if (pathp[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, pathp[-1].node, \ - tnode); \ - } else { \ - rbtn_right_set(a_type, a_field, pathp[-1].node, \ - tnode); \ - } \ - return; \ - } else { \ - /* || */\ - /* pathp(r) */\ - /* / \\ */\ - /* (b) (b) */\ - /* / */\ - /* (b) */\ - rbtn_red_set(a_type, a_field, left); \ - rbtn_black_set(a_type, a_field, pathp->node); \ - /* Balance restored. */ \ - return; \ - } \ - } else { \ - a_type *leftleft = rbtn_left_get(a_type, a_field, left);\ - if (rbtn_red_get(a_type, a_field, leftleft)) { \ - /* || */\ - /* pathp(b) */\ - /* / \\ */\ - /* (b) (b) */\ - /* / */\ - /* (r) */\ - a_type *tnode; \ - rbtn_black_set(a_type, a_field, leftleft); \ - rbtn_rotate_right(a_type, a_field, pathp->node, \ - tnode); \ - /* Balance restored, but rotation modified */\ - /* subtree root, which may actually be the tree */\ - /* root. */\ - if (pathp == path) { \ - /* Set root. */ \ - rbtree->rbt_root = tnode; \ - } else { \ - if (pathp[-1].cmp < 0) { \ - rbtn_left_set(a_type, a_field, \ - pathp[-1].node, tnode); \ - } else { \ - rbtn_right_set(a_type, a_field, \ - pathp[-1].node, tnode); \ - } \ - } \ - return; \ - } else { \ - /* || */\ - /* pathp(b) */\ - /* / \\ */\ - /* (b) (b) */\ - /* / */\ - /* (b) */\ - rbtn_red_set(a_type, a_field, left); \ - } \ - } \ - } \ - } \ - /* Set root. */ \ - rbtree->rbt_root = path->node; \ - assert(rbtn_red_get(a_type, a_field, rbtree->rbt_root) == false); \ -} \ -a_attr a_type * \ -a_prefix##iter_recurse(a_rbt_type *rbtree, a_type *node, \ - a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ - if (node == &rbtree->rbt_nil) { \ - return (&rbtree->rbt_nil); \ - } else { \ - a_type *ret; \ - if ((ret = a_prefix##iter_recurse(rbtree, rbtn_left_get(a_type, \ - a_field, node), cb, arg)) != &rbtree->rbt_nil \ - || (ret = cb(rbtree, node, arg)) != NULL) { \ - return (ret); \ - } \ - return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \ - a_field, node), cb, arg)); \ - } \ -} \ -a_attr a_type * \ -a_prefix##iter_start(a_rbt_type *rbtree, a_type *start, a_type *node, \ - a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ - int cmp = a_cmp(start, node); \ - if (cmp < 0) { \ - a_type *ret; \ - if ((ret = a_prefix##iter_start(rbtree, start, \ - rbtn_left_get(a_type, a_field, node), cb, arg)) != \ - &rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \ - return (ret); \ - } \ - return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \ - a_field, node), cb, arg)); \ - } else if (cmp > 0) { \ - return (a_prefix##iter_start(rbtree, start, \ - rbtn_right_get(a_type, a_field, node), cb, arg)); \ - } else { \ - a_type *ret; \ - if ((ret = cb(rbtree, node, arg)) != NULL) { \ - return (ret); \ - } \ - return (a_prefix##iter_recurse(rbtree, rbtn_right_get(a_type, \ - a_field, node), cb, arg)); \ - } \ -} \ -a_attr a_type * \ -a_prefix##iter(a_rbt_type *rbtree, a_type *start, a_type *(*cb)( \ - a_rbt_type *, a_type *, void *), void *arg) { \ - a_type *ret; \ - if (start != NULL) { \ - ret = a_prefix##iter_start(rbtree, start, rbtree->rbt_root, \ - cb, arg); \ - } else { \ - ret = a_prefix##iter_recurse(rbtree, rbtree->rbt_root, cb, arg);\ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = NULL; \ - } \ - return (ret); \ -} \ -a_attr a_type * \ -a_prefix##reverse_iter_recurse(a_rbt_type *rbtree, a_type *node, \ - a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ - if (node == &rbtree->rbt_nil) { \ - return (&rbtree->rbt_nil); \ - } else { \ - a_type *ret; \ - if ((ret = a_prefix##reverse_iter_recurse(rbtree, \ - rbtn_right_get(a_type, a_field, node), cb, arg)) != \ - &rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \ - return (ret); \ - } \ - return (a_prefix##reverse_iter_recurse(rbtree, \ - rbtn_left_get(a_type, a_field, node), cb, arg)); \ - } \ -} \ -a_attr a_type * \ -a_prefix##reverse_iter_start(a_rbt_type *rbtree, a_type *start, \ - a_type *node, a_type *(*cb)(a_rbt_type *, a_type *, void *), \ - void *arg) { \ - int cmp = a_cmp(start, node); \ - if (cmp > 0) { \ - a_type *ret; \ - if ((ret = a_prefix##reverse_iter_start(rbtree, start, \ - rbtn_right_get(a_type, a_field, node), cb, arg)) != \ - &rbtree->rbt_nil || (ret = cb(rbtree, node, arg)) != NULL) { \ - return (ret); \ - } \ - return (a_prefix##reverse_iter_recurse(rbtree, \ - rbtn_left_get(a_type, a_field, node), cb, arg)); \ - } else if (cmp < 0) { \ - return (a_prefix##reverse_iter_start(rbtree, start, \ - rbtn_left_get(a_type, a_field, node), cb, arg)); \ - } else { \ - a_type *ret; \ - if ((ret = cb(rbtree, node, arg)) != NULL) { \ - return (ret); \ - } \ - return (a_prefix##reverse_iter_recurse(rbtree, \ - rbtn_left_get(a_type, a_field, node), cb, arg)); \ - } \ -} \ -a_attr a_type * \ -a_prefix##reverse_iter(a_rbt_type *rbtree, a_type *start, \ - a_type *(*cb)(a_rbt_type *, a_type *, void *), void *arg) { \ - a_type *ret; \ - if (start != NULL) { \ - ret = a_prefix##reverse_iter_start(rbtree, start, \ - rbtree->rbt_root, cb, arg); \ - } else { \ - ret = a_prefix##reverse_iter_recurse(rbtree, rbtree->rbt_root, \ - cb, arg); \ - } \ - if (ret == &rbtree->rbt_nil) { \ - ret = NULL; \ - } \ - return (ret); \ -} - -#endif /* RB_H_ */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/rtree.h b/deps/jemalloc.orig/include/jemalloc/internal/rtree.h deleted file mode 100644 index 95d6355a5f4..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/rtree.h +++ /dev/null @@ -1,161 +0,0 @@ -/* - * This radix tree implementation is tailored to the singular purpose of - * tracking which chunks are currently owned by jemalloc. This functionality - * is mandatory for OS X, where jemalloc must be able to respond to object - * ownership queries. - * - ******************************************************************************* - */ -#ifdef JEMALLOC_H_TYPES - -typedef struct rtree_s rtree_t; - -/* - * Size of each radix tree node (must be a power of 2). This impacts tree - * depth. - */ -#if (LG_SIZEOF_PTR == 2) -# define RTREE_NODESIZE (1U << 14) -#else -# define RTREE_NODESIZE CACHELINE -#endif - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -struct rtree_s { - malloc_mutex_t mutex; - void **root; - unsigned height; - unsigned level2bits[1]; /* Dynamically sized. */ -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -rtree_t *rtree_new(unsigned bits); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -#ifndef JEMALLOC_DEBUG -void *rtree_get_locked(rtree_t *rtree, uintptr_t key); -#endif -void *rtree_get(rtree_t *rtree, uintptr_t key); -bool rtree_set(rtree_t *rtree, uintptr_t key, void *val); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) -#define RTREE_GET_GENERATE(f) \ -/* The least significant bits of the key are ignored. */ \ -JEMALLOC_INLINE void * \ -f(rtree_t *rtree, uintptr_t key) \ -{ \ - void *ret; \ - uintptr_t subkey; \ - unsigned i, lshift, height, bits; \ - void **node, **child; \ - \ - RTREE_LOCK(&rtree->mutex); \ - for (i = lshift = 0, height = rtree->height, node = rtree->root;\ - i < height - 1; \ - i++, lshift += bits, node = child) { \ - bits = rtree->level2bits[i]; \ - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ - 3)) - bits); \ - child = (void**)node[subkey]; \ - if (child == NULL) { \ - RTREE_UNLOCK(&rtree->mutex); \ - return (NULL); \ - } \ - } \ - \ - /* \ - * node is a leaf, so it contains values rather than node \ - * pointers. \ - */ \ - bits = rtree->level2bits[i]; \ - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - \ - bits); \ - ret = node[subkey]; \ - RTREE_UNLOCK(&rtree->mutex); \ - \ - RTREE_GET_VALIDATE \ - return (ret); \ -} - -#ifdef JEMALLOC_DEBUG -# define RTREE_LOCK(l) malloc_mutex_lock(l) -# define RTREE_UNLOCK(l) malloc_mutex_unlock(l) -# define RTREE_GET_VALIDATE -RTREE_GET_GENERATE(rtree_get_locked) -# undef RTREE_LOCK -# undef RTREE_UNLOCK -# undef RTREE_GET_VALIDATE -#endif - -#define RTREE_LOCK(l) -#define RTREE_UNLOCK(l) -#ifdef JEMALLOC_DEBUG - /* - * Suppose that it were possible for a jemalloc-allocated chunk to be - * munmap()ped, followed by a different allocator in another thread re-using - * overlapping virtual memory, all without invalidating the cached rtree - * value. The result would be a false positive (the rtree would claim that - * jemalloc owns memory that it had actually discarded). This scenario - * seems impossible, but the following assertion is a prudent sanity check. - */ -# define RTREE_GET_VALIDATE \ - assert(rtree_get_locked(rtree, key) == ret); -#else -# define RTREE_GET_VALIDATE -#endif -RTREE_GET_GENERATE(rtree_get) -#undef RTREE_LOCK -#undef RTREE_UNLOCK -#undef RTREE_GET_VALIDATE - -JEMALLOC_INLINE bool -rtree_set(rtree_t *rtree, uintptr_t key, void *val) -{ - uintptr_t subkey; - unsigned i, lshift, height, bits; - void **node, **child; - - malloc_mutex_lock(&rtree->mutex); - for (i = lshift = 0, height = rtree->height, node = rtree->root; - i < height - 1; - i++, lshift += bits, node = child) { - bits = rtree->level2bits[i]; - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - - bits); - child = (void**)node[subkey]; - if (child == NULL) { - child = (void**)base_alloc(sizeof(void *) << - rtree->level2bits[i+1]); - if (child == NULL) { - malloc_mutex_unlock(&rtree->mutex); - return (true); - } - memset(child, 0, sizeof(void *) << - rtree->level2bits[i+1]); - node[subkey] = child; - } - } - - /* node is a leaf, so it contains values rather than node pointers. */ - bits = rtree->level2bits[i]; - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits); - node[subkey] = val; - malloc_mutex_unlock(&rtree->mutex); - - return (false); -} -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/stats.h b/deps/jemalloc.orig/include/jemalloc/internal/stats.h deleted file mode 100644 index 2a9b31d9ffc..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/stats.h +++ /dev/null @@ -1,207 +0,0 @@ -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#define UMAX2S_BUFSIZE 65 - -#ifdef JEMALLOC_STATS -typedef struct tcache_bin_stats_s tcache_bin_stats_t; -typedef struct malloc_bin_stats_s malloc_bin_stats_t; -typedef struct malloc_large_stats_s malloc_large_stats_t; -typedef struct arena_stats_s arena_stats_t; -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) -typedef struct chunk_stats_s chunk_stats_t; -#endif - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#ifdef JEMALLOC_STATS - -#ifdef JEMALLOC_TCACHE -struct tcache_bin_stats_s { - /* - * Number of allocation requests that corresponded to the size of this - * bin. - */ - uint64_t nrequests; -}; -#endif - -struct malloc_bin_stats_s { - /* - * Current number of bytes allocated, including objects currently - * cached by tcache. - */ - size_t allocated; - - /* - * Total number of allocation/deallocation requests served directly by - * the bin. Note that tcache may allocate an object, then recycle it - * many times, resulting many increments to nrequests, but only one - * each to nmalloc and ndalloc. - */ - uint64_t nmalloc; - uint64_t ndalloc; - - /* - * Number of allocation requests that correspond to the size of this - * bin. This includes requests served by tcache, though tcache only - * periodically merges into this counter. - */ - uint64_t nrequests; - -#ifdef JEMALLOC_TCACHE - /* Number of tcache fills from this bin. */ - uint64_t nfills; - - /* Number of tcache flushes to this bin. */ - uint64_t nflushes; -#endif - - /* Total number of runs created for this bin's size class. */ - uint64_t nruns; - - /* - * Total number of runs reused by extracting them from the runs tree for - * this bin's size class. - */ - uint64_t reruns; - - /* High-water mark for this bin. */ - size_t highruns; - - /* Current number of runs in this bin. */ - size_t curruns; -}; - -struct malloc_large_stats_s { - /* - * Total number of allocation/deallocation requests served directly by - * the arena. Note that tcache may allocate an object, then recycle it - * many times, resulting many increments to nrequests, but only one - * each to nmalloc and ndalloc. - */ - uint64_t nmalloc; - uint64_t ndalloc; - - /* - * Number of allocation requests that correspond to this size class. - * This includes requests served by tcache, though tcache only - * periodically merges into this counter. - */ - uint64_t nrequests; - - /* High-water mark for this size class. */ - size_t highruns; - - /* Current number of runs of this size class. */ - size_t curruns; -}; - -struct arena_stats_s { - /* Number of bytes currently mapped. */ - size_t mapped; - - /* - * Total number of purge sweeps, total number of madvise calls made, - * and total pages purged in order to keep dirty unused memory under - * control. - */ - uint64_t npurge; - uint64_t nmadvise; - uint64_t purged; - - /* Per-size-category statistics. */ - size_t allocated_large; - uint64_t nmalloc_large; - uint64_t ndalloc_large; - uint64_t nrequests_large; - - /* - * One element for each possible size class, including sizes that - * overlap with bin size classes. This is necessary because ipalloc() - * sometimes has to use such large objects in order to assure proper - * alignment. - */ - malloc_large_stats_t *lstats; -}; -#endif /* JEMALLOC_STATS */ - -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) -struct chunk_stats_s { -# ifdef JEMALLOC_STATS - /* Number of chunks that were allocated. */ - uint64_t nchunks; -# endif - - /* High-water mark for number of chunks allocated. */ - size_t highchunks; - - /* - * Current number of chunks allocated. This value isn't maintained for - * any other purpose, so keep track of it in order to be able to set - * highchunks. - */ - size_t curchunks; -}; -#endif /* JEMALLOC_STATS */ - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern bool opt_stats_print; - -#ifdef JEMALLOC_STATS -extern size_t stats_cactive; -#endif - -char *u2s(uint64_t x, unsigned base, char *s); -#ifdef JEMALLOC_STATS -void malloc_cprintf(void (*write)(void *, const char *), void *cbopaque, - const char *format, ...) JEMALLOC_ATTR(format(printf, 3, 4)); -void malloc_printf(const char *format, ...) - JEMALLOC_ATTR(format(printf, 1, 2)); -#endif -void stats_print(void (*write)(void *, const char *), void *cbopaque, - const char *opts); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES -#ifdef JEMALLOC_STATS - -#ifndef JEMALLOC_ENABLE_INLINE -size_t stats_cactive_get(void); -void stats_cactive_add(size_t size); -void stats_cactive_sub(size_t size); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_STATS_C_)) -JEMALLOC_INLINE size_t -stats_cactive_get(void) -{ - - return (atomic_read_z(&stats_cactive)); -} - -JEMALLOC_INLINE void -stats_cactive_add(size_t size) -{ - - atomic_add_z(&stats_cactive, size); -} - -JEMALLOC_INLINE void -stats_cactive_sub(size_t size) -{ - - atomic_sub_z(&stats_cactive, size); -} -#endif - -#endif /* JEMALLOC_STATS */ -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/tcache.h b/deps/jemalloc.orig/include/jemalloc/internal/tcache.h deleted file mode 100644 index da3c68c5770..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/tcache.h +++ /dev/null @@ -1,431 +0,0 @@ -#ifdef JEMALLOC_TCACHE -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -typedef struct tcache_bin_info_s tcache_bin_info_t; -typedef struct tcache_bin_s tcache_bin_t; -typedef struct tcache_s tcache_t; - -/* - * Absolute maximum number of cache slots for each small bin in the thread - * cache. This is an additional constraint beyond that imposed as: twice the - * number of regions per run for this size class. - * - * This constant must be an even number. - */ -#define TCACHE_NSLOTS_SMALL_MAX 200 - -/* Number of cache slots for large size classes. */ -#define TCACHE_NSLOTS_LARGE 20 - -/* (1U << opt_lg_tcache_max) is used to compute tcache_maxclass. */ -#define LG_TCACHE_MAXCLASS_DEFAULT 15 - -/* - * (1U << opt_lg_tcache_gc_sweep) is the approximate number of allocation - * events between full GC sweeps (-1: disabled). Integer rounding may cause - * the actual number to be slightly higher, since GC is performed - * incrementally. - */ -#define LG_TCACHE_GC_SWEEP_DEFAULT 13 - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -/* - * Read-only information associated with each element of tcache_t's tbins array - * is stored separately, mainly to reduce memory usage. - */ -struct tcache_bin_info_s { - unsigned ncached_max; /* Upper limit on ncached. */ -}; - -struct tcache_bin_s { -# ifdef JEMALLOC_STATS - tcache_bin_stats_t tstats; -# endif - int low_water; /* Min # cached since last GC. */ - unsigned lg_fill_div; /* Fill (ncached_max >> lg_fill_div). */ - unsigned ncached; /* # of cached objects. */ - void **avail; /* Stack of available objects. */ -}; - -struct tcache_s { -# ifdef JEMALLOC_STATS - ql_elm(tcache_t) link; /* Used for aggregating stats. */ -# endif -# ifdef JEMALLOC_PROF - uint64_t prof_accumbytes;/* Cleared after arena_prof_accum() */ -# endif - arena_t *arena; /* This thread's arena. */ - unsigned ev_cnt; /* Event count since incremental GC. */ - unsigned next_gc_bin; /* Next bin to GC. */ - tcache_bin_t tbins[1]; /* Dynamically sized. */ - /* - * The pointer stacks associated with tbins follow as a contiguous - * array. During tcache initialization, the avail pointer in each - * element of tbins is initialized to point to the proper offset within - * this array. - */ -}; - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -extern bool opt_tcache; -extern ssize_t opt_lg_tcache_max; -extern ssize_t opt_lg_tcache_gc_sweep; - -extern tcache_bin_info_t *tcache_bin_info; - -/* Map of thread-specific caches. */ -#ifndef NO_TLS -extern __thread tcache_t *tcache_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -# define TCACHE_GET() tcache_tls -# define TCACHE_SET(v) do { \ - tcache_tls = (tcache_t *)(v); \ - pthread_setspecific(tcache_tsd, (void *)(v)); \ -} while (0) -#else -# define TCACHE_GET() ((tcache_t *)pthread_getspecific(tcache_tsd)) -# define TCACHE_SET(v) do { \ - pthread_setspecific(tcache_tsd, (void *)(v)); \ -} while (0) -#endif -extern pthread_key_t tcache_tsd; - -/* - * Number of tcache bins. There are nbins small-object bins, plus 0 or more - * large-object bins. - */ -extern size_t nhbins; - -/* Maximum cached size class. */ -extern size_t tcache_maxclass; - -/* Number of tcache allocation/deallocation events between incremental GCs. */ -extern unsigned tcache_gc_incr; - -void tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ); -void tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ); -tcache_t *tcache_create(arena_t *arena); -void *tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, - size_t binind); -void tcache_destroy(tcache_t *tcache); -#ifdef JEMALLOC_STATS -void tcache_stats_merge(tcache_t *tcache, arena_t *arena); -#endif -bool tcache_boot(void); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#ifndef JEMALLOC_ENABLE_INLINE -void tcache_event(tcache_t *tcache); -tcache_t *tcache_get(void); -void *tcache_alloc_easy(tcache_bin_t *tbin); -void *tcache_alloc_small(tcache_t *tcache, size_t size, bool zero); -void *tcache_alloc_large(tcache_t *tcache, size_t size, bool zero); -void tcache_dalloc_small(tcache_t *tcache, void *ptr); -void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); -#endif - -#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) -JEMALLOC_INLINE tcache_t * -tcache_get(void) -{ - tcache_t *tcache; - - if ((isthreaded & opt_tcache) == false) - return (NULL); - - tcache = TCACHE_GET(); - if ((uintptr_t)tcache <= (uintptr_t)2) { - if (tcache == NULL) { - tcache = tcache_create(choose_arena()); - if (tcache == NULL) - return (NULL); - } else { - if (tcache == (void *)(uintptr_t)1) { - /* - * Make a note that an allocator function was - * called after the tcache_thread_cleanup() was - * called. - */ - TCACHE_SET((uintptr_t)2); - } - return (NULL); - } - } - - return (tcache); -} - -JEMALLOC_INLINE void -tcache_event(tcache_t *tcache) -{ - - if (tcache_gc_incr == 0) - return; - - tcache->ev_cnt++; - assert(tcache->ev_cnt <= tcache_gc_incr); - if (tcache->ev_cnt == tcache_gc_incr) { - size_t binind = tcache->next_gc_bin; - tcache_bin_t *tbin = &tcache->tbins[binind]; - tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; - - if (tbin->low_water > 0) { - /* - * Flush (ceiling) 3/4 of the objects below the low - * water mark. - */ - if (binind < nbins) { - tcache_bin_flush_small(tbin, binind, - tbin->ncached - tbin->low_water + - (tbin->low_water >> 2) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } else { - tcache_bin_flush_large(tbin, binind, - tbin->ncached - tbin->low_water + - (tbin->low_water >> 2) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } - /* - * Reduce fill count by 2X. Limit lg_fill_div such that - * the fill count is always at least 1. - */ - if ((tbin_info->ncached_max >> (tbin->lg_fill_div+1)) - >= 1) - tbin->lg_fill_div++; - } else if (tbin->low_water < 0) { - /* - * Increase fill count by 2X. Make sure lg_fill_div - * stays greater than 0. - */ - if (tbin->lg_fill_div > 1) - tbin->lg_fill_div--; - } - tbin->low_water = tbin->ncached; - - tcache->next_gc_bin++; - if (tcache->next_gc_bin == nhbins) - tcache->next_gc_bin = 0; - tcache->ev_cnt = 0; - } -} - -JEMALLOC_INLINE void * -tcache_alloc_easy(tcache_bin_t *tbin) -{ - void *ret; - - if (tbin->ncached == 0) { - tbin->low_water = -1; - return (NULL); - } - tbin->ncached--; - if ((int)tbin->ncached < tbin->low_water) - tbin->low_water = tbin->ncached; - ret = tbin->avail[tbin->ncached]; - return (ret); -} - -JEMALLOC_INLINE void * -tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) -{ - void *ret; - size_t binind; - tcache_bin_t *tbin; - - binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); - tbin = &tcache->tbins[binind]; - ret = tcache_alloc_easy(tbin); - if (ret == NULL) { - ret = tcache_alloc_small_hard(tcache, tbin, binind); - if (ret == NULL) - return (NULL); - } - assert(arena_salloc(ret) == arena_bin_info[binind].reg_size); - - if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else - memset(ret, 0, size); - -#ifdef JEMALLOC_STATS - tbin->tstats.nrequests++; -#endif -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes += arena_bin_info[binind].reg_size; -#endif - tcache_event(tcache); - return (ret); -} - -JEMALLOC_INLINE void * -tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) -{ - void *ret; - size_t binind; - tcache_bin_t *tbin; - - size = PAGE_CEILING(size); - assert(size <= tcache_maxclass); - binind = nbins + (size >> PAGE_SHIFT) - 1; - assert(binind < nhbins); - tbin = &tcache->tbins[binind]; - ret = tcache_alloc_easy(tbin); - if (ret == NULL) { - /* - * Only allocate one large object at a time, because it's quite - * expensive to create one and not use it. - */ - ret = arena_malloc_large(tcache->arena, size, zero); - if (ret == NULL) - return (NULL); - } else { -#ifdef JEMALLOC_PROF - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); - size_t pageind = (((uintptr_t)ret - (uintptr_t)chunk) >> - PAGE_SHIFT); - chunk->map[pageind-map_bias].bits &= ~CHUNK_MAP_CLASS_MASK; -#endif - if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else - memset(ret, 0, size); - -#ifdef JEMALLOC_STATS - tbin->tstats.nrequests++; -#endif -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes += size; -#endif - } - - tcache_event(tcache); - return (ret); -} - -JEMALLOC_INLINE void -tcache_dalloc_small(tcache_t *tcache, void *ptr) -{ - arena_t *arena; - arena_chunk_t *chunk; - arena_run_t *run; - arena_bin_t *bin; - tcache_bin_t *tbin; - tcache_bin_info_t *tbin_info; - size_t pageind, binind; - arena_chunk_map_t *mapelm; - - assert(arena_salloc(ptr) <= small_maxclass); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapelm = &chunk->map[pageind-map_bias]; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; - binind = ((uintptr_t)bin - (uintptr_t)&arena->bins) / - sizeof(arena_bin_t); - assert(binind < nbins); - -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, arena_bin_info[binind].reg_size); -#endif - - tbin = &tcache->tbins[binind]; - tbin_info = &tcache_bin_info[binind]; - if (tbin->ncached == tbin_info->ncached_max) { - tcache_bin_flush_small(tbin, binind, (tbin_info->ncached_max >> - 1) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } - assert(tbin->ncached < tbin_info->ncached_max); - tbin->avail[tbin->ncached] = ptr; - tbin->ncached++; - - tcache_event(tcache); -} - -JEMALLOC_INLINE void -tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) -{ - arena_t *arena; - arena_chunk_t *chunk; - size_t pageind, binind; - tcache_bin_t *tbin; - tcache_bin_info_t *tbin_info; - - assert((size & PAGE_MASK) == 0); - assert(arena_salloc(ptr) > small_maxclass); - assert(arena_salloc(ptr) <= tcache_maxclass); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - binind = nbins + (size >> PAGE_SHIFT) - 1; - -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, size); -#endif - - tbin = &tcache->tbins[binind]; - tbin_info = &tcache_bin_info[binind]; - if (tbin->ncached == tbin_info->ncached_max) { - tcache_bin_flush_large(tbin, binind, (tbin_info->ncached_max >> - 1) -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - } - assert(tbin->ncached < tbin_info->ncached_max); - tbin->avail[tbin->ncached] = ptr; - tbin->ncached++; - - tcache_event(tcache); -} -#endif - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ -#endif /* JEMALLOC_TCACHE */ diff --git a/deps/jemalloc.orig/include/jemalloc/internal/zone.h b/deps/jemalloc.orig/include/jemalloc/internal/zone.h deleted file mode 100644 index 859b529d443..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/internal/zone.h +++ /dev/null @@ -1,23 +0,0 @@ -#ifndef JEMALLOC_ZONE -# error "This source file is for zones on Darwin (OS X)." -#endif -/******************************************************************************/ -#ifdef JEMALLOC_H_TYPES - -#endif /* JEMALLOC_H_TYPES */ -/******************************************************************************/ -#ifdef JEMALLOC_H_STRUCTS - -#endif /* JEMALLOC_H_STRUCTS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_EXTERNS - -malloc_zone_t *create_zone(void); -void szone2ozone(malloc_zone_t *zone); - -#endif /* JEMALLOC_H_EXTERNS */ -/******************************************************************************/ -#ifdef JEMALLOC_H_INLINES - -#endif /* JEMALLOC_H_INLINES */ -/******************************************************************************/ diff --git a/deps/jemalloc.orig/include/jemalloc/jemalloc.h.in b/deps/jemalloc.orig/include/jemalloc/jemalloc.h.in deleted file mode 100644 index 580a5ec5d5f..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/jemalloc.h.in +++ /dev/null @@ -1,66 +0,0 @@ -#ifndef JEMALLOC_H_ -#define JEMALLOC_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#define JEMALLOC_VERSION "@jemalloc_version@" -#define JEMALLOC_VERSION_MAJOR @jemalloc_version_major@ -#define JEMALLOC_VERSION_MINOR @jemalloc_version_minor@ -#define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ -#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ -#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" - -#include "jemalloc_defs@install_suffix@.h" -#ifndef JEMALLOC_P -# define JEMALLOC_P(s) s -#endif - -#define ALLOCM_LG_ALIGN(la) (la) -#if LG_SIZEOF_PTR == 2 -#define ALLOCM_ALIGN(a) (ffs(a)-1) -#else -#define ALLOCM_ALIGN(a) ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) -#endif -#define ALLOCM_ZERO ((int)0x40) -#define ALLOCM_NO_MOVE ((int)0x80) - -#define ALLOCM_SUCCESS 0 -#define ALLOCM_ERR_OOM 1 -#define ALLOCM_ERR_NOT_MOVED 2 - -extern const char *JEMALLOC_P(malloc_conf); -extern void (*JEMALLOC_P(malloc_message))(void *, const char *); - -void *JEMALLOC_P(malloc)(size_t size) JEMALLOC_ATTR(malloc); -void *JEMALLOC_P(calloc)(size_t num, size_t size) JEMALLOC_ATTR(malloc); -int JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) - JEMALLOC_ATTR(nonnull(1)); -void *JEMALLOC_P(realloc)(void *ptr, size_t size); -void JEMALLOC_P(free)(void *ptr); - -size_t JEMALLOC_P(malloc_usable_size)(const void *ptr); -void JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), - void *cbopaque, const char *opts); -int JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, - void *newp, size_t newlen); -int JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, - size_t *miblenp); -int JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen); - -int JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) - JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, - size_t extra, int flags) JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) - JEMALLOC_ATTR(nonnull(1)); -int JEMALLOC_P(dallocm)(void *ptr, int flags) JEMALLOC_ATTR(nonnull(1)); - -#ifdef __cplusplus -}; -#endif -#endif /* JEMALLOC_H_ */ diff --git a/deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in b/deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in deleted file mode 100644 index 9ac7e1c2076..00000000000 --- a/deps/jemalloc.orig/include/jemalloc/jemalloc_defs.h.in +++ /dev/null @@ -1,167 +0,0 @@ -#ifndef JEMALLOC_DEFS_H_ -#define JEMALLOC_DEFS_H_ - -/* - * If JEMALLOC_PREFIX is defined, it will cause all public APIs to be prefixed. - * This makes it possible, with some care, to use multiple allocators - * simultaneously. - * - * In many cases it is more convenient to manually prefix allocator function - * calls than to let macros do it automatically, particularly when using - * multiple allocators simultaneously. Define JEMALLOC_MANGLE before - * #include'ing jemalloc.h in order to cause name mangling that corresponds to - * the API prefixing. - */ -#undef JEMALLOC_PREFIX -#undef JEMALLOC_CPREFIX -#if (defined(JEMALLOC_PREFIX) && defined(JEMALLOC_MANGLE)) -#undef JEMALLOC_P -#endif - -/* - * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. - * For shared libraries, symbol visibility mechanisms prevent these symbols - * from being exported, but for static libraries, naming collisions are a real - * possibility. - */ -#undef JEMALLOC_PRIVATE_NAMESPACE -#undef JEMALLOC_N - -/* - * Hyper-threaded CPUs may need a special instruction inside spin loops in - * order to yield to another virtual CPU. - */ -#undef CPU_SPINWAIT - -/* - * Defined if OSAtomic*() functions are available, as provided by Darwin, and - * documented in the atomic(3) manual page. - */ -#undef JEMALLOC_OSATOMIC - -/* - * Defined if OSSpin*() functions are available, as provided by Darwin, and - * documented in the spinlock(3) manual page. - */ -#undef JEMALLOC_OSSPIN - -/* Defined if __attribute__((...)) syntax is supported. */ -#undef JEMALLOC_HAVE_ATTR -#ifdef JEMALLOC_HAVE_ATTR -# define JEMALLOC_ATTR(s) __attribute__((s)) -#else -# define JEMALLOC_ATTR(s) -#endif - -/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */ -#undef JEMALLOC_CC_SILENCE - -/* - * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables - * inline functions. - */ -#undef JEMALLOC_DEBUG - -/* JEMALLOC_STATS enables statistics calculation. */ -#undef JEMALLOC_STATS - -/* JEMALLOC_PROF enables allocation profiling. */ -#undef JEMALLOC_PROF - -/* Use libunwind for profile backtracing if defined. */ -#undef JEMALLOC_PROF_LIBUNWIND - -/* Use libgcc for profile backtracing if defined. */ -#undef JEMALLOC_PROF_LIBGCC - -/* Use gcc intrinsics for profile backtracing if defined. */ -#undef JEMALLOC_PROF_GCC - -/* - * JEMALLOC_TINY enables support for tiny objects, which are smaller than one - * quantum. - */ -#undef JEMALLOC_TINY - -/* - * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects. - * This makes it possible to allocate/deallocate objects without any locking - * when the cache is in the steady state. - */ -#undef JEMALLOC_TCACHE - -/* - * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage - * segment (DSS). - */ -#undef JEMALLOC_DSS - -/* JEMALLOC_SWAP enables mmap()ed swap file support. */ -#undef JEMALLOC_SWAP - -/* Support memory filling (junk/zero). */ -#undef JEMALLOC_FILL - -/* Support optional abort() on OOM. */ -#undef JEMALLOC_XMALLOC - -/* Support SYSV semantics. */ -#undef JEMALLOC_SYSV - -/* Support lazy locking (avoid locking unless a second thread is launched). */ -#undef JEMALLOC_LAZY_LOCK - -/* Determine page size at run time if defined. */ -#undef DYNAMIC_PAGE_SHIFT - -/* One page is 2^STATIC_PAGE_SHIFT bytes. */ -#undef STATIC_PAGE_SHIFT - -/* TLS is used to map arenas and magazine caches to threads. */ -#undef NO_TLS - -/* - * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside - * within jemalloc-owned chunks before dereferencing them. - */ -#undef JEMALLOC_IVSALLOC - -/* - * Define overrides for non-standard allocator-related functions if they - * are present on the system. - */ -#undef JEMALLOC_OVERRIDE_MEMALIGN -#undef JEMALLOC_OVERRIDE_VALLOC - -/* - * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. - */ -#undef JEMALLOC_ZONE -#undef JEMALLOC_ZONE_VERSION - -/* If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). */ -#undef JEMALLOC_MREMAP_FIXED - -/* - * Methods for purging unused pages differ between operating systems. - * - * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, - * such that new pages will be demand-zeroed if - * the address region is later touched. - * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being - * unused, such that they will be discarded rather - * than swapped out. - */ -#undef JEMALLOC_PURGE_MADVISE_DONTNEED -#undef JEMALLOC_PURGE_MADVISE_FREE - -/* sizeof(void *) == 2^LG_SIZEOF_PTR. */ -#undef LG_SIZEOF_PTR - -/* sizeof(int) == 2^LG_SIZEOF_INT. */ -#undef LG_SIZEOF_INT - -/* sizeof(long) == 2^LG_SIZEOF_LONG. */ -#undef LG_SIZEOF_LONG - -#endif /* JEMALLOC_DEFS_H_ */ diff --git a/deps/jemalloc.orig/install-sh b/deps/jemalloc.orig/install-sh deleted file mode 100755 index ebc66913e94..00000000000 --- a/deps/jemalloc.orig/install-sh +++ /dev/null @@ -1,250 +0,0 @@ -#! /bin/sh -# -# install - install a program, script, or datafile -# This comes from X11R5 (mit/util/scripts/install.sh). -# -# Copyright 1991 by the Massachusetts Institute of Technology -# -# Permission to use, copy, modify, distribute, and sell this software and its -# documentation for any purpose is hereby granted without fee, provided that -# the above copyright notice appear in all copies and that both that -# copyright notice and this permission notice appear in supporting -# documentation, and that the name of M.I.T. not be used in advertising or -# publicity pertaining to distribution of the software without specific, -# written prior permission. M.I.T. makes no representations about the -# suitability of this software for any purpose. It is provided "as is" -# without express or implied warranty. -# -# Calling this script install-sh is preferred over install.sh, to prevent -# `make' implicit rules from creating a file called install from it -# when there is no Makefile. -# -# This script is compatible with the BSD install script, but was written -# from scratch. It can only install one file at a time, a restriction -# shared with many OS's install programs. - - -# set DOITPROG to echo to test this script - -# Don't use :- since 4.3BSD and earlier shells don't like it. -doit="${DOITPROG-}" - - -# put in absolute paths if you don't have them in your path; or use env. vars. - -mvprog="${MVPROG-mv}" -cpprog="${CPPROG-cp}" -chmodprog="${CHMODPROG-chmod}" -chownprog="${CHOWNPROG-chown}" -chgrpprog="${CHGRPPROG-chgrp}" -stripprog="${STRIPPROG-strip}" -rmprog="${RMPROG-rm}" -mkdirprog="${MKDIRPROG-mkdir}" - -transformbasename="" -transform_arg="" -instcmd="$mvprog" -chmodcmd="$chmodprog 0755" -chowncmd="" -chgrpcmd="" -stripcmd="" -rmcmd="$rmprog -f" -mvcmd="$mvprog" -src="" -dst="" -dir_arg="" - -while [ x"$1" != x ]; do - case $1 in - -c) instcmd="$cpprog" - shift - continue;; - - -d) dir_arg=true - shift - continue;; - - -m) chmodcmd="$chmodprog $2" - shift - shift - continue;; - - -o) chowncmd="$chownprog $2" - shift - shift - continue;; - - -g) chgrpcmd="$chgrpprog $2" - shift - shift - continue;; - - -s) stripcmd="$stripprog" - shift - continue;; - - -t=*) transformarg=`echo $1 | sed 's/-t=//'` - shift - continue;; - - -b=*) transformbasename=`echo $1 | sed 's/-b=//'` - shift - continue;; - - *) if [ x"$src" = x ] - then - src=$1 - else - # this colon is to work around a 386BSD /bin/sh bug - : - dst=$1 - fi - shift - continue;; - esac -done - -if [ x"$src" = x ] -then - echo "install: no input file specified" - exit 1 -else - true -fi - -if [ x"$dir_arg" != x ]; then - dst=$src - src="" - - if [ -d $dst ]; then - instcmd=: - else - instcmd=mkdir - fi -else - -# Waiting for this to be detected by the "$instcmd $src $dsttmp" command -# might cause directories to be created, which would be especially bad -# if $src (and thus $dsttmp) contains '*'. - - if [ -f $src -o -d $src ] - then - true - else - echo "install: $src does not exist" - exit 1 - fi - - if [ x"$dst" = x ] - then - echo "install: no destination specified" - exit 1 - else - true - fi - -# If destination is a directory, append the input filename; if your system -# does not like double slashes in filenames, you may need to add some logic - - if [ -d $dst ] - then - dst="$dst"/`basename $src` - else - true - fi -fi - -## this sed command emulates the dirname command -dstdir=`echo $dst | sed -e 's,[^/]*$,,;s,/$,,;s,^$,.,'` - -# Make sure that the destination directory exists. -# this part is taken from Noah Friedman's mkinstalldirs script - -# Skip lots of stat calls in the usual case. -if [ ! -d "$dstdir" ]; then -defaultIFS=' -' -IFS="${IFS-${defaultIFS}}" - -oIFS="${IFS}" -# Some sh's can't handle IFS=/ for some reason. -IFS='%' -set - `echo ${dstdir} | sed -e 's@/@%@g' -e 's@^%@/@'` -IFS="${oIFS}" - -pathcomp='' - -while [ $# -ne 0 ] ; do - pathcomp="${pathcomp}${1}" - shift - - if [ ! -d "${pathcomp}" ] ; - then - $mkdirprog "${pathcomp}" - else - true - fi - - pathcomp="${pathcomp}/" -done -fi - -if [ x"$dir_arg" != x ] -then - $doit $instcmd $dst && - - if [ x"$chowncmd" != x ]; then $doit $chowncmd $dst; else true ; fi && - if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dst; else true ; fi && - if [ x"$stripcmd" != x ]; then $doit $stripcmd $dst; else true ; fi && - if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dst; else true ; fi -else - -# If we're going to rename the final executable, determine the name now. - - if [ x"$transformarg" = x ] - then - dstfile=`basename $dst` - else - dstfile=`basename $dst $transformbasename | - sed $transformarg`$transformbasename - fi - -# don't allow the sed command to completely eliminate the filename - - if [ x"$dstfile" = x ] - then - dstfile=`basename $dst` - else - true - fi - -# Make a temp file name in the proper directory. - - dsttmp=$dstdir/#inst.$$# - -# Move or copy the file name to the temp name - - $doit $instcmd $src $dsttmp && - - trap "rm -f ${dsttmp}" 0 && - -# and set any options; do chmod last to preserve setuid bits - -# If any of these fail, we abort the whole thing. If we want to -# ignore errors from any of these, just make sure not to ignore -# errors from the above "$doit $instcmd $src $dsttmp" command. - - if [ x"$chowncmd" != x ]; then $doit $chowncmd $dsttmp; else true;fi && - if [ x"$chgrpcmd" != x ]; then $doit $chgrpcmd $dsttmp; else true;fi && - if [ x"$stripcmd" != x ]; then $doit $stripcmd $dsttmp; else true;fi && - if [ x"$chmodcmd" != x ]; then $doit $chmodcmd $dsttmp; else true;fi && - -# Now rename the file to the real destination. - - $doit $rmcmd -f $dstdir/$dstfile && - $doit $mvcmd $dsttmp $dstdir/$dstfile - -fi && - - -exit 0 diff --git a/deps/jemalloc.orig/src/arena.c b/deps/jemalloc.orig/src/arena.c deleted file mode 100644 index d166ca1ec4d..00000000000 --- a/deps/jemalloc.orig/src/arena.c +++ /dev/null @@ -1,2704 +0,0 @@ -#define JEMALLOC_ARENA_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -size_t opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; -size_t opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; -ssize_t opt_lg_dirty_mult = LG_DIRTY_MULT_DEFAULT; -uint8_t const *small_size2bin; -arena_bin_info_t *arena_bin_info; - -/* Various bin-related settings. */ -unsigned nqbins; -unsigned ncbins; -unsigned nsbins; -unsigned nbins; -size_t qspace_max; -size_t cspace_min; -size_t cspace_max; -size_t sspace_min; -size_t sspace_max; - -size_t lg_mspace; -size_t mspace_mask; - -/* - * const_small_size2bin is a static constant lookup table that in the common - * case can be used as-is for small_size2bin. - */ -#if (LG_TINY_MIN == 2) -#define S2B_4(i) i, -#define S2B_8(i) S2B_4(i) S2B_4(i) -#elif (LG_TINY_MIN == 3) -#define S2B_8(i) i, -#else -# error "Unsupported LG_TINY_MIN" -#endif -#define S2B_16(i) S2B_8(i) S2B_8(i) -#define S2B_32(i) S2B_16(i) S2B_16(i) -#define S2B_64(i) S2B_32(i) S2B_32(i) -#define S2B_128(i) S2B_64(i) S2B_64(i) -#define S2B_256(i) S2B_128(i) S2B_128(i) -/* - * The number of elements in const_small_size2bin is dependent on the - * definition for SUBPAGE. - */ -static JEMALLOC_ATTR(aligned(CACHELINE)) - const uint8_t const_small_size2bin[] = { -#if (LG_QUANTUM == 4) -/* 16-byte quantum **********************/ -# ifdef JEMALLOC_TINY -# if (LG_TINY_MIN == 2) - S2B_4(0) /* 4 */ - S2B_4(1) /* 8 */ - S2B_8(2) /* 16 */ -# define S2B_QMIN 2 -# elif (LG_TINY_MIN == 3) - S2B_8(0) /* 8 */ - S2B_8(1) /* 16 */ -# define S2B_QMIN 1 -# else -# error "Unsupported LG_TINY_MIN" -# endif -# else - S2B_16(0) /* 16 */ -# define S2B_QMIN 0 -# endif - S2B_16(S2B_QMIN + 1) /* 32 */ - S2B_16(S2B_QMIN + 2) /* 48 */ - S2B_16(S2B_QMIN + 3) /* 64 */ - S2B_16(S2B_QMIN + 4) /* 80 */ - S2B_16(S2B_QMIN + 5) /* 96 */ - S2B_16(S2B_QMIN + 6) /* 112 */ - S2B_16(S2B_QMIN + 7) /* 128 */ -# define S2B_CMIN (S2B_QMIN + 8) -#else -/* 8-byte quantum ***********************/ -# ifdef JEMALLOC_TINY -# if (LG_TINY_MIN == 2) - S2B_4(0) /* 4 */ - S2B_4(1) /* 8 */ -# define S2B_QMIN 1 -# else -# error "Unsupported LG_TINY_MIN" -# endif -# else - S2B_8(0) /* 8 */ -# define S2B_QMIN 0 -# endif - S2B_8(S2B_QMIN + 1) /* 16 */ - S2B_8(S2B_QMIN + 2) /* 24 */ - S2B_8(S2B_QMIN + 3) /* 32 */ - S2B_8(S2B_QMIN + 4) /* 40 */ - S2B_8(S2B_QMIN + 5) /* 48 */ - S2B_8(S2B_QMIN + 6) /* 56 */ - S2B_8(S2B_QMIN + 7) /* 64 */ - S2B_8(S2B_QMIN + 8) /* 72 */ - S2B_8(S2B_QMIN + 9) /* 80 */ - S2B_8(S2B_QMIN + 10) /* 88 */ - S2B_8(S2B_QMIN + 11) /* 96 */ - S2B_8(S2B_QMIN + 12) /* 104 */ - S2B_8(S2B_QMIN + 13) /* 112 */ - S2B_8(S2B_QMIN + 14) /* 120 */ - S2B_8(S2B_QMIN + 15) /* 128 */ -# define S2B_CMIN (S2B_QMIN + 16) -#endif -/****************************************/ - S2B_64(S2B_CMIN + 0) /* 192 */ - S2B_64(S2B_CMIN + 1) /* 256 */ - S2B_64(S2B_CMIN + 2) /* 320 */ - S2B_64(S2B_CMIN + 3) /* 384 */ - S2B_64(S2B_CMIN + 4) /* 448 */ - S2B_64(S2B_CMIN + 5) /* 512 */ -# define S2B_SMIN (S2B_CMIN + 6) - S2B_256(S2B_SMIN + 0) /* 768 */ - S2B_256(S2B_SMIN + 1) /* 1024 */ - S2B_256(S2B_SMIN + 2) /* 1280 */ - S2B_256(S2B_SMIN + 3) /* 1536 */ - S2B_256(S2B_SMIN + 4) /* 1792 */ - S2B_256(S2B_SMIN + 5) /* 2048 */ - S2B_256(S2B_SMIN + 6) /* 2304 */ - S2B_256(S2B_SMIN + 7) /* 2560 */ - S2B_256(S2B_SMIN + 8) /* 2816 */ - S2B_256(S2B_SMIN + 9) /* 3072 */ - S2B_256(S2B_SMIN + 10) /* 3328 */ - S2B_256(S2B_SMIN + 11) /* 3584 */ - S2B_256(S2B_SMIN + 12) /* 3840 */ -#if (STATIC_PAGE_SHIFT == 13) - S2B_256(S2B_SMIN + 13) /* 4096 */ - S2B_256(S2B_SMIN + 14) /* 4352 */ - S2B_256(S2B_SMIN + 15) /* 4608 */ - S2B_256(S2B_SMIN + 16) /* 4864 */ - S2B_256(S2B_SMIN + 17) /* 5120 */ - S2B_256(S2B_SMIN + 18) /* 5376 */ - S2B_256(S2B_SMIN + 19) /* 5632 */ - S2B_256(S2B_SMIN + 20) /* 5888 */ - S2B_256(S2B_SMIN + 21) /* 6144 */ - S2B_256(S2B_SMIN + 22) /* 6400 */ - S2B_256(S2B_SMIN + 23) /* 6656 */ - S2B_256(S2B_SMIN + 24) /* 6912 */ - S2B_256(S2B_SMIN + 25) /* 7168 */ - S2B_256(S2B_SMIN + 26) /* 7424 */ - S2B_256(S2B_SMIN + 27) /* 7680 */ - S2B_256(S2B_SMIN + 28) /* 7936 */ -#endif -}; -#undef S2B_1 -#undef S2B_2 -#undef S2B_4 -#undef S2B_8 -#undef S2B_16 -#undef S2B_32 -#undef S2B_64 -#undef S2B_128 -#undef S2B_256 -#undef S2B_QMIN -#undef S2B_CMIN -#undef S2B_SMIN - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, - bool large, bool zero); -static arena_chunk_t *arena_chunk_alloc(arena_t *arena); -static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk); -static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, - bool zero); -static void arena_purge(arena_t *arena, bool all); -static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); -static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, size_t oldsize, size_t newsize); -static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, size_t oldsize, size_t newsize, bool dirty); -static arena_run_t *arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin); -static void *arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin); -static void arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, - arena_bin_t *bin); -static void arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, arena_bin_t *bin); -static void arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, - arena_run_t *run, arena_bin_t *bin); -static void arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, - void *ptr, size_t oldsize, size_t size); -static bool arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, - void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); -static bool arena_ralloc_large(void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); -static bool small_size2bin_init(void); -#ifdef JEMALLOC_DEBUG -static void small_size2bin_validate(void); -#endif -static bool small_size2bin_init_hard(void); -static size_t bin_info_run_size_calc(arena_bin_info_t *bin_info, - size_t min_run_size); -static bool bin_info_init(void); - -/******************************************************************************/ - -static inline int -arena_run_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) -{ - uintptr_t a_mapelm = (uintptr_t)a; - uintptr_t b_mapelm = (uintptr_t)b; - - assert(a != NULL); - assert(b != NULL); - - return ((a_mapelm > b_mapelm) - (a_mapelm < b_mapelm)); -} - -/* Generate red-black tree functions. */ -rb_gen(static JEMALLOC_ATTR(unused), arena_run_tree_, arena_run_tree_t, - arena_chunk_map_t, u.rb_link, arena_run_comp) - -static inline int -arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) -{ - int ret; - size_t a_size = a->bits & ~PAGE_MASK; - size_t b_size = b->bits & ~PAGE_MASK; - - assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & - CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); - - ret = (a_size > b_size) - (a_size < b_size); - if (ret == 0) { - uintptr_t a_mapelm, b_mapelm; - - if ((a->bits & CHUNK_MAP_KEY) != CHUNK_MAP_KEY) - a_mapelm = (uintptr_t)a; - else { - /* - * Treat keys as though they are lower than anything - * else. - */ - a_mapelm = 0; - } - b_mapelm = (uintptr_t)b; - - ret = (a_mapelm > b_mapelm) - (a_mapelm < b_mapelm); - } - - return (ret); -} - -/* Generate red-black tree functions. */ -rb_gen(static JEMALLOC_ATTR(unused), arena_avail_tree_, arena_avail_tree_t, - arena_chunk_map_t, u.rb_link, arena_avail_comp) - -static inline void * -arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) -{ - void *ret; - unsigned regind; - bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + - (uintptr_t)bin_info->bitmap_offset); - - dassert(run->magic == ARENA_RUN_MAGIC); - assert(run->nfree > 0); - assert(bitmap_full(bitmap, &bin_info->bitmap_info) == false); - - regind = bitmap_sfu(bitmap, &bin_info->bitmap_info); - ret = (void *)((uintptr_t)run + (uintptr_t)bin_info->reg0_offset + - (uintptr_t)(bin_info->reg_size * regind)); - run->nfree--; - if (regind == run->nextind) - run->nextind++; - assert(regind < run->nextind); - return (ret); -} - -static inline void -arena_run_reg_dalloc(arena_run_t *run, void *ptr) -{ - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - unsigned regind = arena_run_regind(run, bin_info, ptr); - bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + - (uintptr_t)bin_info->bitmap_offset); - - assert(run->nfree < bin_info->nregs); - /* Freeing an interior pointer can cause assertion failure. */ - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % (uintptr_t)bin_info->reg_size - == 0); - assert((uintptr_t)ptr >= (uintptr_t)run + - (uintptr_t)bin_info->reg0_offset); - /* Freeing an unallocated pointer can cause assertion failure. */ - assert(bitmap_get(bitmap, &bin_info->bitmap_info, regind)); - - bitmap_unset(bitmap, &bin_info->bitmap_info, regind); - run->nfree++; -} - -#ifdef JEMALLOC_DEBUG -static inline void -arena_chunk_validate_zeroed(arena_chunk_t *chunk, size_t run_ind) -{ - size_t i; - size_t *p = (size_t *)((uintptr_t)chunk + (run_ind << PAGE_SHIFT)); - - for (i = 0; i < PAGE_SIZE / sizeof(size_t); i++) - assert(p[i] == 0); -} -#endif - -static void -arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, - bool zero) -{ - arena_chunk_t *chunk; - size_t old_ndirty, run_ind, total_pages, need_pages, rem_pages, i; - size_t flag_dirty; - arena_avail_tree_t *runs_avail; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - old_ndirty = chunk->ndirty; - run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) - >> PAGE_SHIFT); - flag_dirty = chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY; - runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : - &arena->runs_avail_clean; - total_pages = (chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) >> - PAGE_SHIFT; - assert((chunk->map[run_ind+total_pages-1-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty); - need_pages = (size >> PAGE_SHIFT); - assert(need_pages > 0); - assert(need_pages <= total_pages); - rem_pages = total_pages - need_pages; - - arena_avail_tree_remove(runs_avail, &chunk->map[run_ind-map_bias]); -#ifdef JEMALLOC_STATS - /* Update stats_cactive if nactive is crossing a chunk multiple. */ - cactive_diff = CHUNK_CEILING((arena->nactive + need_pages) << - PAGE_SHIFT) - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); -#endif - arena->nactive += need_pages; - - /* Keep track of trailing unused pages for later use. */ - if (rem_pages > 0) { - if (flag_dirty != 0) { - chunk->map[run_ind+need_pages-map_bias].bits = - (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; - chunk->map[run_ind+total_pages-1-map_bias].bits = - (rem_pages << PAGE_SHIFT) | CHUNK_MAP_DIRTY; - } else { - chunk->map[run_ind+need_pages-map_bias].bits = - (rem_pages << PAGE_SHIFT) | - (chunk->map[run_ind+need_pages-map_bias].bits & - CHUNK_MAP_UNZEROED); - chunk->map[run_ind+total_pages-1-map_bias].bits = - (rem_pages << PAGE_SHIFT) | - (chunk->map[run_ind+total_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED); - } - arena_avail_tree_insert(runs_avail, - &chunk->map[run_ind+need_pages-map_bias]); - } - - /* Update dirty page accounting. */ - if (flag_dirty != 0) { - chunk->ndirty -= need_pages; - arena->ndirty -= need_pages; - } - - /* - * Update the page map separately for large vs. small runs, since it is - * possible to avoid iteration for large mallocs. - */ - if (large) { - if (zero) { - if (flag_dirty == 0) { - /* - * The run is clean, so some pages may be - * zeroed (i.e. never before touched). - */ - for (i = 0; i < need_pages; i++) { - if ((chunk->map[run_ind+i-map_bias].bits - & CHUNK_MAP_UNZEROED) != 0) { - memset((void *)((uintptr_t) - chunk + ((run_ind+i) << - PAGE_SHIFT)), 0, - PAGE_SIZE); - } -#ifdef JEMALLOC_DEBUG - else { - arena_chunk_validate_zeroed( - chunk, run_ind+i); - } -#endif - } - } else { - /* - * The run is dirty, so all pages must be - * zeroed. - */ - memset((void *)((uintptr_t)chunk + (run_ind << - PAGE_SHIFT)), 0, (need_pages << - PAGE_SHIFT)); - } - } - - /* - * Set the last element first, in case the run only contains one - * page (i.e. both statements set the same element). - */ - chunk->map[run_ind+need_pages-1-map_bias].bits = - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED | flag_dirty; - chunk->map[run_ind-map_bias].bits = size | flag_dirty | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - } else { - assert(zero == false); - /* - * Propagate the dirty and unzeroed flags to the allocated - * small run, so that arena_dalloc_bin_run() has the ability to - * conditionally trim clean pages. - */ - chunk->map[run_ind-map_bias].bits = - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) | - CHUNK_MAP_ALLOCATED | flag_dirty; -#ifdef JEMALLOC_DEBUG - /* - * The first page will always be dirtied during small run - * initialization, so a validation failure here would not - * actually cause an observable failure. - */ - if (flag_dirty == 0 && - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED) - == 0) - arena_chunk_validate_zeroed(chunk, run_ind); -#endif - for (i = 1; i < need_pages - 1; i++) { - chunk->map[run_ind+i-map_bias].bits = (i << PAGE_SHIFT) - | (chunk->map[run_ind+i-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED; -#ifdef JEMALLOC_DEBUG - if (flag_dirty == 0 && - (chunk->map[run_ind+i-map_bias].bits & - CHUNK_MAP_UNZEROED) == 0) - arena_chunk_validate_zeroed(chunk, run_ind+i); -#endif - } - chunk->map[run_ind+need_pages-1-map_bias].bits = ((need_pages - - 1) << PAGE_SHIFT) | - (chunk->map[run_ind+need_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_ALLOCATED | flag_dirty; -#ifdef JEMALLOC_DEBUG - if (flag_dirty == 0 && - (chunk->map[run_ind+need_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) == 0) { - arena_chunk_validate_zeroed(chunk, - run_ind+need_pages-1); - } -#endif - } -} - -static arena_chunk_t * -arena_chunk_alloc(arena_t *arena) -{ - arena_chunk_t *chunk; - size_t i; - - if (arena->spare != NULL) { - arena_avail_tree_t *runs_avail; - - chunk = arena->spare; - arena->spare = NULL; - - /* Insert the run into the appropriate runs_avail_* tree. */ - if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) - runs_avail = &arena->runs_avail_clean; - else - runs_avail = &arena->runs_avail_dirty; - assert((chunk->map[0].bits & ~PAGE_MASK) == arena_maxclass); - assert((chunk->map[chunk_npages-1-map_bias].bits & ~PAGE_MASK) - == arena_maxclass); - assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) == - (chunk->map[chunk_npages-1-map_bias].bits & - CHUNK_MAP_DIRTY)); - arena_avail_tree_insert(runs_avail, &chunk->map[0]); - } else { - bool zero; - size_t unzeroed; - - zero = false; - malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc(chunksize, false, &zero); - malloc_mutex_lock(&arena->lock); - if (chunk == NULL) - return (NULL); -#ifdef JEMALLOC_STATS - arena->stats.mapped += chunksize; -#endif - - chunk->arena = arena; - ql_elm_new(chunk, link_dirty); - chunk->dirtied = false; - - /* - * Claim that no pages are in use, since the header is merely - * overhead. - */ - chunk->ndirty = 0; - - /* - * Initialize the map to contain one maximal free untouched run. - * Mark the pages as zeroed iff chunk_alloc() returned a zeroed - * chunk. - */ - unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; - chunk->map[0].bits = arena_maxclass | unzeroed; - /* - * There is no need to initialize the internal page map entries - * unless the chunk is not zeroed. - */ - if (zero == false) { - for (i = map_bias+1; i < chunk_npages-1; i++) - chunk->map[i-map_bias].bits = unzeroed; - } -#ifdef JEMALLOC_DEBUG - else { - for (i = map_bias+1; i < chunk_npages-1; i++) - assert(chunk->map[i-map_bias].bits == unzeroed); - } -#endif - chunk->map[chunk_npages-1-map_bias].bits = arena_maxclass | - unzeroed; - - /* Insert the run into the runs_avail_clean tree. */ - arena_avail_tree_insert(&arena->runs_avail_clean, - &chunk->map[0]); - } - - return (chunk); -} - -static void -arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) -{ - arena_avail_tree_t *runs_avail; - - /* - * Remove run from the appropriate runs_avail_* tree, so that the arena - * does not use it. - */ - if ((chunk->map[0].bits & CHUNK_MAP_DIRTY) == 0) - runs_avail = &arena->runs_avail_clean; - else - runs_avail = &arena->runs_avail_dirty; - arena_avail_tree_remove(runs_avail, &chunk->map[0]); - - if (arena->spare != NULL) { - arena_chunk_t *spare = arena->spare; - - arena->spare = chunk; - if (spare->dirtied) { - ql_remove(&chunk->arena->chunks_dirty, spare, - link_dirty); - arena->ndirty -= spare->ndirty; - } - malloc_mutex_unlock(&arena->lock); - chunk_dealloc((void *)spare, chunksize, true); - malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_STATS - arena->stats.mapped -= chunksize; -#endif - } else - arena->spare = chunk; -} - -static arena_run_t * -arena_run_alloc(arena_t *arena, size_t size, bool large, bool zero) -{ - arena_chunk_t *chunk; - arena_run_t *run; - arena_chunk_map_t *mapelm, key; - - assert(size <= arena_maxclass); - assert((size & PAGE_MASK) == 0); - - /* Search the arena's chunks for the lowest best fit. */ - key.bits = size | CHUNK_MAP_KEY; - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - - /* - * No usable runs. Create a new chunk from which to allocate the run. - */ - chunk = arena_chunk_alloc(arena); - if (chunk != NULL) { - run = (arena_run_t *)((uintptr_t)chunk + (map_bias << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - - /* - * arena_chunk_alloc() failed, but another thread may have made - * sufficient memory available while this one dropped arena->lock in - * arena_chunk_alloc(), so search one more time. - */ - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - PAGE_SHIFT)); - arena_run_split(arena, run, size, large, zero); - return (run); - } - - return (NULL); -} - -static inline void -arena_maybe_purge(arena_t *arena) -{ - - /* Enforce opt_lg_dirty_mult. */ - if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && - (arena->ndirty - arena->npurgatory) > chunk_npages && - (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - - arena->npurgatory)) - arena_purge(arena, false); -} - -static inline void -arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) -{ - ql_head(arena_chunk_map_t) mapelms; - arena_chunk_map_t *mapelm; - size_t pageind, flag_unzeroed; -#ifdef JEMALLOC_DEBUG - size_t ndirty; -#endif -#ifdef JEMALLOC_STATS - size_t nmadvise; -#endif - - ql_new(&mapelms); - - flag_unzeroed = -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - /* - * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous - * mappings, but not for file-backed mappings. - */ -# ifdef JEMALLOC_SWAP - swap_enabled ? CHUNK_MAP_UNZEROED : -# endif - 0; -#else - CHUNK_MAP_UNZEROED; -#endif - - /* - * If chunk is the spare, temporarily re-allocate it, 1) so that its - * run is reinserted into runs_avail_dirty, and 2) so that it cannot be - * completely discarded by another thread while arena->lock is dropped - * by this thread. Note that the arena_run_dalloc() call will - * implicitly deallocate the chunk, so no explicit action is required - * in this function to deallocate the chunk. - * - * Note that once a chunk contains dirty pages, it cannot again contain - * a single run unless 1) it is a dirty run, or 2) this function purges - * dirty pages and causes the transition to a single clean run. Thus - * (chunk == arena->spare) is possible, but it is not possible for - * this function to be called on the spare unless it contains a dirty - * run. - */ - if (chunk == arena->spare) { - assert((chunk->map[0].bits & CHUNK_MAP_DIRTY) != 0); - arena_chunk_alloc(arena); - } - - /* Temporarily allocate all free dirty runs within chunk. */ - for (pageind = map_bias; pageind < chunk_npages;) { - mapelm = &chunk->map[pageind-map_bias]; - if ((mapelm->bits & CHUNK_MAP_ALLOCATED) == 0) { - size_t npages; - - npages = mapelm->bits >> PAGE_SHIFT; - assert(pageind + npages <= chunk_npages); - if (mapelm->bits & CHUNK_MAP_DIRTY) { - size_t i; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif - - arena_avail_tree_remove( - &arena->runs_avail_dirty, mapelm); - - mapelm->bits = (npages << PAGE_SHIFT) | - flag_unzeroed | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED; - /* - * Update internal elements in the page map, so - * that CHUNK_MAP_UNZEROED is properly set. - */ - for (i = 1; i < npages - 1; i++) { - chunk->map[pageind+i-map_bias].bits = - flag_unzeroed; - } - if (npages > 1) { - chunk->map[ - pageind+npages-1-map_bias].bits = - flag_unzeroed | CHUNK_MAP_LARGE | - CHUNK_MAP_ALLOCATED; - } - -#ifdef JEMALLOC_STATS - /* - * Update stats_cactive if nactive is crossing a - * chunk multiple. - */ - cactive_diff = CHUNK_CEILING((arena->nactive + - npages) << PAGE_SHIFT) - - CHUNK_CEILING(arena->nactive << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); -#endif - arena->nactive += npages; - /* Append to list for later processing. */ - ql_elm_new(mapelm, u.ql_link); - ql_tail_insert(&mapelms, mapelm, u.ql_link); - } - - pageind += npages; - } else { - /* Skip allocated run. */ - if (mapelm->bits & CHUNK_MAP_LARGE) - pageind += mapelm->bits >> PAGE_SHIFT; - else { - arena_run_t *run = (arena_run_t *)((uintptr_t) - chunk + (uintptr_t)(pageind << PAGE_SHIFT)); - - assert((mapelm->bits >> PAGE_SHIFT) == 0); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(arena, - run->bin); - arena_bin_info_t *bin_info = - &arena_bin_info[binind]; - pageind += bin_info->run_size >> PAGE_SHIFT; - } - } - } - assert(pageind == chunk_npages); - -#ifdef JEMALLOC_DEBUG - ndirty = chunk->ndirty; -#endif -#ifdef JEMALLOC_STATS - arena->stats.purged += chunk->ndirty; -#endif - arena->ndirty -= chunk->ndirty; - chunk->ndirty = 0; - ql_remove(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = false; - - malloc_mutex_unlock(&arena->lock); -#ifdef JEMALLOC_STATS - nmadvise = 0; -#endif - ql_foreach(mapelm, &mapelms, u.ql_link) { - size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t)) + map_bias; - size_t npages = mapelm->bits >> PAGE_SHIFT; - - assert(pageind + npages <= chunk_npages); -#ifdef JEMALLOC_DEBUG - assert(ndirty >= npages); - ndirty -= npages; -#endif - -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), - (npages << PAGE_SHIFT), MADV_DONTNEED); -#elif defined(JEMALLOC_PURGE_MADVISE_FREE) - madvise((void *)((uintptr_t)chunk + (pageind << PAGE_SHIFT)), - (npages << PAGE_SHIFT), MADV_FREE); -#else -# error "No method defined for purging unused dirty pages." -#endif - -#ifdef JEMALLOC_STATS - nmadvise++; -#endif - } -#ifdef JEMALLOC_DEBUG - assert(ndirty == 0); -#endif - malloc_mutex_lock(&arena->lock); -#ifdef JEMALLOC_STATS - arena->stats.nmadvise += nmadvise; -#endif - - /* Deallocate runs. */ - for (mapelm = ql_first(&mapelms); mapelm != NULL; - mapelm = ql_first(&mapelms)) { - size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t)) + map_bias; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)(pageind << PAGE_SHIFT)); - - ql_remove(&mapelms, mapelm, u.ql_link); - arena_run_dalloc(arena, run, false); - } -} - -static void -arena_purge(arena_t *arena, bool all) -{ - arena_chunk_t *chunk; - size_t npurgatory; -#ifdef JEMALLOC_DEBUG - size_t ndirty = 0; - - ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { - assert(chunk->dirtied); - ndirty += chunk->ndirty; - } - assert(ndirty == arena->ndirty); -#endif - assert(arena->ndirty > arena->npurgatory || all); - assert(arena->ndirty - arena->npurgatory > chunk_npages || all); - assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - - arena->npurgatory) || all); - -#ifdef JEMALLOC_STATS - arena->stats.npurge++; -#endif - - /* - * Compute the minimum number of pages that this thread should try to - * purge, and add the result to arena->npurgatory. This will keep - * multiple threads from racing to reduce ndirty below the threshold. - */ - npurgatory = arena->ndirty - arena->npurgatory; - if (all == false) { - assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); - npurgatory -= arena->nactive >> opt_lg_dirty_mult; - } - arena->npurgatory += npurgatory; - - while (npurgatory > 0) { - /* Get next chunk with dirty pages. */ - chunk = ql_first(&arena->chunks_dirty); - if (chunk == NULL) { - /* - * This thread was unable to purge as many pages as - * originally intended, due to races with other threads - * that either did some of the purging work, or re-used - * dirty pages. - */ - arena->npurgatory -= npurgatory; - return; - } - while (chunk->ndirty == 0) { - ql_remove(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = false; - chunk = ql_first(&arena->chunks_dirty); - if (chunk == NULL) { - /* Same logic as for above. */ - arena->npurgatory -= npurgatory; - return; - } - } - - if (chunk->ndirty > npurgatory) { - /* - * This thread will, at a minimum, purge all the dirty - * pages in chunk, so set npurgatory to reflect this - * thread's commitment to purge the pages. This tends - * to reduce the chances of the following scenario: - * - * 1) This thread sets arena->npurgatory such that - * (arena->ndirty - arena->npurgatory) is at the - * threshold. - * 2) This thread drops arena->lock. - * 3) Another thread causes one or more pages to be - * dirtied, and immediately determines that it must - * purge dirty pages. - * - * If this scenario *does* play out, that's okay, - * because all of the purging work being done really - * needs to happen. - */ - arena->npurgatory += chunk->ndirty - npurgatory; - npurgatory = chunk->ndirty; - } - - arena->npurgatory -= chunk->ndirty; - npurgatory -= chunk->ndirty; - arena_chunk_purge(arena, chunk); - } -} - -void -arena_purge_all(arena_t *arena) -{ - - malloc_mutex_lock(&arena->lock); - arena_purge(arena, true); - malloc_mutex_unlock(&arena->lock); -} - -static void -arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) -{ - arena_chunk_t *chunk; - size_t size, run_ind, run_pages, flag_dirty; - arena_avail_tree_t *runs_avail; -#ifdef JEMALLOC_STATS - size_t cactive_diff; -#endif - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) - >> PAGE_SHIFT); - assert(run_ind >= map_bias); - assert(run_ind < chunk_npages); - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_LARGE) != 0) { - size = chunk->map[run_ind-map_bias].bits & ~PAGE_MASK; - assert(size == PAGE_SIZE || - (chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - ~PAGE_MASK) == 0); - assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - CHUNK_MAP_LARGE) != 0); - assert((chunk->map[run_ind+(size>>PAGE_SHIFT)-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) != 0); - } else { - size_t binind = arena_bin_index(arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - size = bin_info->run_size; - } - run_pages = (size >> PAGE_SHIFT); -#ifdef JEMALLOC_STATS - /* Update stats_cactive if nactive is crossing a chunk multiple. */ - cactive_diff = CHUNK_CEILING(arena->nactive << PAGE_SHIFT) - - CHUNK_CEILING((arena->nactive - run_pages) << PAGE_SHIFT); - if (cactive_diff != 0) - stats_cactive_sub(cactive_diff); -#endif - arena->nactive -= run_pages; - - /* - * The run is dirty if the caller claims to have dirtied it, as well as - * if it was already dirty before being allocated. - */ - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) != 0) - dirty = true; - flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; - runs_avail = dirty ? &arena->runs_avail_dirty : - &arena->runs_avail_clean; - - /* Mark pages as unallocated in the chunk map. */ - if (dirty) { - chunk->map[run_ind-map_bias].bits = size | CHUNK_MAP_DIRTY; - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - CHUNK_MAP_DIRTY; - - chunk->ndirty += run_pages; - arena->ndirty += run_pages; - } else { - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_UNZEROED); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_UNZEROED); - } - - /* Try to coalesce forward. */ - if (run_ind + run_pages < chunk_npages && - (chunk->map[run_ind+run_pages-map_bias].bits & CHUNK_MAP_ALLOCATED) - == 0 && (chunk->map[run_ind+run_pages-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty) { - size_t nrun_size = chunk->map[run_ind+run_pages-map_bias].bits & - ~PAGE_MASK; - size_t nrun_pages = nrun_size >> PAGE_SHIFT; - - /* - * Remove successor from runs_avail; the coalesced run is - * inserted later. - */ - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & ~PAGE_MASK) == nrun_size); - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & CHUNK_MAP_ALLOCATED) == 0); - assert((chunk->map[run_ind+run_pages+nrun_pages-1-map_bias].bits - & CHUNK_MAP_DIRTY) == flag_dirty); - arena_avail_tree_remove(runs_avail, - &chunk->map[run_ind+run_pages-map_bias]); - - size += nrun_size; - run_pages += nrun_pages; - - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - } - - /* Try to coalesce backward. */ - if (run_ind > map_bias && (chunk->map[run_ind-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) == 0 && (chunk->map[run_ind-1-map_bias].bits & - CHUNK_MAP_DIRTY) == flag_dirty) { - size_t prun_size = chunk->map[run_ind-1-map_bias].bits & - ~PAGE_MASK; - size_t prun_pages = prun_size >> PAGE_SHIFT; - - run_ind -= prun_pages; - - /* - * Remove predecessor from runs_avail; the coalesced run is - * inserted later. - */ - assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) - == prun_size); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_ALLOCATED) - == 0); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) - == flag_dirty); - arena_avail_tree_remove(runs_avail, - &chunk->map[run_ind-map_bias]); - - size += prun_size; - run_pages += prun_pages; - - chunk->map[run_ind-map_bias].bits = size | - (chunk->map[run_ind-map_bias].bits & CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind+run_pages-1-map_bias].bits = size | - (chunk->map[run_ind+run_pages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - } - - /* Insert into runs_avail, now that coalescing is complete. */ - assert((chunk->map[run_ind-map_bias].bits & ~PAGE_MASK) == - (chunk->map[run_ind+run_pages-1-map_bias].bits & ~PAGE_MASK)); - assert((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == - (chunk->map[run_ind+run_pages-1-map_bias].bits & CHUNK_MAP_DIRTY)); - arena_avail_tree_insert(runs_avail, &chunk->map[run_ind-map_bias]); - - if (dirty) { - /* - * Insert into chunks_dirty before potentially calling - * arena_chunk_dealloc(), so that chunks_dirty and - * arena->ndirty are consistent. - */ - if (chunk->dirtied == false) { - ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = true; - } - } - - /* - * Deallocate chunk if it is now completely unused. The bit - * manipulation checks whether the first run is unallocated and extends - * to the end of the chunk. - */ - if ((chunk->map[0].bits & (~PAGE_MASK | CHUNK_MAP_ALLOCATED)) == - arena_maxclass) - arena_chunk_dealloc(arena, chunk); - - /* - * It is okay to do dirty page processing here even if the chunk was - * deallocated above, since in that case it is the spare. Waiting - * until after possible chunk deallocation to do dirty processing - * allows for an old spare to be fully deallocated, thus decreasing the - * chances of spuriously crossing the dirty page purging threshold. - */ - if (dirty) - arena_maybe_purge(arena); -} - -static void -arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, - size_t oldsize, size_t newsize) -{ - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t head_npages = (oldsize - newsize) >> PAGE_SHIFT; - size_t flag_dirty = chunk->map[pageind-map_bias].bits & CHUNK_MAP_DIRTY; - - assert(oldsize > newsize); - - /* - * Update the chunk map so that arena_run_dalloc() can treat the - * leading run as separately allocated. Set the last element of each - * run first, in case of single-page runs. - */ - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | - (chunk->map[pageind+head_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind-map_bias].bits = (oldsize - newsize) - | flag_dirty | (chunk->map[pageind-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - -#ifdef JEMALLOC_DEBUG - { - size_t tail_npages = newsize >> PAGE_SHIFT; - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & ~PAGE_MASK) == 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_DIRTY) == flag_dirty); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias] - .bits & CHUNK_MAP_ALLOCATED) != 0); - } -#endif - chunk->map[pageind+head_npages-map_bias].bits = newsize | flag_dirty | - (chunk->map[pageind+head_npages-map_bias].bits & - CHUNK_MAP_FLAGS_MASK) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - - arena_run_dalloc(arena, run, false); -} - -static void -arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, - size_t oldsize, size_t newsize, bool dirty) -{ - size_t pageind = ((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t head_npages = newsize >> PAGE_SHIFT; - size_t tail_npages = (oldsize - newsize) >> PAGE_SHIFT; - size_t flag_dirty = chunk->map[pageind-map_bias].bits & - CHUNK_MAP_DIRTY; - - assert(oldsize > newsize); - - /* - * Update the chunk map so that arena_run_dalloc() can treat the - * trailing run as separately allocated. Set the last element of each - * run first, in case of single-page runs. - */ - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind-map_bias].bits & CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages-1-map_bias].bits = flag_dirty | - (chunk->map[pageind+head_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind-map_bias].bits = newsize | flag_dirty | - (chunk->map[pageind-map_bias].bits & CHUNK_MAP_UNZEROED) | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - ~PAGE_MASK) == 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_LARGE) != 0); - assert((chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_ALLOCATED) != 0); - chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits = - flag_dirty | - (chunk->map[pageind+head_npages+tail_npages-1-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind+head_npages-map_bias].bits = (oldsize - newsize) | - flag_dirty | (chunk->map[pageind+head_npages-map_bias].bits & - CHUNK_MAP_UNZEROED) | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - - arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), - dirty); -} - -static arena_run_t * -arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) -{ - arena_chunk_map_t *mapelm; - arena_run_t *run; - size_t binind; - arena_bin_info_t *bin_info; - - /* Look for a usable run. */ - mapelm = arena_run_tree_first(&bin->runs); - if (mapelm != NULL) { - arena_chunk_t *chunk; - size_t pageind; - - /* run is guaranteed to have available space. */ - arena_run_tree_remove(&bin->runs, mapelm); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t))) + map_bias; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) - << PAGE_SHIFT)); -#ifdef JEMALLOC_STATS - bin->stats.reruns++; -#endif - return (run); - } - /* No existing runs have any space available. */ - - binind = arena_bin_index(arena, bin); - bin_info = &arena_bin_info[binind]; - - /* Allocate a new run. */ - malloc_mutex_unlock(&bin->lock); - /******************************/ - malloc_mutex_lock(&arena->lock); - run = arena_run_alloc(arena, bin_info->run_size, false, false); - if (run != NULL) { - bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + - (uintptr_t)bin_info->bitmap_offset); - - /* Initialize run internals. */ - run->bin = bin; - run->nextind = 0; - run->nfree = bin_info->nregs; - bitmap_init(bitmap, &bin_info->bitmap_info); -#ifdef JEMALLOC_DEBUG - run->magic = ARENA_RUN_MAGIC; -#endif - } - malloc_mutex_unlock(&arena->lock); - /********************************/ - malloc_mutex_lock(&bin->lock); - if (run != NULL) { -#ifdef JEMALLOC_STATS - bin->stats.nruns++; - bin->stats.curruns++; - if (bin->stats.curruns > bin->stats.highruns) - bin->stats.highruns = bin->stats.curruns; -#endif - return (run); - } - - /* - * arena_run_alloc() failed, but another thread may have made - * sufficient memory available while this one dropped bin->lock above, - * so search one more time. - */ - mapelm = arena_run_tree_first(&bin->runs); - if (mapelm != NULL) { - arena_chunk_t *chunk; - size_t pageind; - - /* run is guaranteed to have available space. */ - arena_run_tree_remove(&bin->runs, mapelm); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(mapelm); - pageind = ((((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t))) + map_bias; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) - << PAGE_SHIFT)); -#ifdef JEMALLOC_STATS - bin->stats.reruns++; -#endif - return (run); - } - - return (NULL); -} - -/* Re-fill bin->runcur, then call arena_run_reg_alloc(). */ -static void * -arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) -{ - void *ret; - size_t binind; - arena_bin_info_t *bin_info; - arena_run_t *run; - - binind = arena_bin_index(arena, bin); - bin_info = &arena_bin_info[binind]; - bin->runcur = NULL; - run = arena_bin_nonfull_run_get(arena, bin); - if (bin->runcur != NULL && bin->runcur->nfree > 0) { - /* - * Another thread updated runcur while this one ran without the - * bin lock in arena_bin_nonfull_run_get(). - */ - dassert(bin->runcur->magic == ARENA_RUN_MAGIC); - assert(bin->runcur->nfree > 0); - ret = arena_run_reg_alloc(bin->runcur, bin_info); - if (run != NULL) { - arena_chunk_t *chunk; - - /* - * arena_run_alloc() may have allocated run, or it may - * have pulled run from the bin's run tree. Therefore - * it is unsafe to make any assumptions about how run - * has previously been used, and arena_bin_lower_run() - * must be called, as if a region were just deallocated - * from the run. - */ - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - if (run->nfree == bin_info->nregs) - arena_dalloc_bin_run(arena, chunk, run, bin); - else - arena_bin_lower_run(arena, chunk, run, bin); - } - return (ret); - } - - if (run == NULL) - return (NULL); - - bin->runcur = run; - - dassert(bin->runcur->magic == ARENA_RUN_MAGIC); - assert(bin->runcur->nfree > 0); - - return (arena_run_reg_alloc(bin->runcur, bin_info)); -} - -#ifdef JEMALLOC_PROF -void -arena_prof_accum(arena_t *arena, uint64_t accumbytes) -{ - - if (prof_interval != 0) { - arena->prof_accumbytes += accumbytes; - if (arena->prof_accumbytes >= prof_interval) { - prof_idump(); - arena->prof_accumbytes -= prof_interval; - } - } -} -#endif - -#ifdef JEMALLOC_TCACHE -void -arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind -# ifdef JEMALLOC_PROF - , uint64_t prof_accumbytes -# endif - ) -{ - unsigned i, nfill; - arena_bin_t *bin; - arena_run_t *run; - void *ptr; - - assert(tbin->ncached == 0); - -#ifdef JEMALLOC_PROF - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, prof_accumbytes); - malloc_mutex_unlock(&arena->lock); -#endif - bin = &arena->bins[binind]; - malloc_mutex_lock(&bin->lock); - for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> - tbin->lg_fill_div); i < nfill; i++) { - if ((run = bin->runcur) != NULL && run->nfree > 0) - ptr = arena_run_reg_alloc(run, &arena_bin_info[binind]); - else - ptr = arena_bin_malloc_hard(arena, bin); - if (ptr == NULL) - break; - /* Insert such that low regions get used first. */ - tbin->avail[nfill - 1 - i] = ptr; - } -#ifdef JEMALLOC_STATS - bin->stats.allocated += i * arena_bin_info[binind].reg_size; - bin->stats.nmalloc += i; - bin->stats.nrequests += tbin->tstats.nrequests; - bin->stats.nfills++; - tbin->tstats.nrequests = 0; -#endif - malloc_mutex_unlock(&bin->lock); - tbin->ncached = i; -} -#endif - -void * -arena_malloc_small(arena_t *arena, size_t size, bool zero) -{ - void *ret; - arena_bin_t *bin; - arena_run_t *run; - size_t binind; - - binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); - bin = &arena->bins[binind]; - size = arena_bin_info[binind].reg_size; - - malloc_mutex_lock(&bin->lock); - if ((run = bin->runcur) != NULL && run->nfree > 0) - ret = arena_run_reg_alloc(run, &arena_bin_info[binind]); - else - ret = arena_bin_malloc_hard(arena, bin); - - if (ret == NULL) { - malloc_mutex_unlock(&bin->lock); - return (NULL); - } - -#ifdef JEMALLOC_STATS - bin->stats.allocated += size; - bin->stats.nmalloc++; - bin->stats.nrequests++; -#endif - malloc_mutex_unlock(&bin->lock); -#ifdef JEMALLOC_PROF - if (isthreaded == false) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, size); - malloc_mutex_unlock(&arena->lock); - } -#endif - - if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } else - memset(ret, 0, size); - - return (ret); -} - -void * -arena_malloc_large(arena_t *arena, size_t size, bool zero) -{ - void *ret; - - /* Large allocation. */ - size = PAGE_CEILING(size); - malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, size, true, zero); - if (ret == NULL) { - malloc_mutex_unlock(&arena->lock); - return (NULL); - } -#ifdef JEMALLOC_STATS - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; - } -#endif -#ifdef JEMALLOC_PROF - arena_prof_accum(arena, size); -#endif - malloc_mutex_unlock(&arena->lock); - - if (zero == false) { -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); -#endif - } - - return (ret); -} - -void * -arena_malloc(size_t size, bool zero) -{ - - assert(size != 0); - assert(QUANTUM_CEILING(size) <= arena_maxclass); - - if (size <= small_maxclass) { -#ifdef JEMALLOC_TCACHE - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - return (tcache_alloc_small(tcache, size, zero)); - else - -#endif - return (arena_malloc_small(choose_arena(), size, zero)); - } else { -#ifdef JEMALLOC_TCACHE - if (size <= tcache_maxclass) { - tcache_t *tcache; - - if ((tcache = tcache_get()) != NULL) - return (tcache_alloc_large(tcache, size, zero)); - else { - return (arena_malloc_large(choose_arena(), - size, zero)); - } - } else -#endif - return (arena_malloc_large(choose_arena(), size, zero)); - } -} - -/* Only handles large allocations that require more than page alignment. */ -void * -arena_palloc(arena_t *arena, size_t size, size_t alloc_size, size_t alignment, - bool zero) -{ - void *ret; - size_t offset; - arena_chunk_t *chunk; - - assert((size & PAGE_MASK) == 0); - - alignment = PAGE_CEILING(alignment); - - malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, alloc_size, true, zero); - if (ret == NULL) { - malloc_mutex_unlock(&arena->lock); - return (NULL); - } - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ret); - - offset = (uintptr_t)ret & (alignment - 1); - assert((offset & PAGE_MASK) == 0); - assert(offset < alloc_size); - if (offset == 0) - arena_run_trim_tail(arena, chunk, ret, alloc_size, size, false); - else { - size_t leadsize, trailsize; - - leadsize = alignment - offset; - if (leadsize > 0) { - arena_run_trim_head(arena, chunk, ret, alloc_size, - alloc_size - leadsize); - ret = (void *)((uintptr_t)ret + leadsize); - } - - trailsize = alloc_size - leadsize - size; - if (trailsize != 0) { - /* Trim trailing space. */ - assert(trailsize < alloc_size); - arena_run_trim_tail(arena, chunk, ret, size + trailsize, - size, false); - } - } - -#ifdef JEMALLOC_STATS - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; - } -#endif - malloc_mutex_unlock(&arena->lock); - -#ifdef JEMALLOC_FILL - if (zero == false) { - if (opt_junk) - memset(ret, 0xa5, size); - else if (opt_zero) - memset(ret, 0, size); - } -#endif - return (ret); -} - -/* Return the size of the allocation pointed to by ptr. */ -size_t -arena_salloc(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - assert(ret != 0); - } - - return (ret); -} - -#ifdef JEMALLOC_PROF -void -arena_prof_promoted(const void *ptr, size_t size) -{ - arena_chunk_t *chunk; - size_t pageind, binind; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - assert(isalloc(ptr) == PAGE_SIZE); - assert(size <= small_maxclass); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - binind = SMALL_SIZE2BIN(size); - assert(binind < nbins); - chunk->map[pageind-map_bias].bits = (chunk->map[pageind-map_bias].bits & - ~CHUNK_MAP_CLASS_MASK) | ((binind+1) << CHUNK_MAP_CLASS_SHIFT); -} - -size_t -arena_salloc_demote(const void *ptr) -{ - size_t ret; - arena_chunk_t *chunk; - size_t pageind, mapbits; - - assert(ptr != NULL); - assert(CHUNK_ADDR2BASE(ptr) != ptr); - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - mapbits = chunk->map[pageind-map_bias].bits; - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapbits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - size_t binind = arena_bin_index(chunk->arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - assert(((uintptr_t)ptr - ((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset)) % bin_info->reg_size == - 0); - ret = bin_info->reg_size; - } else { - assert(((uintptr_t)ptr & PAGE_MASK) == 0); - ret = mapbits & ~PAGE_MASK; - if (prof_promote && ret == PAGE_SIZE && (mapbits & - CHUNK_MAP_CLASS_MASK) != 0) { - size_t binind = ((mapbits & CHUNK_MAP_CLASS_MASK) >> - CHUNK_MAP_CLASS_SHIFT) - 1; - assert(binind < nbins); - ret = arena_bin_info[binind].reg_size; - } - assert(ret != 0); - } - - return (ret); -} -#endif - -static void -arena_dissociate_bin_run(arena_chunk_t *chunk, arena_run_t *run, - arena_bin_t *bin) -{ - - /* Dissociate run from bin. */ - if (run == bin->runcur) - bin->runcur = NULL; - else { - size_t binind = arena_bin_index(chunk->arena, bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - - if (bin_info->nregs != 1) { - size_t run_pageind = (((uintptr_t)run - - (uintptr_t)chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *run_mapelm = - &chunk->map[run_pageind-map_bias]; - /* - * This block's conditional is necessary because if the - * run only contains one region, then it never gets - * inserted into the non-full runs tree. - */ - arena_run_tree_remove(&bin->runs, run_mapelm); - } - } -} - -static void -arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, - arena_bin_t *bin) -{ - size_t binind; - arena_bin_info_t *bin_info; - size_t npages, run_ind, past; - - assert(run != bin->runcur); - assert(arena_run_tree_search(&bin->runs, &chunk->map[ - (((uintptr_t)run-(uintptr_t)chunk)>>PAGE_SHIFT)-map_bias]) == NULL); - - binind = arena_bin_index(chunk->arena, run->bin); - bin_info = &arena_bin_info[binind]; - - malloc_mutex_unlock(&bin->lock); - /******************************/ - npages = bin_info->run_size >> PAGE_SHIFT; - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> PAGE_SHIFT); - past = (size_t)(PAGE_CEILING((uintptr_t)run + - (uintptr_t)bin_info->reg0_offset + (uintptr_t)(run->nextind * - bin_info->reg_size) - (uintptr_t)chunk) >> PAGE_SHIFT); - malloc_mutex_lock(&arena->lock); - - /* - * If the run was originally clean, and some pages were never touched, - * trim the clean pages before deallocating the dirty portion of the - * run. - */ - if ((chunk->map[run_ind-map_bias].bits & CHUNK_MAP_DIRTY) == 0 && past - - run_ind < npages) { - /* - * Trim clean pages. Convert to large run beforehand. Set the - * last map element first, in case this is a one-page run. - */ - chunk->map[run_ind+npages-1-map_bias].bits = CHUNK_MAP_LARGE | - (chunk->map[run_ind+npages-1-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - chunk->map[run_ind-map_bias].bits = bin_info->run_size | - CHUNK_MAP_LARGE | (chunk->map[run_ind-map_bias].bits & - CHUNK_MAP_FLAGS_MASK); - arena_run_trim_tail(arena, chunk, run, (npages << PAGE_SHIFT), - ((past - run_ind) << PAGE_SHIFT), false); - /* npages = past - run_ind; */ - } -#ifdef JEMALLOC_DEBUG - run->magic = 0; -#endif - arena_run_dalloc(arena, run, true); - malloc_mutex_unlock(&arena->lock); - /****************************/ - malloc_mutex_lock(&bin->lock); -#ifdef JEMALLOC_STATS - bin->stats.curruns--; -#endif -} - -static void -arena_bin_lower_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, - arena_bin_t *bin) -{ - - /* - * Make sure that bin->runcur always refers to the lowest non-full run, - * if one exists. - */ - if (bin->runcur == NULL) - bin->runcur = run; - else if ((uintptr_t)run < (uintptr_t)bin->runcur) { - /* Switch runcur. */ - if (bin->runcur->nfree > 0) { - arena_chunk_t *runcur_chunk = - CHUNK_ADDR2BASE(bin->runcur); - size_t runcur_pageind = (((uintptr_t)bin->runcur - - (uintptr_t)runcur_chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *runcur_mapelm = - &runcur_chunk->map[runcur_pageind-map_bias]; - - /* Insert runcur. */ - arena_run_tree_insert(&bin->runs, runcur_mapelm); - } - bin->runcur = run; - } else { - size_t run_pageind = (((uintptr_t)run - - (uintptr_t)chunk)) >> PAGE_SHIFT; - arena_chunk_map_t *run_mapelm = - &chunk->map[run_pageind-map_bias]; - - assert(arena_run_tree_search(&bin->runs, run_mapelm) == NULL); - arena_run_tree_insert(&bin->runs, run_mapelm); - } -} - -void -arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, - arena_chunk_map_t *mapelm) -{ - size_t pageind; - arena_run_t *run; - arena_bin_t *bin; -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size_t size; -#endif - - pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - - (mapelm->bits >> PAGE_SHIFT)) << PAGE_SHIFT)); - dassert(run->magic == ARENA_RUN_MAGIC); - bin = run->bin; - size_t binind = arena_bin_index(arena, bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size = bin_info->reg_size; -#endif - -#ifdef JEMALLOC_FILL - if (opt_junk) - memset(ptr, 0x5a, size); -#endif - - arena_run_reg_dalloc(run, ptr); - if (run->nfree == bin_info->nregs) { - arena_dissociate_bin_run(chunk, run, bin); - arena_dalloc_bin_run(arena, chunk, run, bin); - } else if (run->nfree == 1 && run != bin->runcur) - arena_bin_lower_run(arena, chunk, run, bin); - -#ifdef JEMALLOC_STATS - bin->stats.allocated -= size; - bin->stats.ndalloc++; -#endif -} - -#ifdef JEMALLOC_STATS -void -arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats) -{ - unsigned i; - - malloc_mutex_lock(&arena->lock); - *nactive += arena->nactive; - *ndirty += arena->ndirty; - - astats->mapped += arena->stats.mapped; - astats->npurge += arena->stats.npurge; - astats->nmadvise += arena->stats.nmadvise; - astats->purged += arena->stats.purged; - astats->allocated_large += arena->stats.allocated_large; - astats->nmalloc_large += arena->stats.nmalloc_large; - astats->ndalloc_large += arena->stats.ndalloc_large; - astats->nrequests_large += arena->stats.nrequests_large; - - for (i = 0; i < nlclasses; i++) { - lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; - lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; - lstats[i].nrequests += arena->stats.lstats[i].nrequests; - lstats[i].highruns += arena->stats.lstats[i].highruns; - lstats[i].curruns += arena->stats.lstats[i].curruns; - } - malloc_mutex_unlock(&arena->lock); - - for (i = 0; i < nbins; i++) { - arena_bin_t *bin = &arena->bins[i]; - - malloc_mutex_lock(&bin->lock); - bstats[i].allocated += bin->stats.allocated; - bstats[i].nmalloc += bin->stats.nmalloc; - bstats[i].ndalloc += bin->stats.ndalloc; - bstats[i].nrequests += bin->stats.nrequests; -#ifdef JEMALLOC_TCACHE - bstats[i].nfills += bin->stats.nfills; - bstats[i].nflushes += bin->stats.nflushes; -#endif - bstats[i].nruns += bin->stats.nruns; - bstats[i].reruns += bin->stats.reruns; - bstats[i].highruns += bin->stats.highruns; - bstats[i].curruns += bin->stats.curruns; - malloc_mutex_unlock(&bin->lock); - } -} -#endif - -void -arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr) -{ - - /* Large allocation. */ -#ifdef JEMALLOC_FILL -# ifndef JEMALLOC_STATS - if (opt_junk) -# endif -#endif - { -#if (defined(JEMALLOC_FILL) || defined(JEMALLOC_STATS)) - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> - PAGE_SHIFT; - size_t size = chunk->map[pageind-map_bias].bits & ~PAGE_MASK; -#endif - -#ifdef JEMALLOC_FILL -# ifdef JEMALLOC_STATS - if (opt_junk) -# endif - memset(ptr, 0x5a, size); -#endif -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns--; -#endif - } - - arena_run_dalloc(arena, (arena_run_t *)ptr, true); -} - -static void -arena_ralloc_large_shrink(arena_t *arena, arena_chunk_t *chunk, void *ptr, - size_t oldsize, size_t size) -{ - - assert(size < oldsize); - - /* - * Shrink the run, and make trailing pages available for other - * allocations. - */ - malloc_mutex_lock(&arena->lock); - arena_run_trim_tail(arena, chunk, (arena_run_t *)ptr, oldsize, size, - true); -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; - - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns; - } -#endif - malloc_mutex_unlock(&arena->lock); -} - -static bool -arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, - size_t oldsize, size_t size, size_t extra, bool zero) -{ - size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> PAGE_SHIFT; - size_t npages = oldsize >> PAGE_SHIFT; - size_t followsize; - - assert(oldsize == (chunk->map[pageind-map_bias].bits & ~PAGE_MASK)); - - /* Try to extend the run. */ - assert(size + extra > oldsize); - malloc_mutex_lock(&arena->lock); - if (pageind + npages < chunk_npages && - (chunk->map[pageind+npages-map_bias].bits - & CHUNK_MAP_ALLOCATED) == 0 && (followsize = - chunk->map[pageind+npages-map_bias].bits & ~PAGE_MASK) >= size - - oldsize) { - /* - * The next run is available and sufficiently large. Split the - * following run, then merge the first part with the existing - * allocation. - */ - size_t flag_dirty; - size_t splitsize = (oldsize + followsize <= size + extra) - ? followsize : size + extra - oldsize; - arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + - ((pageind+npages) << PAGE_SHIFT)), splitsize, true, zero); - - size = oldsize + splitsize; - npages = size >> PAGE_SHIFT; - - /* - * Mark the extended run as dirty if either portion of the run - * was dirty before allocation. This is rather pedantic, - * because there's not actually any sequence of events that - * could cause the resulting run to be passed to - * arena_run_dalloc() with the dirty argument set to false - * (which is when dirty flag consistency would really matter). - */ - flag_dirty = (chunk->map[pageind-map_bias].bits & - CHUNK_MAP_DIRTY) | - (chunk->map[pageind+npages-1-map_bias].bits & - CHUNK_MAP_DIRTY); - chunk->map[pageind-map_bias].bits = size | flag_dirty - | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - chunk->map[pageind+npages-1-map_bias].bits = flag_dirty | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; - -#ifdef JEMALLOC_STATS - arena->stats.ndalloc_large++; - arena->stats.allocated_large -= oldsize; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].ndalloc++; - arena->stats.lstats[(oldsize >> PAGE_SHIFT) - 1].curruns--; - - arena->stats.nmalloc_large++; - arena->stats.nrequests_large++; - arena->stats.allocated_large += size; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nmalloc++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].nrequests++; - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns++; - if (arena->stats.lstats[(size >> PAGE_SHIFT) - 1].curruns > - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns) { - arena->stats.lstats[(size >> PAGE_SHIFT) - 1].highruns = - arena->stats.lstats[(size >> PAGE_SHIFT) - - 1].curruns; - } -#endif - malloc_mutex_unlock(&arena->lock); - return (false); - } - malloc_mutex_unlock(&arena->lock); - - return (true); -} - -/* - * Try to resize a large allocation, in order to avoid copying. This will - * always fail if growing an object, and the following run is already in use. - */ -static bool -arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, - bool zero) -{ - size_t psize; - - psize = PAGE_CEILING(size + extra); - if (psize == oldsize) { - /* Same size class. */ -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - - size); - } -#endif - return (false); - } else { - arena_chunk_t *chunk; - arena_t *arena; - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - arena = chunk->arena; - dassert(arena->magic == ARENA_MAGIC); - - if (psize < oldsize) { -#ifdef JEMALLOC_FILL - /* Fill before shrinking in order avoid a race. */ - if (opt_junk) { - memset((void *)((uintptr_t)ptr + size), 0x5a, - oldsize - size); - } -#endif - arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, - psize); - return (false); - } else { - bool ret = arena_ralloc_large_grow(arena, chunk, ptr, - oldsize, PAGE_CEILING(size), - psize - PAGE_CEILING(size), zero); -#ifdef JEMALLOC_FILL - if (ret == false && zero == false && opt_zero) { - memset((void *)((uintptr_t)ptr + oldsize), 0, - size - oldsize); - } -#endif - return (ret); - } - } -} - -void * -arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, - bool zero) -{ - - /* - * Avoid moving the allocation if the size class can be left the same. - */ - if (oldsize <= arena_maxclass) { - if (oldsize <= small_maxclass) { - assert(arena_bin_info[SMALL_SIZE2BIN(oldsize)].reg_size - == oldsize); - if ((size + extra <= small_maxclass && - SMALL_SIZE2BIN(size + extra) == - SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && - size + extra >= oldsize)) { -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), - 0x5a, oldsize - size); - } -#endif - return (ptr); - } - } else { - assert(size <= arena_maxclass); - if (size + extra > small_maxclass) { - if (arena_ralloc_large(ptr, oldsize, size, - extra, zero) == false) - return (ptr); - } - } - } - - /* Reallocation would require a move. */ - return (NULL); -} - -void * -arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) -{ - void *ret; - size_t copysize; - - /* Try to avoid moving the allocation. */ - ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); - if (ret != NULL) - return (ret); - - /* - * size and oldsize are different enough that we need to move the - * object. In that case, fall back to allocating new space and - * copying. - */ - if (alignment != 0) { - size_t usize = sa2u(size + extra, alignment, NULL); - if (usize == 0) - return (NULL); - ret = ipalloc(usize, alignment, zero); - } else - ret = arena_malloc(size + extra, zero); - - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, this time without extra. */ - if (alignment != 0) { - size_t usize = sa2u(size, alignment, NULL); - if (usize == 0) - return (NULL); - ret = ipalloc(usize, alignment, zero); - } else - ret = arena_malloc(size, zero); - - if (ret == NULL) - return (NULL); - } - - /* Junk/zero-filling were already done by ipalloc()/arena_malloc(). */ - - /* - * Copy at most size bytes (not size+extra), since the caller has no - * expectation that the extra bytes will be reliably preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - memcpy(ret, ptr, copysize); - idalloc(ptr); - return (ret); -} - -bool -arena_new(arena_t *arena, unsigned ind) -{ - unsigned i; - arena_bin_t *bin; - - arena->ind = ind; - arena->nthreads = 0; - - if (malloc_mutex_init(&arena->lock)) - return (true); - -#ifdef JEMALLOC_STATS - memset(&arena->stats, 0, sizeof(arena_stats_t)); - arena->stats.lstats = (malloc_large_stats_t *)base_alloc(nlclasses * - sizeof(malloc_large_stats_t)); - if (arena->stats.lstats == NULL) - return (true); - memset(arena->stats.lstats, 0, nlclasses * - sizeof(malloc_large_stats_t)); -# ifdef JEMALLOC_TCACHE - ql_new(&arena->tcache_ql); -# endif -#endif - -#ifdef JEMALLOC_PROF - arena->prof_accumbytes = 0; -#endif - - /* Initialize chunks. */ - ql_new(&arena->chunks_dirty); - arena->spare = NULL; - - arena->nactive = 0; - arena->ndirty = 0; - arena->npurgatory = 0; - - arena_avail_tree_new(&arena->runs_avail_clean); - arena_avail_tree_new(&arena->runs_avail_dirty); - - /* Initialize bins. */ - i = 0; -#ifdef JEMALLOC_TINY - /* (2^n)-spaced tiny bins. */ - for (; i < ntbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } -#endif - - /* Quantum-spaced bins. */ - for (; i < ntbins + nqbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - - /* Cacheline-spaced bins. */ - for (; i < ntbins + nqbins + ncbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - - /* Subpage-spaced bins. */ - for (; i < nbins; i++) { - bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock)) - return (true); - bin->runcur = NULL; - arena_run_tree_new(&bin->runs); -#ifdef JEMALLOC_STATS - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); -#endif - } - -#ifdef JEMALLOC_DEBUG - arena->magic = ARENA_MAGIC; -#endif - - return (false); -} - -#ifdef JEMALLOC_DEBUG -static void -small_size2bin_validate(void) -{ - size_t i, size, binind; - - i = 1; -# ifdef JEMALLOC_TINY - /* Tiny. */ - for (; i < (1U << LG_TINY_MIN); i++) { - size = pow2_ceil(1U << LG_TINY_MIN); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - assert(SMALL_SIZE2BIN(i) == binind); - } - for (; i < qspace_min; i++) { - size = pow2_ceil(i); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - assert(SMALL_SIZE2BIN(i) == binind); - } -# endif - /* Quantum-spaced. */ - for (; i <= qspace_max; i++) { - size = QUANTUM_CEILING(i); - binind = ntbins + (size >> LG_QUANTUM) - 1; - assert(SMALL_SIZE2BIN(i) == binind); - } - /* Cacheline-spaced. */ - for (; i <= cspace_max; i++) { - size = CACHELINE_CEILING(i); - binind = ntbins + nqbins + ((size - cspace_min) >> - LG_CACHELINE); - assert(SMALL_SIZE2BIN(i) == binind); - } - /* Sub-page. */ - for (; i <= sspace_max; i++) { - size = SUBPAGE_CEILING(i); - binind = ntbins + nqbins + ncbins + ((size - sspace_min) - >> LG_SUBPAGE); - assert(SMALL_SIZE2BIN(i) == binind); - } -} -#endif - -static bool -small_size2bin_init(void) -{ - - if (opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT - || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT - || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> - LG_TINY_MIN) + 1)) - return (small_size2bin_init_hard()); - - small_size2bin = const_small_size2bin; -#ifdef JEMALLOC_DEBUG - small_size2bin_validate(); -#endif - return (false); -} - -static bool -small_size2bin_init_hard(void) -{ - size_t i, size, binind; - uint8_t *custom_small_size2bin; -#define CUSTOM_SMALL_SIZE2BIN(s) \ - custom_small_size2bin[(s-1) >> LG_TINY_MIN] - - assert(opt_lg_qspace_max != LG_QSPACE_MAX_DEFAULT - || opt_lg_cspace_max != LG_CSPACE_MAX_DEFAULT - || (sizeof(const_small_size2bin) != ((small_maxclass-1) >> - LG_TINY_MIN) + 1)); - - custom_small_size2bin = (uint8_t *) - base_alloc(small_maxclass >> LG_TINY_MIN); - if (custom_small_size2bin == NULL) - return (true); - - i = 1; -#ifdef JEMALLOC_TINY - /* Tiny. */ - for (; i < (1U << LG_TINY_MIN); i += TINY_MIN) { - size = pow2_ceil(1U << LG_TINY_MIN); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - for (; i < qspace_min; i += TINY_MIN) { - size = pow2_ceil(i); - binind = ffs((int)(size >> (LG_TINY_MIN + 1))); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } -#endif - /* Quantum-spaced. */ - for (; i <= qspace_max; i += TINY_MIN) { - size = QUANTUM_CEILING(i); - binind = ntbins + (size >> LG_QUANTUM) - 1; - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - /* Cacheline-spaced. */ - for (; i <= cspace_max; i += TINY_MIN) { - size = CACHELINE_CEILING(i); - binind = ntbins + nqbins + ((size - cspace_min) >> - LG_CACHELINE); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - /* Sub-page. */ - for (; i <= sspace_max; i += TINY_MIN) { - size = SUBPAGE_CEILING(i); - binind = ntbins + nqbins + ncbins + ((size - sspace_min) >> - LG_SUBPAGE); - CUSTOM_SMALL_SIZE2BIN(i) = binind; - } - - small_size2bin = custom_small_size2bin; -#ifdef JEMALLOC_DEBUG - small_size2bin_validate(); -#endif - return (false); -#undef CUSTOM_SMALL_SIZE2BIN -} - -/* - * Calculate bin_info->run_size such that it meets the following constraints: - * - * *) bin_info->run_size >= min_run_size - * *) bin_info->run_size <= arena_maxclass - * *) run header overhead <= RUN_MAX_OVRHD (or header overhead relaxed). - * *) bin_info->nregs <= RUN_MAXREGS - * - * bin_info->nregs, bin_info->bitmap_offset, and bin_info->reg0_offset are also - * calculated here, since these settings are all interdependent. - */ -static size_t -bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) -{ - size_t try_run_size, good_run_size; - uint32_t try_nregs, good_nregs; - uint32_t try_hdr_size, good_hdr_size; - uint32_t try_bitmap_offset, good_bitmap_offset; -#ifdef JEMALLOC_PROF - uint32_t try_ctx0_offset, good_ctx0_offset; -#endif - uint32_t try_reg0_offset, good_reg0_offset; - - assert(min_run_size >= PAGE_SIZE); - assert(min_run_size <= arena_maxclass); - - /* - * Calculate known-valid settings before entering the run_size - * expansion loop, so that the first part of the loop always copies - * valid settings. - * - * The do..while loop iteratively reduces the number of regions until - * the run header and the regions no longer overlap. A closed formula - * would be quite messy, since there is an interdependency between the - * header's mask length and the number of regions. - */ - try_run_size = min_run_size; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / bin_info->reg_size) - + 1; /* Counter-act try_nregs-- in loop. */ - if (try_nregs > RUN_MAXREGS) { - try_nregs = RUN_MAXREGS - + 1; /* Counter-act try_nregs-- in loop. */ - } - do { - try_nregs--; - try_hdr_size = sizeof(arena_run_t); - /* Pad to a long boundary. */ - try_hdr_size = LONG_CEILING(try_hdr_size); - try_bitmap_offset = try_hdr_size; - /* Add space for bitmap. */ - try_hdr_size += bitmap_size(try_nregs); -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote == false) { - /* Pad to a quantum boundary. */ - try_hdr_size = QUANTUM_CEILING(try_hdr_size); - try_ctx0_offset = try_hdr_size; - /* Add space for one (prof_ctx_t *) per region. */ - try_hdr_size += try_nregs * sizeof(prof_ctx_t *); - } else - try_ctx0_offset = 0; -#endif - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); - - /* run_size expansion loop. */ - do { - /* - * Copy valid settings before trying more aggressive settings. - */ - good_run_size = try_run_size; - good_nregs = try_nregs; - good_hdr_size = try_hdr_size; - good_bitmap_offset = try_bitmap_offset; -#ifdef JEMALLOC_PROF - good_ctx0_offset = try_ctx0_offset; -#endif - good_reg0_offset = try_reg0_offset; - - /* Try more aggressive settings. */ - try_run_size += PAGE_SIZE; - try_nregs = ((try_run_size - sizeof(arena_run_t)) / - bin_info->reg_size) - + 1; /* Counter-act try_nregs-- in loop. */ - if (try_nregs > RUN_MAXREGS) { - try_nregs = RUN_MAXREGS - + 1; /* Counter-act try_nregs-- in loop. */ - } - do { - try_nregs--; - try_hdr_size = sizeof(arena_run_t); - /* Pad to a long boundary. */ - try_hdr_size = LONG_CEILING(try_hdr_size); - try_bitmap_offset = try_hdr_size; - /* Add space for bitmap. */ - try_hdr_size += bitmap_size(try_nregs); -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote == false) { - /* Pad to a quantum boundary. */ - try_hdr_size = QUANTUM_CEILING(try_hdr_size); - try_ctx0_offset = try_hdr_size; - /* - * Add space for one (prof_ctx_t *) per region. - */ - try_hdr_size += try_nregs * - sizeof(prof_ctx_t *); - } -#endif - try_reg0_offset = try_run_size - (try_nregs * - bin_info->reg_size); - } while (try_hdr_size > try_reg0_offset); - } while (try_run_size <= arena_maxclass - && try_run_size <= arena_maxclass - && RUN_MAX_OVRHD * (bin_info->reg_size << 3) > RUN_MAX_OVRHD_RELAX - && (try_reg0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size - && try_nregs < RUN_MAXREGS); - - assert(good_hdr_size <= good_reg0_offset); - - /* Copy final settings. */ - bin_info->run_size = good_run_size; - bin_info->nregs = good_nregs; - bin_info->bitmap_offset = good_bitmap_offset; -#ifdef JEMALLOC_PROF - bin_info->ctx0_offset = good_ctx0_offset; -#endif - bin_info->reg0_offset = good_reg0_offset; - - return (good_run_size); -} - -static bool -bin_info_init(void) -{ - arena_bin_info_t *bin_info; - unsigned i; - size_t prev_run_size; - - arena_bin_info = base_alloc(sizeof(arena_bin_info_t) * nbins); - if (arena_bin_info == NULL) - return (true); - - prev_run_size = PAGE_SIZE; - i = 0; -#ifdef JEMALLOC_TINY - /* (2^n)-spaced tiny bins. */ - for (; i < ntbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = (1U << (LG_TINY_MIN + i)); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } -#endif - - /* Quantum-spaced bins. */ - for (; i < ntbins + nqbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = (i - ntbins + 1) << LG_QUANTUM; - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - /* Cacheline-spaced bins. */ - for (; i < ntbins + nqbins + ncbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = cspace_min + ((i - (ntbins + nqbins)) << - LG_CACHELINE); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - /* Subpage-spaced bins. */ - for (; i < nbins; i++) { - bin_info = &arena_bin_info[i]; - bin_info->reg_size = sspace_min + ((i - (ntbins + nqbins + - ncbins)) << LG_SUBPAGE); - prev_run_size = bin_info_run_size_calc(bin_info, prev_run_size); - bitmap_info_init(&bin_info->bitmap_info, bin_info->nregs); - } - - return (false); -} - -bool -arena_boot(void) -{ - size_t header_size; - unsigned i; - - /* Set variables according to the value of opt_lg_[qc]space_max. */ - qspace_max = (1U << opt_lg_qspace_max); - cspace_min = CACHELINE_CEILING(qspace_max); - if (cspace_min == qspace_max) - cspace_min += CACHELINE; - cspace_max = (1U << opt_lg_cspace_max); - sspace_min = SUBPAGE_CEILING(cspace_max); - if (sspace_min == cspace_max) - sspace_min += SUBPAGE; - assert(sspace_min < PAGE_SIZE); - sspace_max = PAGE_SIZE - SUBPAGE; - -#ifdef JEMALLOC_TINY - assert(LG_QUANTUM >= LG_TINY_MIN); -#endif - assert(ntbins <= LG_QUANTUM); - nqbins = qspace_max >> LG_QUANTUM; - ncbins = ((cspace_max - cspace_min) >> LG_CACHELINE) + 1; - nsbins = ((sspace_max - sspace_min) >> LG_SUBPAGE) + 1; - nbins = ntbins + nqbins + ncbins + nsbins; - - /* - * The small_size2bin lookup table uses uint8_t to encode each bin - * index, so we cannot support more than 256 small size classes. This - * limit is difficult to exceed (not even possible with 16B quantum and - * 4KiB pages), and such configurations are impractical, but - * nonetheless we need to protect against this case in order to avoid - * undefined behavior. - * - * Further constrain nbins to 255 if prof_promote is true, since all - * small size classes, plus a "not small" size class must be stored in - * 8 bits of arena_chunk_map_t's bits field. - */ -#ifdef JEMALLOC_PROF - if (opt_prof && prof_promote) { - if (nbins > 255) { - char line_buf[UMAX2S_BUFSIZE]; - malloc_write(": Too many small size classes ("); - malloc_write(u2s(nbins, 10, line_buf)); - malloc_write(" > max 255)\n"); - abort(); - } - } else -#endif - if (nbins > 256) { - char line_buf[UMAX2S_BUFSIZE]; - malloc_write(": Too many small size classes ("); - malloc_write(u2s(nbins, 10, line_buf)); - malloc_write(" > max 256)\n"); - abort(); - } - - /* - * Compute the header size such that it is large enough to contain the - * page map. The page map is biased to omit entries for the header - * itself, so some iteration is necessary to compute the map bias. - * - * 1) Compute safe header_size and map_bias values that include enough - * space for an unbiased page map. - * 2) Refine map_bias based on (1) to omit the header pages in the page - * map. The resulting map_bias may be one too small. - * 3) Refine map_bias based on (2). The result will be >= the result - * from (2), and will always be correct. - */ - map_bias = 0; - for (i = 0; i < 3; i++) { - header_size = offsetof(arena_chunk_t, map) - + (sizeof(arena_chunk_map_t) * (chunk_npages-map_bias)); - map_bias = (header_size >> PAGE_SHIFT) + ((header_size & - PAGE_MASK) != 0); - } - assert(map_bias > 0); - - arena_maxclass = chunksize - (map_bias << PAGE_SHIFT); - - if (small_size2bin_init()) - return (true); - - if (bin_info_init()) - return (true); - - return (false); -} diff --git a/deps/jemalloc.orig/src/atomic.c b/deps/jemalloc.orig/src/atomic.c deleted file mode 100644 index 77ee313113b..00000000000 --- a/deps/jemalloc.orig/src/atomic.c +++ /dev/null @@ -1,2 +0,0 @@ -#define JEMALLOC_ATOMIC_C_ -#include "jemalloc/internal/jemalloc_internal.h" diff --git a/deps/jemalloc.orig/src/base.c b/deps/jemalloc.orig/src/base.c deleted file mode 100644 index cc85e8494ec..00000000000 --- a/deps/jemalloc.orig/src/base.c +++ /dev/null @@ -1,106 +0,0 @@ -#define JEMALLOC_BASE_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -malloc_mutex_t base_mtx; - -/* - * Current pages that are being used for internal memory allocations. These - * pages are carved up in cacheline-size quanta, so that there is no chance of - * false cache line sharing. - */ -static void *base_pages; -static void *base_next_addr; -static void *base_past_addr; /* Addr immediately past base_pages. */ -static extent_node_t *base_nodes; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static bool base_pages_alloc(size_t minsize); - -/******************************************************************************/ - -static bool -base_pages_alloc(size_t minsize) -{ - size_t csize; - bool zero; - - assert(minsize != 0); - csize = CHUNK_CEILING(minsize); - zero = false; - base_pages = chunk_alloc(csize, true, &zero); - if (base_pages == NULL) - return (true); - base_next_addr = base_pages; - base_past_addr = (void *)((uintptr_t)base_pages + csize); - - return (false); -} - -void * -base_alloc(size_t size) -{ - void *ret; - size_t csize; - - /* Round size up to nearest multiple of the cacheline size. */ - csize = CACHELINE_CEILING(size); - - malloc_mutex_lock(&base_mtx); - /* Make sure there's enough space for the allocation. */ - if ((uintptr_t)base_next_addr + csize > (uintptr_t)base_past_addr) { - if (base_pages_alloc(csize)) { - malloc_mutex_unlock(&base_mtx); - return (NULL); - } - } - /* Allocate. */ - ret = base_next_addr; - base_next_addr = (void *)((uintptr_t)base_next_addr + csize); - malloc_mutex_unlock(&base_mtx); - - return (ret); -} - -extent_node_t * -base_node_alloc(void) -{ - extent_node_t *ret; - - malloc_mutex_lock(&base_mtx); - if (base_nodes != NULL) { - ret = base_nodes; - base_nodes = *(extent_node_t **)ret; - malloc_mutex_unlock(&base_mtx); - } else { - malloc_mutex_unlock(&base_mtx); - ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); - } - - return (ret); -} - -void -base_node_dealloc(extent_node_t *node) -{ - - malloc_mutex_lock(&base_mtx); - *(extent_node_t **)node = base_nodes; - base_nodes = node; - malloc_mutex_unlock(&base_mtx); -} - -bool -base_boot(void) -{ - - base_nodes = NULL; - if (malloc_mutex_init(&base_mtx)) - return (true); - - return (false); -} diff --git a/deps/jemalloc.orig/src/bitmap.c b/deps/jemalloc.orig/src/bitmap.c deleted file mode 100644 index b47e2629093..00000000000 --- a/deps/jemalloc.orig/src/bitmap.c +++ /dev/null @@ -1,90 +0,0 @@ -#define JEMALLOC_BITMAP_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static size_t bits2groups(size_t nbits); - -/******************************************************************************/ - -static size_t -bits2groups(size_t nbits) -{ - - return ((nbits >> LG_BITMAP_GROUP_NBITS) + - !!(nbits & BITMAP_GROUP_NBITS_MASK)); -} - -void -bitmap_info_init(bitmap_info_t *binfo, size_t nbits) -{ - unsigned i; - size_t group_count; - - assert(nbits > 0); - assert(nbits <= (ZU(1) << LG_BITMAP_MAXBITS)); - - /* - * Compute the number of groups necessary to store nbits bits, and - * progressively work upward through the levels until reaching a level - * that requires only one group. - */ - binfo->levels[0].group_offset = 0; - group_count = bits2groups(nbits); - for (i = 1; group_count > 1; i++) { - assert(i < BITMAP_MAX_LEVELS); - binfo->levels[i].group_offset = binfo->levels[i-1].group_offset - + group_count; - group_count = bits2groups(group_count); - } - binfo->levels[i].group_offset = binfo->levels[i-1].group_offset - + group_count; - binfo->nlevels = i; - binfo->nbits = nbits; -} - -size_t -bitmap_info_ngroups(const bitmap_info_t *binfo) -{ - - return (binfo->levels[binfo->nlevels].group_offset << LG_SIZEOF_BITMAP); -} - -size_t -bitmap_size(size_t nbits) -{ - bitmap_info_t binfo; - - bitmap_info_init(&binfo, nbits); - return (bitmap_info_ngroups(&binfo)); -} - -void -bitmap_init(bitmap_t *bitmap, const bitmap_info_t *binfo) -{ - size_t extra; - unsigned i; - - /* - * Bits are actually inverted with regard to the external bitmap - * interface, so the bitmap starts out with all 1 bits, except for - * trailing unused bits (if any). Note that each group uses bit 0 to - * correspond to the first logical bit in the group, so extra bits - * are the most significant bits of the last group. - */ - memset(bitmap, 0xffU, binfo->levels[binfo->nlevels].group_offset << - LG_SIZEOF_BITMAP); - extra = (BITMAP_GROUP_NBITS - (binfo->nbits & BITMAP_GROUP_NBITS_MASK)) - & BITMAP_GROUP_NBITS_MASK; - if (extra != 0) - bitmap[binfo->levels[1].group_offset - 1] >>= extra; - for (i = 1; i < binfo->nlevels; i++) { - size_t group_count = binfo->levels[i].group_offset - - binfo->levels[i-1].group_offset; - extra = (BITMAP_GROUP_NBITS - (group_count & - BITMAP_GROUP_NBITS_MASK)) & BITMAP_GROUP_NBITS_MASK; - if (extra != 0) - bitmap[binfo->levels[i+1].group_offset - 1] >>= extra; - } -} diff --git a/deps/jemalloc.orig/src/chunk.c b/deps/jemalloc.orig/src/chunk.c deleted file mode 100644 index d190c6f49b3..00000000000 --- a/deps/jemalloc.orig/src/chunk.c +++ /dev/null @@ -1,173 +0,0 @@ -#define JEMALLOC_CHUNK_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -size_t opt_lg_chunk = LG_CHUNK_DEFAULT; -#ifdef JEMALLOC_SWAP -bool opt_overcommit = true; -#endif - -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) -malloc_mutex_t chunks_mtx; -chunk_stats_t stats_chunks; -#endif - -#ifdef JEMALLOC_IVSALLOC -rtree_t *chunks_rtree; -#endif - -/* Various chunk-related settings. */ -size_t chunksize; -size_t chunksize_mask; /* (chunksize - 1). */ -size_t chunk_npages; -size_t map_bias; -size_t arena_maxclass; /* Max size class for arenas. */ - -/******************************************************************************/ - -/* - * If the caller specifies (*zero == false), it is still possible to receive - * zeroed memory, in which case *zero is toggled to true. arena_chunk_alloc() - * takes advantage of this to avoid demanding zeroed chunks, but taking - * advantage of them if they are returned. - */ -void * -chunk_alloc(size_t size, bool base, bool *zero) -{ - void *ret; - - assert(size != 0); - assert((size & chunksize_mask) == 0); - -#ifdef JEMALLOC_SWAP - if (swap_enabled) { - ret = chunk_alloc_swap(size, zero); - if (ret != NULL) - goto RETURN; - } - - if (swap_enabled == false || opt_overcommit) { -#endif -#ifdef JEMALLOC_DSS - ret = chunk_alloc_dss(size, zero); - if (ret != NULL) - goto RETURN; -#endif - ret = chunk_alloc_mmap(size); - if (ret != NULL) { - *zero = true; - goto RETURN; - } -#ifdef JEMALLOC_SWAP - } -#endif - - /* All strategies for allocation failed. */ - ret = NULL; -RETURN: -#ifdef JEMALLOC_IVSALLOC - if (base == false && ret != NULL) { - if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) { - chunk_dealloc(ret, size, true); - return (NULL); - } - } -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - if (ret != NULL) { -# ifdef JEMALLOC_PROF - bool gdump; -# endif - malloc_mutex_lock(&chunks_mtx); -# ifdef JEMALLOC_STATS - stats_chunks.nchunks += (size / chunksize); -# endif - stats_chunks.curchunks += (size / chunksize); - if (stats_chunks.curchunks > stats_chunks.highchunks) { - stats_chunks.highchunks = stats_chunks.curchunks; -# ifdef JEMALLOC_PROF - gdump = true; -# endif - } -# ifdef JEMALLOC_PROF - else - gdump = false; -# endif - malloc_mutex_unlock(&chunks_mtx); -# ifdef JEMALLOC_PROF - if (opt_prof && opt_prof_gdump && gdump) - prof_gdump(); -# endif - } -#endif - - assert(CHUNK_ADDR2BASE(ret) == ret); - return (ret); -} - -void -chunk_dealloc(void *chunk, size_t size, bool unmap) -{ - - assert(chunk != NULL); - assert(CHUNK_ADDR2BASE(chunk) == chunk); - assert(size != 0); - assert((size & chunksize_mask) == 0); - -#ifdef JEMALLOC_IVSALLOC - rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); -#endif -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - malloc_mutex_lock(&chunks_mtx); - stats_chunks.curchunks -= (size / chunksize); - malloc_mutex_unlock(&chunks_mtx); -#endif - - if (unmap) { -#ifdef JEMALLOC_SWAP - if (swap_enabled && chunk_dealloc_swap(chunk, size) == false) - return; -#endif -#ifdef JEMALLOC_DSS - if (chunk_dealloc_dss(chunk, size) == false) - return; -#endif - chunk_dealloc_mmap(chunk, size); - } -} - -bool -chunk_boot(void) -{ - - /* Set variables according to the value of opt_lg_chunk. */ - chunksize = (ZU(1) << opt_lg_chunk); - assert(chunksize >= PAGE_SIZE); - chunksize_mask = chunksize - 1; - chunk_npages = (chunksize >> PAGE_SHIFT); - -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - if (malloc_mutex_init(&chunks_mtx)) - return (true); - memset(&stats_chunks, 0, sizeof(chunk_stats_t)); -#endif -#ifdef JEMALLOC_SWAP - if (chunk_swap_boot()) - return (true); -#endif - if (chunk_mmap_boot()) - return (true); -#ifdef JEMALLOC_DSS - if (chunk_dss_boot()) - return (true); -#endif -#ifdef JEMALLOC_IVSALLOC - chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk); - if (chunks_rtree == NULL) - return (true); -#endif - - return (false); -} diff --git a/deps/jemalloc.orig/src/chunk_dss.c b/deps/jemalloc.orig/src/chunk_dss.c deleted file mode 100644 index 5c0e290e441..00000000000 --- a/deps/jemalloc.orig/src/chunk_dss.c +++ /dev/null @@ -1,284 +0,0 @@ -#define JEMALLOC_CHUNK_DSS_C_ -#include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_DSS -/******************************************************************************/ -/* Data. */ - -malloc_mutex_t dss_mtx; - -/* Base address of the DSS. */ -static void *dss_base; -/* Current end of the DSS, or ((void *)-1) if the DSS is exhausted. */ -static void *dss_prev; -/* Current upper limit on DSS addresses. */ -static void *dss_max; - -/* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. - */ -static extent_tree_t dss_chunks_szad; -static extent_tree_t dss_chunks_ad; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void *chunk_recycle_dss(size_t size, bool *zero); -static extent_node_t *chunk_dealloc_dss_record(void *chunk, size_t size); - -/******************************************************************************/ - -static void * -chunk_recycle_dss(size_t size, bool *zero) -{ - extent_node_t *node, key; - - key.addr = NULL; - key.size = size; - malloc_mutex_lock(&dss_mtx); - node = extent_tree_szad_nsearch(&dss_chunks_szad, &key); - if (node != NULL) { - void *ret = node->addr; - - /* Remove node from the tree. */ - extent_tree_szad_remove(&dss_chunks_szad, node); - if (node->size == size) { - extent_tree_ad_remove(&dss_chunks_ad, node); - base_node_dealloc(node); - } else { - /* - * Insert the remainder of node's address range as a - * smaller chunk. Its position within dss_chunks_ad - * does not change. - */ - assert(node->size > size); - node->addr = (void *)((uintptr_t)node->addr + size); - node->size -= size; - extent_tree_szad_insert(&dss_chunks_szad, node); - } - malloc_mutex_unlock(&dss_mtx); - - if (*zero) - memset(ret, 0, size); - return (ret); - } - malloc_mutex_unlock(&dss_mtx); - - return (NULL); -} - -void * -chunk_alloc_dss(size_t size, bool *zero) -{ - void *ret; - - ret = chunk_recycle_dss(size, zero); - if (ret != NULL) - return (ret); - - /* - * sbrk() uses a signed increment argument, so take care not to - * interpret a huge allocation request as a negative increment. - */ - if ((intptr_t)size < 0) - return (NULL); - - malloc_mutex_lock(&dss_mtx); - if (dss_prev != (void *)-1) { - intptr_t incr; - - /* - * The loop is necessary to recover from races with other - * threads that are using the DSS for something other than - * malloc. - */ - do { - /* Get the current end of the DSS. */ - dss_max = sbrk(0); - - /* - * Calculate how much padding is necessary to - * chunk-align the end of the DSS. - */ - incr = (intptr_t)size - - (intptr_t)CHUNK_ADDR2OFFSET(dss_max); - if (incr == (intptr_t)size) - ret = dss_max; - else { - ret = (void *)((intptr_t)dss_max + incr); - incr += size; - } - - dss_prev = sbrk(incr); - if (dss_prev == dss_max) { - /* Success. */ - dss_max = (void *)((intptr_t)dss_prev + incr); - malloc_mutex_unlock(&dss_mtx); - *zero = true; - return (ret); - } - } while (dss_prev != (void *)-1); - } - malloc_mutex_unlock(&dss_mtx); - - return (NULL); -} - -static extent_node_t * -chunk_dealloc_dss_record(void *chunk, size_t size) -{ - extent_node_t *xnode, *node, *prev, key; - - xnode = NULL; - while (true) { - key.addr = (void *)((uintptr_t)chunk + size); - node = extent_tree_ad_nsearch(&dss_chunks_ad, &key); - /* Try to coalesce forward. */ - if (node != NULL && node->addr == key.addr) { - /* - * Coalesce chunk with the following address range. - * This does not change the position within - * dss_chunks_ad, so only remove/insert from/into - * dss_chunks_szad. - */ - extent_tree_szad_remove(&dss_chunks_szad, node); - node->addr = chunk; - node->size += size; - extent_tree_szad_insert(&dss_chunks_szad, node); - break; - } else if (xnode == NULL) { - /* - * It is possible that base_node_alloc() will cause a - * new base chunk to be allocated, so take care not to - * deadlock on dss_mtx, and recover if another thread - * deallocates an adjacent chunk while this one is busy - * allocating xnode. - */ - malloc_mutex_unlock(&dss_mtx); - xnode = base_node_alloc(); - malloc_mutex_lock(&dss_mtx); - if (xnode == NULL) - return (NULL); - } else { - /* Coalescing forward failed, so insert a new node. */ - node = xnode; - xnode = NULL; - node->addr = chunk; - node->size = size; - extent_tree_ad_insert(&dss_chunks_ad, node); - extent_tree_szad_insert(&dss_chunks_szad, node); - break; - } - } - /* Discard xnode if it ended up unused do to a race. */ - if (xnode != NULL) - base_node_dealloc(xnode); - - /* Try to coalesce backward. */ - prev = extent_tree_ad_prev(&dss_chunks_ad, node); - if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == - chunk) { - /* - * Coalesce chunk with the previous address range. This does - * not change the position within dss_chunks_ad, so only - * remove/insert node from/into dss_chunks_szad. - */ - extent_tree_szad_remove(&dss_chunks_szad, prev); - extent_tree_ad_remove(&dss_chunks_ad, prev); - - extent_tree_szad_remove(&dss_chunks_szad, node); - node->addr = prev->addr; - node->size += prev->size; - extent_tree_szad_insert(&dss_chunks_szad, node); - - base_node_dealloc(prev); - } - - return (node); -} - -bool -chunk_in_dss(void *chunk) -{ - bool ret; - - malloc_mutex_lock(&dss_mtx); - if ((uintptr_t)chunk >= (uintptr_t)dss_base - && (uintptr_t)chunk < (uintptr_t)dss_max) - ret = true; - else - ret = false; - malloc_mutex_unlock(&dss_mtx); - - return (ret); -} - -bool -chunk_dealloc_dss(void *chunk, size_t size) -{ - bool ret; - - malloc_mutex_lock(&dss_mtx); - if ((uintptr_t)chunk >= (uintptr_t)dss_base - && (uintptr_t)chunk < (uintptr_t)dss_max) { - extent_node_t *node; - - /* Try to coalesce with other unused chunks. */ - node = chunk_dealloc_dss_record(chunk, size); - if (node != NULL) { - chunk = node->addr; - size = node->size; - } - - /* Get the current end of the DSS. */ - dss_max = sbrk(0); - - /* - * Try to shrink the DSS if this chunk is at the end of the - * DSS. The sbrk() call here is subject to a race condition - * with threads that use brk(2) or sbrk(2) directly, but the - * alternative would be to leak memory for the sake of poorly - * designed multi-threaded programs. - */ - if ((void *)((uintptr_t)chunk + size) == dss_max - && (dss_prev = sbrk(-(intptr_t)size)) == dss_max) { - /* Success. */ - dss_max = (void *)((intptr_t)dss_prev - (intptr_t)size); - - if (node != NULL) { - extent_tree_szad_remove(&dss_chunks_szad, node); - extent_tree_ad_remove(&dss_chunks_ad, node); - base_node_dealloc(node); - } - } else - madvise(chunk, size, MADV_DONTNEED); - - ret = false; - goto RETURN; - } - - ret = true; -RETURN: - malloc_mutex_unlock(&dss_mtx); - return (ret); -} - -bool -chunk_dss_boot(void) -{ - - if (malloc_mutex_init(&dss_mtx)) - return (true); - dss_base = sbrk(0); - dss_prev = dss_base; - dss_max = dss_base; - extent_tree_szad_new(&dss_chunks_szad); - extent_tree_ad_new(&dss_chunks_ad); - - return (false); -} - -/******************************************************************************/ -#endif /* JEMALLOC_DSS */ diff --git a/deps/jemalloc.orig/src/chunk_mmap.c b/deps/jemalloc.orig/src/chunk_mmap.c deleted file mode 100644 index 164e86e7b38..00000000000 --- a/deps/jemalloc.orig/src/chunk_mmap.c +++ /dev/null @@ -1,239 +0,0 @@ -#define JEMALLOC_CHUNK_MMAP_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -/* - * Used by chunk_alloc_mmap() to decide whether to attempt the fast path and - * potentially avoid some system calls. - */ -#ifndef NO_TLS -static __thread bool mmap_unaligned_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -#define MMAP_UNALIGNED_GET() mmap_unaligned_tls -#define MMAP_UNALIGNED_SET(v) do { \ - mmap_unaligned_tls = (v); \ -} while (0) -#else -static pthread_key_t mmap_unaligned_tsd; -#define MMAP_UNALIGNED_GET() ((bool)pthread_getspecific(mmap_unaligned_tsd)) -#define MMAP_UNALIGNED_SET(v) do { \ - pthread_setspecific(mmap_unaligned_tsd, (void *)(v)); \ -} while (0) -#endif - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void *pages_map(void *addr, size_t size, bool noreserve); -static void pages_unmap(void *addr, size_t size); -static void *chunk_alloc_mmap_slow(size_t size, bool unaligned, - bool noreserve); -static void *chunk_alloc_mmap_internal(size_t size, bool noreserve); - -/******************************************************************************/ - -static void * -pages_map(void *addr, size_t size, bool noreserve) -{ - void *ret; - - /* - * We don't use MAP_FIXED here, because it can cause the *replacement* - * of existing mappings, and we only want to create new mappings. - */ - int flags = MAP_PRIVATE | MAP_ANON; -#ifdef MAP_NORESERVE - if (noreserve) - flags |= MAP_NORESERVE; -#endif - ret = mmap(addr, size, PROT_READ | PROT_WRITE, flags, -1, 0); - assert(ret != NULL); - - if (ret == MAP_FAILED) - ret = NULL; - else if (addr != NULL && ret != addr) { - /* - * We succeeded in mapping memory, but not in the right place. - */ - if (munmap(ret, size) == -1) { - char buf[BUFERROR_BUF]; - - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); - if (opt_abort) - abort(); - } - ret = NULL; - } - - assert(ret == NULL || (addr == NULL && ret != addr) - || (addr != NULL && ret == addr)); - return (ret); -} - -static void -pages_unmap(void *addr, size_t size) -{ - - if (munmap(addr, size) == -1) { - char buf[BUFERROR_BUF]; - - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); - if (opt_abort) - abort(); - } -} - -static void * -chunk_alloc_mmap_slow(size_t size, bool unaligned, bool noreserve) -{ - void *ret; - size_t offset; - - /* Beware size_t wrap-around. */ - if (size + chunksize <= size) - return (NULL); - - ret = pages_map(NULL, size + chunksize, noreserve); - if (ret == NULL) - return (NULL); - - /* Clean up unneeded leading/trailing space. */ - offset = CHUNK_ADDR2OFFSET(ret); - if (offset != 0) { - /* Note that mmap() returned an unaligned mapping. */ - unaligned = true; - - /* Leading space. */ - pages_unmap(ret, chunksize - offset); - - ret = (void *)((uintptr_t)ret + - (chunksize - offset)); - - /* Trailing space. */ - pages_unmap((void *)((uintptr_t)ret + size), - offset); - } else { - /* Trailing space only. */ - pages_unmap((void *)((uintptr_t)ret + size), - chunksize); - } - - /* - * If mmap() returned an aligned mapping, reset mmap_unaligned so that - * the next chunk_alloc_mmap() execution tries the fast allocation - * method. - */ - if (unaligned == false) - MMAP_UNALIGNED_SET(false); - - return (ret); -} - -static void * -chunk_alloc_mmap_internal(size_t size, bool noreserve) -{ - void *ret; - - /* - * Ideally, there would be a way to specify alignment to mmap() (like - * NetBSD has), but in the absence of such a feature, we have to work - * hard to efficiently create aligned mappings. The reliable, but - * slow method is to create a mapping that is over-sized, then trim the - * excess. However, that always results in at least one call to - * pages_unmap(). - * - * A more optimistic approach is to try mapping precisely the right - * amount, then try to append another mapping if alignment is off. In - * practice, this works out well as long as the application is not - * interleaving mappings via direct mmap() calls. If we do run into a - * situation where there is an interleaved mapping and we are unable to - * extend an unaligned mapping, our best option is to switch to the - * slow method until mmap() returns another aligned mapping. This will - * tend to leave a gap in the memory map that is too small to cause - * later problems for the optimistic method. - * - * Another possible confounding factor is address space layout - * randomization (ASLR), which causes mmap(2) to disregard the - * requested address. mmap_unaligned tracks whether the previous - * chunk_alloc_mmap() execution received any unaligned or relocated - * mappings, and if so, the current execution will immediately fall - * back to the slow method. However, we keep track of whether the fast - * method would have succeeded, and if so, we make a note to try the - * fast method next time. - */ - - if (MMAP_UNALIGNED_GET() == false) { - size_t offset; - - ret = pages_map(NULL, size, noreserve); - if (ret == NULL) - return (NULL); - - offset = CHUNK_ADDR2OFFSET(ret); - if (offset != 0) { - MMAP_UNALIGNED_SET(true); - /* Try to extend chunk boundary. */ - if (pages_map((void *)((uintptr_t)ret + size), - chunksize - offset, noreserve) == NULL) { - /* - * Extension failed. Clean up, then revert to - * the reliable-but-expensive method. - */ - pages_unmap(ret, size); - ret = chunk_alloc_mmap_slow(size, true, - noreserve); - } else { - /* Clean up unneeded leading space. */ - pages_unmap(ret, chunksize - offset); - ret = (void *)((uintptr_t)ret + (chunksize - - offset)); - } - } - } else - ret = chunk_alloc_mmap_slow(size, false, noreserve); - - return (ret); -} - -void * -chunk_alloc_mmap(size_t size) -{ - - return (chunk_alloc_mmap_internal(size, false)); -} - -void * -chunk_alloc_mmap_noreserve(size_t size) -{ - - return (chunk_alloc_mmap_internal(size, true)); -} - -void -chunk_dealloc_mmap(void *chunk, size_t size) -{ - - pages_unmap(chunk, size); -} - -bool -chunk_mmap_boot(void) -{ - -#ifdef NO_TLS - if (pthread_key_create(&mmap_unaligned_tsd, NULL) != 0) { - malloc_write(": Error in pthread_key_create()\n"); - return (true); - } -#endif - - return (false); -} diff --git a/deps/jemalloc.orig/src/chunk_swap.c b/deps/jemalloc.orig/src/chunk_swap.c deleted file mode 100644 index cb25ae0dde2..00000000000 --- a/deps/jemalloc.orig/src/chunk_swap.c +++ /dev/null @@ -1,402 +0,0 @@ -#define JEMALLOC_CHUNK_SWAP_C_ -#include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_SWAP -/******************************************************************************/ -/* Data. */ - -malloc_mutex_t swap_mtx; -bool swap_enabled; -bool swap_prezeroed; -size_t swap_nfds; -int *swap_fds; -#ifdef JEMALLOC_STATS -size_t swap_avail; -#endif - -/* Base address of the mmap()ed file(s). */ -static void *swap_base; -/* Current end of the space in use (<= swap_max). */ -static void *swap_end; -/* Absolute upper limit on file-backed addresses. */ -static void *swap_max; - -/* - * Trees of chunks that were previously allocated (trees differ only in node - * ordering). These are used when allocating chunks, in an attempt to re-use - * address space. Depending on function, different tree orderings are needed, - * which is why there are two trees with the same contents. - */ -static extent_tree_t swap_chunks_szad; -static extent_tree_t swap_chunks_ad; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void *chunk_recycle_swap(size_t size, bool *zero); -static extent_node_t *chunk_dealloc_swap_record(void *chunk, size_t size); - -/******************************************************************************/ - -static void * -chunk_recycle_swap(size_t size, bool *zero) -{ - extent_node_t *node, key; - - key.addr = NULL; - key.size = size; - malloc_mutex_lock(&swap_mtx); - node = extent_tree_szad_nsearch(&swap_chunks_szad, &key); - if (node != NULL) { - void *ret = node->addr; - - /* Remove node from the tree. */ - extent_tree_szad_remove(&swap_chunks_szad, node); - if (node->size == size) { - extent_tree_ad_remove(&swap_chunks_ad, node); - base_node_dealloc(node); - } else { - /* - * Insert the remainder of node's address range as a - * smaller chunk. Its position within swap_chunks_ad - * does not change. - */ - assert(node->size > size); - node->addr = (void *)((uintptr_t)node->addr + size); - node->size -= size; - extent_tree_szad_insert(&swap_chunks_szad, node); - } -#ifdef JEMALLOC_STATS - swap_avail -= size; -#endif - malloc_mutex_unlock(&swap_mtx); - - if (*zero) - memset(ret, 0, size); - return (ret); - } - malloc_mutex_unlock(&swap_mtx); - - return (NULL); -} - -void * -chunk_alloc_swap(size_t size, bool *zero) -{ - void *ret; - - assert(swap_enabled); - - ret = chunk_recycle_swap(size, zero); - if (ret != NULL) - return (ret); - - malloc_mutex_lock(&swap_mtx); - if ((uintptr_t)swap_end + size <= (uintptr_t)swap_max) { - ret = swap_end; - swap_end = (void *)((uintptr_t)swap_end + size); -#ifdef JEMALLOC_STATS - swap_avail -= size; -#endif - malloc_mutex_unlock(&swap_mtx); - - if (swap_prezeroed) - *zero = true; - else if (*zero) - memset(ret, 0, size); - } else { - malloc_mutex_unlock(&swap_mtx); - return (NULL); - } - - return (ret); -} - -static extent_node_t * -chunk_dealloc_swap_record(void *chunk, size_t size) -{ - extent_node_t *xnode, *node, *prev, key; - - xnode = NULL; - while (true) { - key.addr = (void *)((uintptr_t)chunk + size); - node = extent_tree_ad_nsearch(&swap_chunks_ad, &key); - /* Try to coalesce forward. */ - if (node != NULL && node->addr == key.addr) { - /* - * Coalesce chunk with the following address range. - * This does not change the position within - * swap_chunks_ad, so only remove/insert from/into - * swap_chunks_szad. - */ - extent_tree_szad_remove(&swap_chunks_szad, node); - node->addr = chunk; - node->size += size; - extent_tree_szad_insert(&swap_chunks_szad, node); - break; - } else if (xnode == NULL) { - /* - * It is possible that base_node_alloc() will cause a - * new base chunk to be allocated, so take care not to - * deadlock on swap_mtx, and recover if another thread - * deallocates an adjacent chunk while this one is busy - * allocating xnode. - */ - malloc_mutex_unlock(&swap_mtx); - xnode = base_node_alloc(); - malloc_mutex_lock(&swap_mtx); - if (xnode == NULL) - return (NULL); - } else { - /* Coalescing forward failed, so insert a new node. */ - node = xnode; - xnode = NULL; - node->addr = chunk; - node->size = size; - extent_tree_ad_insert(&swap_chunks_ad, node); - extent_tree_szad_insert(&swap_chunks_szad, node); - break; - } - } - /* Discard xnode if it ended up unused do to a race. */ - if (xnode != NULL) - base_node_dealloc(xnode); - - /* Try to coalesce backward. */ - prev = extent_tree_ad_prev(&swap_chunks_ad, node); - if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == - chunk) { - /* - * Coalesce chunk with the previous address range. This does - * not change the position within swap_chunks_ad, so only - * remove/insert node from/into swap_chunks_szad. - */ - extent_tree_szad_remove(&swap_chunks_szad, prev); - extent_tree_ad_remove(&swap_chunks_ad, prev); - - extent_tree_szad_remove(&swap_chunks_szad, node); - node->addr = prev->addr; - node->size += prev->size; - extent_tree_szad_insert(&swap_chunks_szad, node); - - base_node_dealloc(prev); - } - - return (node); -} - -bool -chunk_in_swap(void *chunk) -{ - bool ret; - - assert(swap_enabled); - - malloc_mutex_lock(&swap_mtx); - if ((uintptr_t)chunk >= (uintptr_t)swap_base - && (uintptr_t)chunk < (uintptr_t)swap_max) - ret = true; - else - ret = false; - malloc_mutex_unlock(&swap_mtx); - - return (ret); -} - -bool -chunk_dealloc_swap(void *chunk, size_t size) -{ - bool ret; - - assert(swap_enabled); - - malloc_mutex_lock(&swap_mtx); - if ((uintptr_t)chunk >= (uintptr_t)swap_base - && (uintptr_t)chunk < (uintptr_t)swap_max) { - extent_node_t *node; - - /* Try to coalesce with other unused chunks. */ - node = chunk_dealloc_swap_record(chunk, size); - if (node != NULL) { - chunk = node->addr; - size = node->size; - } - - /* - * Try to shrink the in-use memory if this chunk is at the end - * of the in-use memory. - */ - if ((void *)((uintptr_t)chunk + size) == swap_end) { - swap_end = (void *)((uintptr_t)swap_end - size); - - if (node != NULL) { - extent_tree_szad_remove(&swap_chunks_szad, - node); - extent_tree_ad_remove(&swap_chunks_ad, node); - base_node_dealloc(node); - } - } else - madvise(chunk, size, MADV_DONTNEED); - -#ifdef JEMALLOC_STATS - swap_avail += size; -#endif - ret = false; - goto RETURN; - } - - ret = true; -RETURN: - malloc_mutex_unlock(&swap_mtx); - return (ret); -} - -bool -chunk_swap_enable(const int *fds, unsigned nfds, bool prezeroed) -{ - bool ret; - unsigned i; - off_t off; - void *vaddr; - size_t cumsize, voff; - size_t sizes[nfds]; - - malloc_mutex_lock(&swap_mtx); - - /* Get file sizes. */ - for (i = 0, cumsize = 0; i < nfds; i++) { - off = lseek(fds[i], 0, SEEK_END); - if (off == ((off_t)-1)) { - ret = true; - goto RETURN; - } - if (PAGE_CEILING(off) != off) { - /* Truncate to a multiple of the page size. */ - off &= ~PAGE_MASK; - if (ftruncate(fds[i], off) != 0) { - ret = true; - goto RETURN; - } - } - sizes[i] = off; - if (cumsize + off < cumsize) { - /* - * Cumulative file size is greater than the total - * address space. Bail out while it's still obvious - * what the problem is. - */ - ret = true; - goto RETURN; - } - cumsize += off; - } - - /* Round down to a multiple of the chunk size. */ - cumsize &= ~chunksize_mask; - if (cumsize == 0) { - ret = true; - goto RETURN; - } - - /* - * Allocate a chunk-aligned region of anonymous memory, which will - * be the final location for the memory-mapped files. - */ - vaddr = chunk_alloc_mmap_noreserve(cumsize); - if (vaddr == NULL) { - ret = true; - goto RETURN; - } - - /* Overlay the files onto the anonymous mapping. */ - for (i = 0, voff = 0; i < nfds; i++) { - void *addr = mmap((void *)((uintptr_t)vaddr + voff), sizes[i], - PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fds[i], 0); - if (addr == MAP_FAILED) { - char buf[BUFERROR_BUF]; - - - buferror(errno, buf, sizeof(buf)); - malloc_write( - ": Error in mmap(..., MAP_FIXED, ...): "); - malloc_write(buf); - malloc_write("\n"); - if (opt_abort) - abort(); - if (munmap(vaddr, voff) == -1) { - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in munmap(): "); - malloc_write(buf); - malloc_write("\n"); - } - ret = true; - goto RETURN; - } - assert(addr == (void *)((uintptr_t)vaddr + voff)); - - /* - * Tell the kernel that the mapping will be accessed randomly, - * and that it should not gratuitously sync pages to the - * filesystem. - */ -#ifdef MADV_RANDOM - madvise(addr, sizes[i], MADV_RANDOM); -#endif -#ifdef MADV_NOSYNC - madvise(addr, sizes[i], MADV_NOSYNC); -#endif - - voff += sizes[i]; - } - - swap_prezeroed = prezeroed; - swap_base = vaddr; - swap_end = swap_base; - swap_max = (void *)((uintptr_t)vaddr + cumsize); - - /* Copy the fds array for mallctl purposes. */ - swap_fds = (int *)base_alloc(nfds * sizeof(int)); - if (swap_fds == NULL) { - ret = true; - goto RETURN; - } - memcpy(swap_fds, fds, nfds * sizeof(int)); - swap_nfds = nfds; - -#ifdef JEMALLOC_STATS - swap_avail = cumsize; -#endif - - swap_enabled = true; - - ret = false; -RETURN: - malloc_mutex_unlock(&swap_mtx); - return (ret); -} - -bool -chunk_swap_boot(void) -{ - - if (malloc_mutex_init(&swap_mtx)) - return (true); - - swap_enabled = false; - swap_prezeroed = false; /* swap.* mallctl's depend on this. */ - swap_nfds = 0; - swap_fds = NULL; -#ifdef JEMALLOC_STATS - swap_avail = 0; -#endif - swap_base = NULL; - swap_end = NULL; - swap_max = NULL; - - extent_tree_szad_new(&swap_chunks_szad); - extent_tree_ad_new(&swap_chunks_ad); - - return (false); -} - -/******************************************************************************/ -#endif /* JEMALLOC_SWAP */ diff --git a/deps/jemalloc.orig/src/ckh.c b/deps/jemalloc.orig/src/ckh.c deleted file mode 100644 index 43fcc25239d..00000000000 --- a/deps/jemalloc.orig/src/ckh.c +++ /dev/null @@ -1,619 +0,0 @@ -/* - ******************************************************************************* - * Implementation of (2^1+,2) cuckoo hashing, where 2^1+ indicates that each - * hash bucket contains 2^n cells, for n >= 1, and 2 indicates that two hash - * functions are employed. The original cuckoo hashing algorithm was described - * in: - * - * Pagh, R., F.F. Rodler (2004) Cuckoo Hashing. Journal of Algorithms - * 51(2):122-144. - * - * Generalization of cuckoo hashing was discussed in: - * - * Erlingsson, U., M. Manasse, F. McSherry (2006) A cool and practical - * alternative to traditional hash tables. In Proceedings of the 7th - * Workshop on Distributed Data and Structures (WDAS'06), Santa Clara, CA, - * January 2006. - * - * This implementation uses precisely two hash functions because that is the - * fewest that can work, and supporting multiple hashes is an implementation - * burden. Here is a reproduction of Figure 1 from Erlingsson et al. (2006) - * that shows approximate expected maximum load factors for various - * configurations: - * - * | #cells/bucket | - * #hashes | 1 | 2 | 4 | 8 | - * --------+-------+-------+-------+-------+ - * 1 | 0.006 | 0.006 | 0.03 | 0.12 | - * 2 | 0.49 | 0.86 |>0.93< |>0.96< | - * 3 | 0.91 | 0.97 | 0.98 | 0.999 | - * 4 | 0.97 | 0.99 | 0.999 | | - * - * The number of cells per bucket is chosen such that a bucket fits in one cache - * line. So, on 32- and 64-bit systems, we use (8,2) and (4,2) cuckoo hashing, - * respectively. - * - ******************************************************************************/ -#define JEMALLOC_CKH_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static bool ckh_grow(ckh_t *ckh); -static void ckh_shrink(ckh_t *ckh); - -/******************************************************************************/ - -/* - * Search bucket for key and return the cell number if found; SIZE_T_MAX - * otherwise. - */ -JEMALLOC_INLINE size_t -ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) -{ - ckhc_t *cell; - unsigned i; - - for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { - cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; - if (cell->key != NULL && ckh->keycomp(key, cell->key)) - return ((bucket << LG_CKH_BUCKET_CELLS) + i); - } - - return (SIZE_T_MAX); -} - -/* - * Search table for key and return cell number if found; SIZE_T_MAX otherwise. - */ -JEMALLOC_INLINE size_t -ckh_isearch(ckh_t *ckh, const void *key) -{ - size_t hash1, hash2, bucket, cell; - - assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); - - /* Search primary bucket. */ - bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); - cell = ckh_bucket_search(ckh, bucket, key); - if (cell != SIZE_T_MAX) - return (cell); - - /* Search secondary bucket. */ - bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); - cell = ckh_bucket_search(ckh, bucket, key); - return (cell); -} - -JEMALLOC_INLINE bool -ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, - const void *data) -{ - ckhc_t *cell; - unsigned offset, i; - - /* - * Cycle through the cells in the bucket, starting at a random position. - * The randomness avoids worst-case search overhead as buckets fill up. - */ - prn32(offset, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); - for (i = 0; i < (ZU(1) << LG_CKH_BUCKET_CELLS); i++) { - cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + - ((i + offset) & ((ZU(1) << LG_CKH_BUCKET_CELLS) - 1))]; - if (cell->key == NULL) { - cell->key = key; - cell->data = data; - ckh->count++; - return (false); - } - } - - return (true); -} - -/* - * No space is available in bucket. Randomly evict an item, then try to find an - * alternate location for that item. Iteratively repeat this - * eviction/relocation procedure until either success or detection of an - * eviction/relocation bucket cycle. - */ -JEMALLOC_INLINE bool -ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, - void const **argdata) -{ - const void *key, *data, *tkey, *tdata; - ckhc_t *cell; - size_t hash1, hash2, bucket, tbucket; - unsigned i; - - bucket = argbucket; - key = *argkey; - data = *argdata; - while (true) { - /* - * Choose a random item within the bucket to evict. This is - * critical to correct function, because without (eventually) - * evicting all items within a bucket during iteration, it - * would be possible to get stuck in an infinite loop if there - * were an item for which both hashes indicated the same - * bucket. - */ - prn32(i, LG_CKH_BUCKET_CELLS, ckh->prn_state, CKH_A, CKH_C); - cell = &ckh->tab[(bucket << LG_CKH_BUCKET_CELLS) + i]; - assert(cell->key != NULL); - - /* Swap cell->{key,data} and {key,data} (evict). */ - tkey = cell->key; tdata = cell->data; - cell->key = key; cell->data = data; - key = tkey; data = tdata; - -#ifdef CKH_COUNT - ckh->nrelocs++; -#endif - - /* Find the alternate bucket for the evicted item. */ - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); - tbucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (tbucket == bucket) { - tbucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); - /* - * It may be that (tbucket == bucket) still, if the - * item's hashes both indicate this bucket. However, - * we are guaranteed to eventually escape this bucket - * during iteration, assuming pseudo-random item - * selection (true randomness would make infinite - * looping a remote possibility). The reason we can - * never get trapped forever is that there are two - * cases: - * - * 1) This bucket == argbucket, so we will quickly - * detect an eviction cycle and terminate. - * 2) An item was evicted to this bucket from another, - * which means that at least one item in this bucket - * has hashes that indicate distinct buckets. - */ - } - /* Check for a cycle. */ - if (tbucket == argbucket) { - *argkey = key; - *argdata = data; - return (true); - } - - bucket = tbucket; - if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) - return (false); - } -} - -JEMALLOC_INLINE bool -ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) -{ - size_t hash1, hash2, bucket; - const void *key = *argkey; - const void *data = *argdata; - - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); - - /* Try to insert in primary bucket. */ - bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) - return (false); - - /* Try to insert in secondary bucket. */ - bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); - if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) - return (false); - - /* - * Try to find a place for this item via iterative eviction/relocation. - */ - return (ckh_evict_reloc_insert(ckh, bucket, argkey, argdata)); -} - -/* - * Try to rebuild the hash table from scratch by inserting all items from the - * old table into the new. - */ -JEMALLOC_INLINE bool -ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) -{ - size_t count, i, nins; - const void *key, *data; - - count = ckh->count; - ckh->count = 0; - for (i = nins = 0; nins < count; i++) { - if (aTab[i].key != NULL) { - key = aTab[i].key; - data = aTab[i].data; - if (ckh_try_insert(ckh, &key, &data)) { - ckh->count = count; - return (true); - } - nins++; - } - } - - return (false); -} - -static bool -ckh_grow(ckh_t *ckh) -{ - bool ret; - ckhc_t *tab, *ttab; - size_t lg_curcells; - unsigned lg_prevbuckets; - -#ifdef CKH_COUNT - ckh->ngrows++; -#endif - - /* - * It is possible (though unlikely, given well behaved hashes) that the - * table will have to be doubled more than once in order to create a - * usable table. - */ - lg_prevbuckets = ckh->lg_curbuckets; - lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS; - while (true) { - size_t usize; - - lg_curcells++; - usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); - if (usize == 0) { - ret = true; - goto RETURN; - } - tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); - if (tab == NULL) { - ret = true; - goto RETURN; - } - /* Swap in new table. */ - ttab = ckh->tab; - ckh->tab = tab; - tab = ttab; - ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; - - if (ckh_rebuild(ckh, tab) == false) { - idalloc(tab); - break; - } - - /* Rebuilding failed, so back out partially rebuilt table. */ - idalloc(ckh->tab); - ckh->tab = tab; - ckh->lg_curbuckets = lg_prevbuckets; - } - - ret = false; -RETURN: - return (ret); -} - -static void -ckh_shrink(ckh_t *ckh) -{ - ckhc_t *tab, *ttab; - size_t lg_curcells, usize; - unsigned lg_prevbuckets; - - /* - * It is possible (though unlikely, given well behaved hashes) that the - * table rebuild will fail. - */ - lg_prevbuckets = ckh->lg_curbuckets; - lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; - usize = sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE, NULL); - if (usize == 0) - return; - tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); - if (tab == NULL) { - /* - * An OOM error isn't worth propagating, since it doesn't - * prevent this or future operations from proceeding. - */ - return; - } - /* Swap in new table. */ - ttab = ckh->tab; - ckh->tab = tab; - tab = ttab; - ckh->lg_curbuckets = lg_curcells - LG_CKH_BUCKET_CELLS; - - if (ckh_rebuild(ckh, tab) == false) { - idalloc(tab); -#ifdef CKH_COUNT - ckh->nshrinks++; -#endif - return; - } - - /* Rebuilding failed, so back out partially rebuilt table. */ - idalloc(ckh->tab); - ckh->tab = tab; - ckh->lg_curbuckets = lg_prevbuckets; -#ifdef CKH_COUNT - ckh->nshrinkfails++; -#endif -} - -bool -ckh_new(ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh_keycomp_t *keycomp) -{ - bool ret; - size_t mincells, usize; - unsigned lg_mincells; - - assert(minitems > 0); - assert(hash != NULL); - assert(keycomp != NULL); - -#ifdef CKH_COUNT - ckh->ngrows = 0; - ckh->nshrinks = 0; - ckh->nshrinkfails = 0; - ckh->ninserts = 0; - ckh->nrelocs = 0; -#endif - ckh->prn_state = 42; /* Value doesn't really matter. */ - ckh->count = 0; - - /* - * Find the minimum power of 2 that is large enough to fit aBaseCount - * entries. We are using (2+,2) cuckoo hashing, which has an expected - * maximum load factor of at least ~0.86, so 0.75 is a conservative load - * factor that will typically allow 2^aLgMinItems to fit without ever - * growing the table. - */ - assert(LG_CKH_BUCKET_CELLS > 0); - mincells = ((minitems + (3 - (minitems % 3))) / 3) << 2; - for (lg_mincells = LG_CKH_BUCKET_CELLS; - (ZU(1) << lg_mincells) < mincells; - lg_mincells++) - ; /* Do nothing. */ - ckh->lg_minbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; - ckh->lg_curbuckets = lg_mincells - LG_CKH_BUCKET_CELLS; - ckh->hash = hash; - ckh->keycomp = keycomp; - - usize = sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE, NULL); - if (usize == 0) { - ret = true; - goto RETURN; - } - ckh->tab = (ckhc_t *)ipalloc(usize, CACHELINE, true); - if (ckh->tab == NULL) { - ret = true; - goto RETURN; - } - -#ifdef JEMALLOC_DEBUG - ckh->magic = CKH_MAGIC; -#endif - - ret = false; -RETURN: - return (ret); -} - -void -ckh_delete(ckh_t *ckh) -{ - - assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - -#ifdef CKH_VERBOSE - malloc_printf( - "%s(%p): ngrows: %"PRIu64", nshrinks: %"PRIu64"," - " nshrinkfails: %"PRIu64", ninserts: %"PRIu64"," - " nrelocs: %"PRIu64"\n", __func__, ckh, - (unsigned long long)ckh->ngrows, - (unsigned long long)ckh->nshrinks, - (unsigned long long)ckh->nshrinkfails, - (unsigned long long)ckh->ninserts, - (unsigned long long)ckh->nrelocs); -#endif - - idalloc(ckh->tab); -#ifdef JEMALLOC_DEBUG - memset(ckh, 0x5a, sizeof(ckh_t)); -#endif -} - -size_t -ckh_count(ckh_t *ckh) -{ - - assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - - return (ckh->count); -} - -bool -ckh_iter(ckh_t *ckh, size_t *tabind, void **key, void **data) -{ - size_t i, ncells; - - for (i = *tabind, ncells = (ZU(1) << (ckh->lg_curbuckets + - LG_CKH_BUCKET_CELLS)); i < ncells; i++) { - if (ckh->tab[i].key != NULL) { - if (key != NULL) - *key = (void *)ckh->tab[i].key; - if (data != NULL) - *data = (void *)ckh->tab[i].data; - *tabind = i + 1; - return (false); - } - } - - return (true); -} - -bool -ckh_insert(ckh_t *ckh, const void *key, const void *data) -{ - bool ret; - - assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - assert(ckh_search(ckh, key, NULL, NULL)); - -#ifdef CKH_COUNT - ckh->ninserts++; -#endif - - while (ckh_try_insert(ckh, &key, &data)) { - if (ckh_grow(ckh)) { - ret = true; - goto RETURN; - } - } - - ret = false; -RETURN: - return (ret); -} - -bool -ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data) -{ - size_t cell; - - assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - - cell = ckh_isearch(ckh, searchkey); - if (cell != SIZE_T_MAX) { - if (key != NULL) - *key = (void *)ckh->tab[cell].key; - if (data != NULL) - *data = (void *)ckh->tab[cell].data; - ckh->tab[cell].key = NULL; - ckh->tab[cell].data = NULL; /* Not necessary. */ - - ckh->count--; - /* Try to halve the table if it is less than 1/4 full. */ - if (ckh->count < (ZU(1) << (ckh->lg_curbuckets - + LG_CKH_BUCKET_CELLS - 2)) && ckh->lg_curbuckets - > ckh->lg_minbuckets) { - /* Ignore error due to OOM. */ - ckh_shrink(ckh); - } - - return (false); - } - - return (true); -} - -bool -ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) -{ - size_t cell; - - assert(ckh != NULL); - dassert(ckh->magic == CKH_MAGIC); - - cell = ckh_isearch(ckh, searchkey); - if (cell != SIZE_T_MAX) { - if (key != NULL) - *key = (void *)ckh->tab[cell].key; - if (data != NULL) - *data = (void *)ckh->tab[cell].data; - return (false); - } - - return (true); -} - -void -ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) -{ - size_t ret1, ret2; - uint64_t h; - - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - - h = hash(key, strlen((const char *)key), 0x94122f335b332aeaLLU); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - ret1 = h; - ret2 = hash(key, strlen((const char *)key), - 0x8432a476666bbc13LLU); - } - - *hash1 = ret1; - *hash2 = ret2; -} - -bool -ckh_string_keycomp(const void *k1, const void *k2) -{ - - assert(k1 != NULL); - assert(k2 != NULL); - - return (strcmp((char *)k1, (char *)k2) ? false : true); -} - -void -ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2) -{ - size_t ret1, ret2; - uint64_t h; - union { - const void *v; - uint64_t i; - } u; - - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - - assert(sizeof(u.v) == sizeof(u.i)); -#if (LG_SIZEOF_PTR != LG_SIZEOF_INT) - u.i = 0; -#endif - u.v = key; - h = hash(&u.i, sizeof(u.i), 0xd983396e68886082LLU); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - assert(SIZEOF_PTR == 8); - ret1 = h; - ret2 = hash(&u.i, sizeof(u.i), 0x5e2be9aff8709a5dLLU); - } - - *hash1 = ret1; - *hash2 = ret2; -} - -bool -ckh_pointer_keycomp(const void *k1, const void *k2) -{ - - return ((k1 == k2) ? true : false); -} diff --git a/deps/jemalloc.orig/src/ctl.c b/deps/jemalloc.orig/src/ctl.c deleted file mode 100644 index e5336d36949..00000000000 --- a/deps/jemalloc.orig/src/ctl.c +++ /dev/null @@ -1,1670 +0,0 @@ -#define JEMALLOC_CTL_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -/* - * ctl_mtx protects the following: - * - ctl_stats.* - * - opt_prof_active - * - swap_enabled - * - swap_prezeroed - */ -static malloc_mutex_t ctl_mtx; -static bool ctl_initialized; -static uint64_t ctl_epoch; -static ctl_stats_t ctl_stats; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -#define CTL_PROTO(n) \ -static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen); - -#define INDEX_PROTO(n) \ -const ctl_node_t *n##_index(const size_t *mib, size_t miblen, \ - size_t i); - -#ifdef JEMALLOC_STATS -static bool ctl_arena_init(ctl_arena_stats_t *astats); -#endif -static void ctl_arena_clear(ctl_arena_stats_t *astats); -#ifdef JEMALLOC_STATS -static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, - arena_t *arena); -static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, - ctl_arena_stats_t *astats); -#endif -static void ctl_arena_refresh(arena_t *arena, unsigned i); -static void ctl_refresh(void); -static bool ctl_init(void); -static int ctl_lookup(const char *name, ctl_node_t const **nodesp, - size_t *mibp, size_t *depthp); - -CTL_PROTO(version) -CTL_PROTO(epoch) -#ifdef JEMALLOC_TCACHE -CTL_PROTO(tcache_flush) -#endif -CTL_PROTO(thread_arena) -#ifdef JEMALLOC_STATS -CTL_PROTO(thread_allocated) -CTL_PROTO(thread_allocatedp) -CTL_PROTO(thread_deallocated) -CTL_PROTO(thread_deallocatedp) -#endif -CTL_PROTO(config_debug) -CTL_PROTO(config_dss) -CTL_PROTO(config_dynamic_page_shift) -CTL_PROTO(config_fill) -CTL_PROTO(config_lazy_lock) -CTL_PROTO(config_prof) -CTL_PROTO(config_prof_libgcc) -CTL_PROTO(config_prof_libunwind) -CTL_PROTO(config_stats) -CTL_PROTO(config_swap) -CTL_PROTO(config_sysv) -CTL_PROTO(config_tcache) -CTL_PROTO(config_tiny) -CTL_PROTO(config_tls) -CTL_PROTO(config_xmalloc) -CTL_PROTO(opt_abort) -CTL_PROTO(opt_lg_qspace_max) -CTL_PROTO(opt_lg_cspace_max) -CTL_PROTO(opt_lg_chunk) -CTL_PROTO(opt_narenas) -CTL_PROTO(opt_lg_dirty_mult) -CTL_PROTO(opt_stats_print) -#ifdef JEMALLOC_FILL -CTL_PROTO(opt_junk) -CTL_PROTO(opt_zero) -#endif -#ifdef JEMALLOC_SYSV -CTL_PROTO(opt_sysv) -#endif -#ifdef JEMALLOC_XMALLOC -CTL_PROTO(opt_xmalloc) -#endif -#ifdef JEMALLOC_TCACHE -CTL_PROTO(opt_tcache) -CTL_PROTO(opt_lg_tcache_gc_sweep) -#endif -#ifdef JEMALLOC_PROF -CTL_PROTO(opt_prof) -CTL_PROTO(opt_prof_prefix) -CTL_PROTO(opt_prof_active) -CTL_PROTO(opt_lg_prof_bt_max) -CTL_PROTO(opt_lg_prof_sample) -CTL_PROTO(opt_lg_prof_interval) -CTL_PROTO(opt_prof_gdump) -CTL_PROTO(opt_prof_leak) -CTL_PROTO(opt_prof_accum) -CTL_PROTO(opt_lg_prof_tcmax) -#endif -#ifdef JEMALLOC_SWAP -CTL_PROTO(opt_overcommit) -#endif -CTL_PROTO(arenas_bin_i_size) -CTL_PROTO(arenas_bin_i_nregs) -CTL_PROTO(arenas_bin_i_run_size) -INDEX_PROTO(arenas_bin_i) -CTL_PROTO(arenas_lrun_i_size) -INDEX_PROTO(arenas_lrun_i) -CTL_PROTO(arenas_narenas) -CTL_PROTO(arenas_initialized) -CTL_PROTO(arenas_quantum) -CTL_PROTO(arenas_cacheline) -CTL_PROTO(arenas_subpage) -CTL_PROTO(arenas_pagesize) -CTL_PROTO(arenas_chunksize) -#ifdef JEMALLOC_TINY -CTL_PROTO(arenas_tspace_min) -CTL_PROTO(arenas_tspace_max) -#endif -CTL_PROTO(arenas_qspace_min) -CTL_PROTO(arenas_qspace_max) -CTL_PROTO(arenas_cspace_min) -CTL_PROTO(arenas_cspace_max) -CTL_PROTO(arenas_sspace_min) -CTL_PROTO(arenas_sspace_max) -#ifdef JEMALLOC_TCACHE -CTL_PROTO(arenas_tcache_max) -#endif -CTL_PROTO(arenas_ntbins) -CTL_PROTO(arenas_nqbins) -CTL_PROTO(arenas_ncbins) -CTL_PROTO(arenas_nsbins) -CTL_PROTO(arenas_nbins) -#ifdef JEMALLOC_TCACHE -CTL_PROTO(arenas_nhbins) -#endif -CTL_PROTO(arenas_nlruns) -CTL_PROTO(arenas_purge) -#ifdef JEMALLOC_PROF -CTL_PROTO(prof_active) -CTL_PROTO(prof_dump) -CTL_PROTO(prof_interval) -#endif -#ifdef JEMALLOC_STATS -CTL_PROTO(stats_chunks_current) -CTL_PROTO(stats_chunks_total) -CTL_PROTO(stats_chunks_high) -CTL_PROTO(stats_huge_allocated) -CTL_PROTO(stats_huge_nmalloc) -CTL_PROTO(stats_huge_ndalloc) -CTL_PROTO(stats_arenas_i_small_allocated) -CTL_PROTO(stats_arenas_i_small_nmalloc) -CTL_PROTO(stats_arenas_i_small_ndalloc) -CTL_PROTO(stats_arenas_i_small_nrequests) -CTL_PROTO(stats_arenas_i_large_allocated) -CTL_PROTO(stats_arenas_i_large_nmalloc) -CTL_PROTO(stats_arenas_i_large_ndalloc) -CTL_PROTO(stats_arenas_i_large_nrequests) -CTL_PROTO(stats_arenas_i_bins_j_allocated) -CTL_PROTO(stats_arenas_i_bins_j_nmalloc) -CTL_PROTO(stats_arenas_i_bins_j_ndalloc) -CTL_PROTO(stats_arenas_i_bins_j_nrequests) -#ifdef JEMALLOC_TCACHE -CTL_PROTO(stats_arenas_i_bins_j_nfills) -CTL_PROTO(stats_arenas_i_bins_j_nflushes) -#endif -CTL_PROTO(stats_arenas_i_bins_j_nruns) -CTL_PROTO(stats_arenas_i_bins_j_nreruns) -CTL_PROTO(stats_arenas_i_bins_j_highruns) -CTL_PROTO(stats_arenas_i_bins_j_curruns) -INDEX_PROTO(stats_arenas_i_bins_j) -CTL_PROTO(stats_arenas_i_lruns_j_nmalloc) -CTL_PROTO(stats_arenas_i_lruns_j_ndalloc) -CTL_PROTO(stats_arenas_i_lruns_j_nrequests) -CTL_PROTO(stats_arenas_i_lruns_j_highruns) -CTL_PROTO(stats_arenas_i_lruns_j_curruns) -INDEX_PROTO(stats_arenas_i_lruns_j) -#endif -CTL_PROTO(stats_arenas_i_nthreads) -CTL_PROTO(stats_arenas_i_pactive) -CTL_PROTO(stats_arenas_i_pdirty) -#ifdef JEMALLOC_STATS -CTL_PROTO(stats_arenas_i_mapped) -CTL_PROTO(stats_arenas_i_npurge) -CTL_PROTO(stats_arenas_i_nmadvise) -CTL_PROTO(stats_arenas_i_purged) -#endif -INDEX_PROTO(stats_arenas_i) -#ifdef JEMALLOC_STATS -CTL_PROTO(stats_cactive) -CTL_PROTO(stats_allocated) -CTL_PROTO(stats_active) -CTL_PROTO(stats_mapped) -#endif -#ifdef JEMALLOC_SWAP -# ifdef JEMALLOC_STATS -CTL_PROTO(swap_avail) -# endif -CTL_PROTO(swap_prezeroed) -CTL_PROTO(swap_nfds) -CTL_PROTO(swap_fds) -#endif - -/******************************************************************************/ -/* mallctl tree. */ - -/* Maximum tree depth. */ -#define CTL_MAX_DEPTH 6 - -#define NAME(n) true, {.named = {n -#define CHILD(c) sizeof(c##_node) / sizeof(ctl_node_t), c##_node}}, NULL -#define CTL(c) 0, NULL}}, c##_ctl - -/* - * Only handles internal indexed nodes, since there are currently no external - * ones. - */ -#define INDEX(i) false, {.indexed = {i##_index}}, NULL - -#ifdef JEMALLOC_TCACHE -static const ctl_node_t tcache_node[] = { - {NAME("flush"), CTL(tcache_flush)} -}; -#endif - -static const ctl_node_t thread_node[] = { - {NAME("arena"), CTL(thread_arena)} -#ifdef JEMALLOC_STATS - , - {NAME("allocated"), CTL(thread_allocated)}, - {NAME("allocatedp"), CTL(thread_allocatedp)}, - {NAME("deallocated"), CTL(thread_deallocated)}, - {NAME("deallocatedp"), CTL(thread_deallocatedp)} -#endif -}; - -static const ctl_node_t config_node[] = { - {NAME("debug"), CTL(config_debug)}, - {NAME("dss"), CTL(config_dss)}, - {NAME("dynamic_page_shift"), CTL(config_dynamic_page_shift)}, - {NAME("fill"), CTL(config_fill)}, - {NAME("lazy_lock"), CTL(config_lazy_lock)}, - {NAME("prof"), CTL(config_prof)}, - {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, - {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, - {NAME("stats"), CTL(config_stats)}, - {NAME("swap"), CTL(config_swap)}, - {NAME("sysv"), CTL(config_sysv)}, - {NAME("tcache"), CTL(config_tcache)}, - {NAME("tiny"), CTL(config_tiny)}, - {NAME("tls"), CTL(config_tls)}, - {NAME("xmalloc"), CTL(config_xmalloc)} -}; - -static const ctl_node_t opt_node[] = { - {NAME("abort"), CTL(opt_abort)}, - {NAME("lg_qspace_max"), CTL(opt_lg_qspace_max)}, - {NAME("lg_cspace_max"), CTL(opt_lg_cspace_max)}, - {NAME("lg_chunk"), CTL(opt_lg_chunk)}, - {NAME("narenas"), CTL(opt_narenas)}, - {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, - {NAME("stats_print"), CTL(opt_stats_print)} -#ifdef JEMALLOC_FILL - , - {NAME("junk"), CTL(opt_junk)}, - {NAME("zero"), CTL(opt_zero)} -#endif -#ifdef JEMALLOC_SYSV - , - {NAME("sysv"), CTL(opt_sysv)} -#endif -#ifdef JEMALLOC_XMALLOC - , - {NAME("xmalloc"), CTL(opt_xmalloc)} -#endif -#ifdef JEMALLOC_TCACHE - , - {NAME("tcache"), CTL(opt_tcache)}, - {NAME("lg_tcache_gc_sweep"), CTL(opt_lg_tcache_gc_sweep)} -#endif -#ifdef JEMALLOC_PROF - , - {NAME("prof"), CTL(opt_prof)}, - {NAME("prof_prefix"), CTL(opt_prof_prefix)}, - {NAME("prof_active"), CTL(opt_prof_active)}, - {NAME("lg_prof_bt_max"), CTL(opt_lg_prof_bt_max)}, - {NAME("lg_prof_sample"), CTL(opt_lg_prof_sample)}, - {NAME("lg_prof_interval"), CTL(opt_lg_prof_interval)}, - {NAME("prof_gdump"), CTL(opt_prof_gdump)}, - {NAME("prof_leak"), CTL(opt_prof_leak)}, - {NAME("prof_accum"), CTL(opt_prof_accum)}, - {NAME("lg_prof_tcmax"), CTL(opt_lg_prof_tcmax)} -#endif -#ifdef JEMALLOC_SWAP - , - {NAME("overcommit"), CTL(opt_overcommit)} -#endif -}; - -static const ctl_node_t arenas_bin_i_node[] = { - {NAME("size"), CTL(arenas_bin_i_size)}, - {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("run_size"), CTL(arenas_bin_i_run_size)} -}; -static const ctl_node_t super_arenas_bin_i_node[] = { - {NAME(""), CHILD(arenas_bin_i)} -}; - -static const ctl_node_t arenas_bin_node[] = { - {INDEX(arenas_bin_i)} -}; - -static const ctl_node_t arenas_lrun_i_node[] = { - {NAME("size"), CTL(arenas_lrun_i_size)} -}; -static const ctl_node_t super_arenas_lrun_i_node[] = { - {NAME(""), CHILD(arenas_lrun_i)} -}; - -static const ctl_node_t arenas_lrun_node[] = { - {INDEX(arenas_lrun_i)} -}; - -static const ctl_node_t arenas_node[] = { - {NAME("narenas"), CTL(arenas_narenas)}, - {NAME("initialized"), CTL(arenas_initialized)}, - {NAME("quantum"), CTL(arenas_quantum)}, - {NAME("cacheline"), CTL(arenas_cacheline)}, - {NAME("subpage"), CTL(arenas_subpage)}, - {NAME("pagesize"), CTL(arenas_pagesize)}, - {NAME("chunksize"), CTL(arenas_chunksize)}, -#ifdef JEMALLOC_TINY - {NAME("tspace_min"), CTL(arenas_tspace_min)}, - {NAME("tspace_max"), CTL(arenas_tspace_max)}, -#endif - {NAME("qspace_min"), CTL(arenas_qspace_min)}, - {NAME("qspace_max"), CTL(arenas_qspace_max)}, - {NAME("cspace_min"), CTL(arenas_cspace_min)}, - {NAME("cspace_max"), CTL(arenas_cspace_max)}, - {NAME("sspace_min"), CTL(arenas_sspace_min)}, - {NAME("sspace_max"), CTL(arenas_sspace_max)}, -#ifdef JEMALLOC_TCACHE - {NAME("tcache_max"), CTL(arenas_tcache_max)}, -#endif - {NAME("ntbins"), CTL(arenas_ntbins)}, - {NAME("nqbins"), CTL(arenas_nqbins)}, - {NAME("ncbins"), CTL(arenas_ncbins)}, - {NAME("nsbins"), CTL(arenas_nsbins)}, - {NAME("nbins"), CTL(arenas_nbins)}, -#ifdef JEMALLOC_TCACHE - {NAME("nhbins"), CTL(arenas_nhbins)}, -#endif - {NAME("bin"), CHILD(arenas_bin)}, - {NAME("nlruns"), CTL(arenas_nlruns)}, - {NAME("lrun"), CHILD(arenas_lrun)}, - {NAME("purge"), CTL(arenas_purge)} -}; - -#ifdef JEMALLOC_PROF -static const ctl_node_t prof_node[] = { - {NAME("active"), CTL(prof_active)}, - {NAME("dump"), CTL(prof_dump)}, - {NAME("interval"), CTL(prof_interval)} -}; -#endif - -#ifdef JEMALLOC_STATS -static const ctl_node_t stats_chunks_node[] = { - {NAME("current"), CTL(stats_chunks_current)}, - {NAME("total"), CTL(stats_chunks_total)}, - {NAME("high"), CTL(stats_chunks_high)} -}; - -static const ctl_node_t stats_huge_node[] = { - {NAME("allocated"), CTL(stats_huge_allocated)}, - {NAME("nmalloc"), CTL(stats_huge_nmalloc)}, - {NAME("ndalloc"), CTL(stats_huge_ndalloc)} -}; - -static const ctl_node_t stats_arenas_i_small_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_small_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_small_nrequests)} -}; - -static const ctl_node_t stats_arenas_i_large_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_large_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_large_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_large_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_large_nrequests)} -}; - -static const ctl_node_t stats_arenas_i_bins_j_node[] = { - {NAME("allocated"), CTL(stats_arenas_i_bins_j_allocated)}, - {NAME("nmalloc"), CTL(stats_arenas_i_bins_j_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_bins_j_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_bins_j_nrequests)}, -#ifdef JEMALLOC_TCACHE - {NAME("nfills"), CTL(stats_arenas_i_bins_j_nfills)}, - {NAME("nflushes"), CTL(stats_arenas_i_bins_j_nflushes)}, -#endif - {NAME("nruns"), CTL(stats_arenas_i_bins_j_nruns)}, - {NAME("nreruns"), CTL(stats_arenas_i_bins_j_nreruns)}, - {NAME("highruns"), CTL(stats_arenas_i_bins_j_highruns)}, - {NAME("curruns"), CTL(stats_arenas_i_bins_j_curruns)} -}; -static const ctl_node_t super_stats_arenas_i_bins_j_node[] = { - {NAME(""), CHILD(stats_arenas_i_bins_j)} -}; - -static const ctl_node_t stats_arenas_i_bins_node[] = { - {INDEX(stats_arenas_i_bins_j)} -}; - -static const ctl_node_t stats_arenas_i_lruns_j_node[] = { - {NAME("nmalloc"), CTL(stats_arenas_i_lruns_j_nmalloc)}, - {NAME("ndalloc"), CTL(stats_arenas_i_lruns_j_ndalloc)}, - {NAME("nrequests"), CTL(stats_arenas_i_lruns_j_nrequests)}, - {NAME("highruns"), CTL(stats_arenas_i_lruns_j_highruns)}, - {NAME("curruns"), CTL(stats_arenas_i_lruns_j_curruns)} -}; -static const ctl_node_t super_stats_arenas_i_lruns_j_node[] = { - {NAME(""), CHILD(stats_arenas_i_lruns_j)} -}; - -static const ctl_node_t stats_arenas_i_lruns_node[] = { - {INDEX(stats_arenas_i_lruns_j)} -}; -#endif - -static const ctl_node_t stats_arenas_i_node[] = { - {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, - {NAME("pactive"), CTL(stats_arenas_i_pactive)}, - {NAME("pdirty"), CTL(stats_arenas_i_pdirty)} -#ifdef JEMALLOC_STATS - , - {NAME("mapped"), CTL(stats_arenas_i_mapped)}, - {NAME("npurge"), CTL(stats_arenas_i_npurge)}, - {NAME("nmadvise"), CTL(stats_arenas_i_nmadvise)}, - {NAME("purged"), CTL(stats_arenas_i_purged)}, - {NAME("small"), CHILD(stats_arenas_i_small)}, - {NAME("large"), CHILD(stats_arenas_i_large)}, - {NAME("bins"), CHILD(stats_arenas_i_bins)}, - {NAME("lruns"), CHILD(stats_arenas_i_lruns)} -#endif -}; -static const ctl_node_t super_stats_arenas_i_node[] = { - {NAME(""), CHILD(stats_arenas_i)} -}; - -static const ctl_node_t stats_arenas_node[] = { - {INDEX(stats_arenas_i)} -}; - -static const ctl_node_t stats_node[] = { -#ifdef JEMALLOC_STATS - {NAME("cactive"), CTL(stats_cactive)}, - {NAME("allocated"), CTL(stats_allocated)}, - {NAME("active"), CTL(stats_active)}, - {NAME("mapped"), CTL(stats_mapped)}, - {NAME("chunks"), CHILD(stats_chunks)}, - {NAME("huge"), CHILD(stats_huge)}, -#endif - {NAME("arenas"), CHILD(stats_arenas)} -}; - -#ifdef JEMALLOC_SWAP -static const ctl_node_t swap_node[] = { -# ifdef JEMALLOC_STATS - {NAME("avail"), CTL(swap_avail)}, -# endif - {NAME("prezeroed"), CTL(swap_prezeroed)}, - {NAME("nfds"), CTL(swap_nfds)}, - {NAME("fds"), CTL(swap_fds)} -}; -#endif - -static const ctl_node_t root_node[] = { - {NAME("version"), CTL(version)}, - {NAME("epoch"), CTL(epoch)}, -#ifdef JEMALLOC_TCACHE - {NAME("tcache"), CHILD(tcache)}, -#endif - {NAME("thread"), CHILD(thread)}, - {NAME("config"), CHILD(config)}, - {NAME("opt"), CHILD(opt)}, - {NAME("arenas"), CHILD(arenas)}, -#ifdef JEMALLOC_PROF - {NAME("prof"), CHILD(prof)}, -#endif - {NAME("stats"), CHILD(stats)} -#ifdef JEMALLOC_SWAP - , - {NAME("swap"), CHILD(swap)} -#endif -}; -static const ctl_node_t super_root_node[] = { - {NAME(""), CHILD(root)} -}; - -#undef NAME -#undef CHILD -#undef CTL -#undef INDEX - -/******************************************************************************/ - -#ifdef JEMALLOC_STATS -static bool -ctl_arena_init(ctl_arena_stats_t *astats) -{ - - if (astats->bstats == NULL) { - astats->bstats = (malloc_bin_stats_t *)base_alloc(nbins * - sizeof(malloc_bin_stats_t)); - if (astats->bstats == NULL) - return (true); - } - if (astats->lstats == NULL) { - astats->lstats = (malloc_large_stats_t *)base_alloc(nlclasses * - sizeof(malloc_large_stats_t)); - if (astats->lstats == NULL) - return (true); - } - - return (false); -} -#endif - -static void -ctl_arena_clear(ctl_arena_stats_t *astats) -{ - - astats->pactive = 0; - astats->pdirty = 0; -#ifdef JEMALLOC_STATS - memset(&astats->astats, 0, sizeof(arena_stats_t)); - astats->allocated_small = 0; - astats->nmalloc_small = 0; - astats->ndalloc_small = 0; - astats->nrequests_small = 0; - memset(astats->bstats, 0, nbins * sizeof(malloc_bin_stats_t)); - memset(astats->lstats, 0, nlclasses * sizeof(malloc_large_stats_t)); -#endif -} - -#ifdef JEMALLOC_STATS -static void -ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) -{ - unsigned i; - - arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty, - &cstats->astats, cstats->bstats, cstats->lstats); - - for (i = 0; i < nbins; i++) { - cstats->allocated_small += cstats->bstats[i].allocated; - cstats->nmalloc_small += cstats->bstats[i].nmalloc; - cstats->ndalloc_small += cstats->bstats[i].ndalloc; - cstats->nrequests_small += cstats->bstats[i].nrequests; - } -} - -static void -ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats) -{ - unsigned i; - - sstats->pactive += astats->pactive; - sstats->pdirty += astats->pdirty; - - sstats->astats.mapped += astats->astats.mapped; - sstats->astats.npurge += astats->astats.npurge; - sstats->astats.nmadvise += astats->astats.nmadvise; - sstats->astats.purged += astats->astats.purged; - - sstats->allocated_small += astats->allocated_small; - sstats->nmalloc_small += astats->nmalloc_small; - sstats->ndalloc_small += astats->ndalloc_small; - sstats->nrequests_small += astats->nrequests_small; - - sstats->astats.allocated_large += astats->astats.allocated_large; - sstats->astats.nmalloc_large += astats->astats.nmalloc_large; - sstats->astats.ndalloc_large += astats->astats.ndalloc_large; - sstats->astats.nrequests_large += astats->astats.nrequests_large; - - for (i = 0; i < nlclasses; i++) { - sstats->lstats[i].nmalloc += astats->lstats[i].nmalloc; - sstats->lstats[i].ndalloc += astats->lstats[i].ndalloc; - sstats->lstats[i].nrequests += astats->lstats[i].nrequests; - sstats->lstats[i].highruns += astats->lstats[i].highruns; - sstats->lstats[i].curruns += astats->lstats[i].curruns; - } - - for (i = 0; i < nbins; i++) { - sstats->bstats[i].allocated += astats->bstats[i].allocated; - sstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; - sstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; - sstats->bstats[i].nrequests += astats->bstats[i].nrequests; -#ifdef JEMALLOC_TCACHE - sstats->bstats[i].nfills += astats->bstats[i].nfills; - sstats->bstats[i].nflushes += astats->bstats[i].nflushes; -#endif - sstats->bstats[i].nruns += astats->bstats[i].nruns; - sstats->bstats[i].reruns += astats->bstats[i].reruns; - sstats->bstats[i].highruns += astats->bstats[i].highruns; - sstats->bstats[i].curruns += astats->bstats[i].curruns; - } -} -#endif - -static void -ctl_arena_refresh(arena_t *arena, unsigned i) -{ - ctl_arena_stats_t *astats = &ctl_stats.arenas[i]; - ctl_arena_stats_t *sstats = &ctl_stats.arenas[narenas]; - - ctl_arena_clear(astats); - - sstats->nthreads += astats->nthreads; -#ifdef JEMALLOC_STATS - ctl_arena_stats_amerge(astats, arena); - /* Merge into sum stats as well. */ - ctl_arena_stats_smerge(sstats, astats); -#else - astats->pactive += arena->nactive; - astats->pdirty += arena->ndirty; - /* Merge into sum stats as well. */ - sstats->pactive += arena->nactive; - sstats->pdirty += arena->ndirty; -#endif -} - -static void -ctl_refresh(void) -{ - unsigned i; - arena_t *tarenas[narenas]; - -#ifdef JEMALLOC_STATS - malloc_mutex_lock(&chunks_mtx); - ctl_stats.chunks.current = stats_chunks.curchunks; - ctl_stats.chunks.total = stats_chunks.nchunks; - ctl_stats.chunks.high = stats_chunks.highchunks; - malloc_mutex_unlock(&chunks_mtx); - - malloc_mutex_lock(&huge_mtx); - ctl_stats.huge.allocated = huge_allocated; - ctl_stats.huge.nmalloc = huge_nmalloc; - ctl_stats.huge.ndalloc = huge_ndalloc; - malloc_mutex_unlock(&huge_mtx); -#endif - - /* - * Clear sum stats, since they will be merged into by - * ctl_arena_refresh(). - */ - ctl_stats.arenas[narenas].nthreads = 0; - ctl_arena_clear(&ctl_stats.arenas[narenas]); - - malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); - for (i = 0; i < narenas; i++) { - if (arenas[i] != NULL) - ctl_stats.arenas[i].nthreads = arenas[i]->nthreads; - else - ctl_stats.arenas[i].nthreads = 0; - } - malloc_mutex_unlock(&arenas_lock); - for (i = 0; i < narenas; i++) { - bool initialized = (tarenas[i] != NULL); - - ctl_stats.arenas[i].initialized = initialized; - if (initialized) - ctl_arena_refresh(tarenas[i], i); - } - -#ifdef JEMALLOC_STATS - ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small - + ctl_stats.arenas[narenas].astats.allocated_large - + ctl_stats.huge.allocated; - ctl_stats.active = (ctl_stats.arenas[narenas].pactive << PAGE_SHIFT) - + ctl_stats.huge.allocated; - ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); - -# ifdef JEMALLOC_SWAP - malloc_mutex_lock(&swap_mtx); - ctl_stats.swap_avail = swap_avail; - malloc_mutex_unlock(&swap_mtx); -# endif -#endif - - ctl_epoch++; -} - -static bool -ctl_init(void) -{ - bool ret; - - malloc_mutex_lock(&ctl_mtx); - if (ctl_initialized == false) { -#ifdef JEMALLOC_STATS - unsigned i; -#endif - - /* - * Allocate space for one extra arena stats element, which - * contains summed stats across all arenas. - */ - ctl_stats.arenas = (ctl_arena_stats_t *)base_alloc( - (narenas + 1) * sizeof(ctl_arena_stats_t)); - if (ctl_stats.arenas == NULL) { - ret = true; - goto RETURN; - } - memset(ctl_stats.arenas, 0, (narenas + 1) * - sizeof(ctl_arena_stats_t)); - - /* - * Initialize all stats structures, regardless of whether they - * ever get used. Lazy initialization would allow errors to - * cause inconsistent state to be viewable by the application. - */ -#ifdef JEMALLOC_STATS - for (i = 0; i <= narenas; i++) { - if (ctl_arena_init(&ctl_stats.arenas[i])) { - ret = true; - goto RETURN; - } - } -#endif - ctl_stats.arenas[narenas].initialized = true; - - ctl_epoch = 0; - ctl_refresh(); - ctl_initialized = true; - } - - ret = false; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -static int -ctl_lookup(const char *name, ctl_node_t const **nodesp, size_t *mibp, - size_t *depthp) -{ - int ret; - const char *elm, *tdot, *dot; - size_t elen, i, j; - const ctl_node_t *node; - - elm = name; - /* Equivalent to strchrnul(). */ - dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : strchr(elm, '\0'); - elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); - if (elen == 0) { - ret = ENOENT; - goto RETURN; - } - node = super_root_node; - for (i = 0; i < *depthp; i++) { - assert(node->named); - assert(node->u.named.nchildren > 0); - if (node->u.named.children[0].named) { - const ctl_node_t *pnode = node; - - /* Children are named. */ - for (j = 0; j < node->u.named.nchildren; j++) { - const ctl_node_t *child = - &node->u.named.children[j]; - if (strlen(child->u.named.name) == elen - && strncmp(elm, child->u.named.name, - elen) == 0) { - node = child; - if (nodesp != NULL) - nodesp[i] = node; - mibp[i] = j; - break; - } - } - if (node == pnode) { - ret = ENOENT; - goto RETURN; - } - } else { - unsigned long index; - const ctl_node_t *inode; - - /* Children are indexed. */ - index = strtoul(elm, NULL, 10); - if (index == ULONG_MAX) { - ret = ENOENT; - goto RETURN; - } - - inode = &node->u.named.children[0]; - node = inode->u.indexed.index(mibp, *depthp, - index); - if (node == NULL) { - ret = ENOENT; - goto RETURN; - } - - if (nodesp != NULL) - nodesp[i] = node; - mibp[i] = (size_t)index; - } - - if (node->ctl != NULL) { - /* Terminal node. */ - if (*dot != '\0') { - /* - * The name contains more elements than are - * in this path through the tree. - */ - ret = ENOENT; - goto RETURN; - } - /* Complete lookup successful. */ - *depthp = i + 1; - break; - } - - /* Update elm. */ - if (*dot == '\0') { - /* No more elements. */ - ret = ENOENT; - goto RETURN; - } - elm = &dot[1]; - dot = ((tdot = strchr(elm, '.')) != NULL) ? tdot : - strchr(elm, '\0'); - elen = (size_t)((uintptr_t)dot - (uintptr_t)elm); - } - - ret = 0; -RETURN: - return (ret); -} - -int -ctl_byname(const char *name, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) -{ - int ret; - size_t depth; - ctl_node_t const *nodes[CTL_MAX_DEPTH]; - size_t mib[CTL_MAX_DEPTH]; - - if (ctl_initialized == false && ctl_init()) { - ret = EAGAIN; - goto RETURN; - } - - depth = CTL_MAX_DEPTH; - ret = ctl_lookup(name, nodes, mib, &depth); - if (ret != 0) - goto RETURN; - - if (nodes[depth-1]->ctl == NULL) { - /* The name refers to a partial path through the ctl tree. */ - ret = ENOENT; - goto RETURN; - } - - ret = nodes[depth-1]->ctl(mib, depth, oldp, oldlenp, newp, newlen); -RETURN: - return(ret); -} - -int -ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp) -{ - int ret; - - if (ctl_initialized == false && ctl_init()) { - ret = EAGAIN; - goto RETURN; - } - - ret = ctl_lookup(name, NULL, mibp, miblenp); -RETURN: - return(ret); -} - -int -ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - const ctl_node_t *node; - size_t i; - - if (ctl_initialized == false && ctl_init()) { - ret = EAGAIN; - goto RETURN; - } - - /* Iterate down the tree. */ - node = super_root_node; - for (i = 0; i < miblen; i++) { - if (node->u.named.children[0].named) { - /* Children are named. */ - if (node->u.named.nchildren <= mib[i]) { - ret = ENOENT; - goto RETURN; - } - node = &node->u.named.children[mib[i]]; - } else { - const ctl_node_t *inode; - - /* Indexed element. */ - inode = &node->u.named.children[0]; - node = inode->u.indexed.index(mib, miblen, mib[i]); - if (node == NULL) { - ret = ENOENT; - goto RETURN; - } - } - } - - /* Call the ctl function. */ - if (node->ctl == NULL) { - /* Partial MIB. */ - ret = ENOENT; - goto RETURN; - } - ret = node->ctl(mib, miblen, oldp, oldlenp, newp, newlen); - -RETURN: - return(ret); -} - -bool -ctl_boot(void) -{ - - if (malloc_mutex_init(&ctl_mtx)) - return (true); - - ctl_initialized = false; - - return (false); -} - -/******************************************************************************/ -/* *_ctl() functions. */ - -#define READONLY() do { \ - if (newp != NULL || newlen != 0) { \ - ret = EPERM; \ - goto RETURN; \ - } \ -} while (0) - -#define WRITEONLY() do { \ - if (oldp != NULL || oldlenp != NULL) { \ - ret = EPERM; \ - goto RETURN; \ - } \ -} while (0) - -#define VOID() do { \ - READONLY(); \ - WRITEONLY(); \ -} while (0) - -#define READ(v, t) do { \ - if (oldp != NULL && oldlenp != NULL) { \ - if (*oldlenp != sizeof(t)) { \ - size_t copylen = (sizeof(t) <= *oldlenp) \ - ? sizeof(t) : *oldlenp; \ - memcpy(oldp, (void *)&v, copylen); \ - ret = EINVAL; \ - goto RETURN; \ - } else \ - *(t *)oldp = v; \ - } \ -} while (0) - -#define WRITE(v, t) do { \ - if (newp != NULL) { \ - if (newlen != sizeof(t)) { \ - ret = EINVAL; \ - goto RETURN; \ - } \ - v = *(t *)newp; \ - } \ -} while (0) - -#define CTL_RO_GEN(n, v, t) \ -static int \ -n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ - void *newp, size_t newlen) \ -{ \ - int ret; \ - t oldval; \ - \ - malloc_mutex_lock(&ctl_mtx); \ - READONLY(); \ - oldval = v; \ - READ(oldval, t); \ - \ - ret = 0; \ -RETURN: \ - malloc_mutex_unlock(&ctl_mtx); \ - return (ret); \ -} - -/* - * ctl_mtx is not acquired, under the assumption that no pertinent data will - * mutate during the call. - */ -#define CTL_RO_NL_GEN(n, v, t) \ -static int \ -n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ - void *newp, size_t newlen) \ -{ \ - int ret; \ - t oldval; \ - \ - READONLY(); \ - oldval = v; \ - READ(oldval, t); \ - \ - ret = 0; \ -RETURN: \ - return (ret); \ -} - -#define CTL_RO_TRUE_GEN(n) \ -static int \ -n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ - void *newp, size_t newlen) \ -{ \ - int ret; \ - bool oldval; \ - \ - READONLY(); \ - oldval = true; \ - READ(oldval, bool); \ - \ - ret = 0; \ -RETURN: \ - return (ret); \ -} - -#define CTL_RO_FALSE_GEN(n) \ -static int \ -n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ - void *newp, size_t newlen) \ -{ \ - int ret; \ - bool oldval; \ - \ - READONLY(); \ - oldval = false; \ - READ(oldval, bool); \ - \ - ret = 0; \ -RETURN: \ - return (ret); \ -} - -CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) - -static int -epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - uint64_t newval; - - malloc_mutex_lock(&ctl_mtx); - newval = 0; - WRITE(newval, uint64_t); - if (newval != 0) - ctl_refresh(); - READ(ctl_epoch, uint64_t); - - ret = 0; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -#ifdef JEMALLOC_TCACHE -static int -tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - tcache_t *tcache; - - VOID(); - - tcache = TCACHE_GET(); - if (tcache == NULL) { - ret = 0; - goto RETURN; - } - tcache_destroy(tcache); - TCACHE_SET(NULL); - - ret = 0; -RETURN: - return (ret); -} -#endif - -static int -thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - unsigned newind, oldind; - - newind = oldind = choose_arena()->ind; - WRITE(newind, unsigned); - READ(oldind, unsigned); - if (newind != oldind) { - arena_t *arena; - - if (newind >= narenas) { - /* New arena index is out of range. */ - ret = EFAULT; - goto RETURN; - } - - /* Initialize arena if necessary. */ - malloc_mutex_lock(&arenas_lock); - if ((arena = arenas[newind]) == NULL) - arena = arenas_extend(newind); - arenas[oldind]->nthreads--; - arenas[newind]->nthreads++; - malloc_mutex_unlock(&arenas_lock); - if (arena == NULL) { - ret = EAGAIN; - goto RETURN; - } - - /* Set new arena association. */ - ARENA_SET(arena); -#ifdef JEMALLOC_TCACHE - { - tcache_t *tcache = TCACHE_GET(); - if (tcache != NULL) - tcache->arena = arena; - } -#endif - } - - ret = 0; -RETURN: - return (ret); -} - -#ifdef JEMALLOC_STATS -CTL_RO_NL_GEN(thread_allocated, ALLOCATED_GET(), uint64_t); -CTL_RO_NL_GEN(thread_allocatedp, ALLOCATEDP_GET(), uint64_t *); -CTL_RO_NL_GEN(thread_deallocated, DEALLOCATED_GET(), uint64_t); -CTL_RO_NL_GEN(thread_deallocatedp, DEALLOCATEDP_GET(), uint64_t *); -#endif - -/******************************************************************************/ - -#ifdef JEMALLOC_DEBUG -CTL_RO_TRUE_GEN(config_debug) -#else -CTL_RO_FALSE_GEN(config_debug) -#endif - -#ifdef JEMALLOC_DSS -CTL_RO_TRUE_GEN(config_dss) -#else -CTL_RO_FALSE_GEN(config_dss) -#endif - -#ifdef JEMALLOC_DYNAMIC_PAGE_SHIFT -CTL_RO_TRUE_GEN(config_dynamic_page_shift) -#else -CTL_RO_FALSE_GEN(config_dynamic_page_shift) -#endif - -#ifdef JEMALLOC_FILL -CTL_RO_TRUE_GEN(config_fill) -#else -CTL_RO_FALSE_GEN(config_fill) -#endif - -#ifdef JEMALLOC_LAZY_LOCK -CTL_RO_TRUE_GEN(config_lazy_lock) -#else -CTL_RO_FALSE_GEN(config_lazy_lock) -#endif - -#ifdef JEMALLOC_PROF -CTL_RO_TRUE_GEN(config_prof) -#else -CTL_RO_FALSE_GEN(config_prof) -#endif - -#ifdef JEMALLOC_PROF_LIBGCC -CTL_RO_TRUE_GEN(config_prof_libgcc) -#else -CTL_RO_FALSE_GEN(config_prof_libgcc) -#endif - -#ifdef JEMALLOC_PROF_LIBUNWIND -CTL_RO_TRUE_GEN(config_prof_libunwind) -#else -CTL_RO_FALSE_GEN(config_prof_libunwind) -#endif - -#ifdef JEMALLOC_STATS -CTL_RO_TRUE_GEN(config_stats) -#else -CTL_RO_FALSE_GEN(config_stats) -#endif - -#ifdef JEMALLOC_SWAP -CTL_RO_TRUE_GEN(config_swap) -#else -CTL_RO_FALSE_GEN(config_swap) -#endif - -#ifdef JEMALLOC_SYSV -CTL_RO_TRUE_GEN(config_sysv) -#else -CTL_RO_FALSE_GEN(config_sysv) -#endif - -#ifdef JEMALLOC_TCACHE -CTL_RO_TRUE_GEN(config_tcache) -#else -CTL_RO_FALSE_GEN(config_tcache) -#endif - -#ifdef JEMALLOC_TINY -CTL_RO_TRUE_GEN(config_tiny) -#else -CTL_RO_FALSE_GEN(config_tiny) -#endif - -#ifdef JEMALLOC_TLS -CTL_RO_TRUE_GEN(config_tls) -#else -CTL_RO_FALSE_GEN(config_tls) -#endif - -#ifdef JEMALLOC_XMALLOC -CTL_RO_TRUE_GEN(config_xmalloc) -#else -CTL_RO_FALSE_GEN(config_xmalloc) -#endif - -/******************************************************************************/ - -CTL_RO_NL_GEN(opt_abort, opt_abort, bool) -CTL_RO_NL_GEN(opt_lg_qspace_max, opt_lg_qspace_max, size_t) -CTL_RO_NL_GEN(opt_lg_cspace_max, opt_lg_cspace_max, size_t) -CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) -CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) -CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) -CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) -#ifdef JEMALLOC_FILL -CTL_RO_NL_GEN(opt_junk, opt_junk, bool) -CTL_RO_NL_GEN(opt_zero, opt_zero, bool) -#endif -#ifdef JEMALLOC_SYSV -CTL_RO_NL_GEN(opt_sysv, opt_sysv, bool) -#endif -#ifdef JEMALLOC_XMALLOC -CTL_RO_NL_GEN(opt_xmalloc, opt_xmalloc, bool) -#endif -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) -CTL_RO_NL_GEN(opt_lg_tcache_gc_sweep, opt_lg_tcache_gc_sweep, ssize_t) -#endif -#ifdef JEMALLOC_PROF -CTL_RO_NL_GEN(opt_prof, opt_prof, bool) -CTL_RO_NL_GEN(opt_prof_prefix, opt_prof_prefix, const char *) -CTL_RO_GEN(opt_prof_active, opt_prof_active, bool) /* Mutable. */ -CTL_RO_NL_GEN(opt_lg_prof_bt_max, opt_lg_prof_bt_max, size_t) -CTL_RO_NL_GEN(opt_lg_prof_sample, opt_lg_prof_sample, size_t) -CTL_RO_NL_GEN(opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) -CTL_RO_NL_GEN(opt_prof_gdump, opt_prof_gdump, bool) -CTL_RO_NL_GEN(opt_prof_leak, opt_prof_leak, bool) -CTL_RO_NL_GEN(opt_prof_accum, opt_prof_accum, bool) -CTL_RO_NL_GEN(opt_lg_prof_tcmax, opt_lg_prof_tcmax, ssize_t) -#endif -#ifdef JEMALLOC_SWAP -CTL_RO_NL_GEN(opt_overcommit, opt_overcommit, bool) -#endif - -/******************************************************************************/ - -CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) -CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) -CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) -const ctl_node_t * -arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) -{ - - if (i > nbins) - return (NULL); - return (super_arenas_bin_i_node); -} - -CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << PAGE_SHIFT), size_t) -const ctl_node_t * -arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) -{ - - if (i > nlclasses) - return (NULL); - return (super_arenas_lrun_i_node); -} - -CTL_RO_NL_GEN(arenas_narenas, narenas, unsigned) - -static int -arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - int ret; - unsigned nread, i; - - malloc_mutex_lock(&ctl_mtx); - READONLY(); - if (*oldlenp != narenas * sizeof(bool)) { - ret = EINVAL; - nread = (*oldlenp < narenas * sizeof(bool)) - ? (*oldlenp / sizeof(bool)) : narenas; - } else { - ret = 0; - nread = narenas; - } - - for (i = 0; i < nread; i++) - ((bool *)oldp)[i] = ctl_stats.arenas[i].initialized; - -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) -CTL_RO_NL_GEN(arenas_cacheline, CACHELINE, size_t) -CTL_RO_NL_GEN(arenas_subpage, SUBPAGE, size_t) -CTL_RO_NL_GEN(arenas_pagesize, PAGE_SIZE, size_t) -CTL_RO_NL_GEN(arenas_chunksize, chunksize, size_t) -#ifdef JEMALLOC_TINY -CTL_RO_NL_GEN(arenas_tspace_min, (1U << LG_TINY_MIN), size_t) -CTL_RO_NL_GEN(arenas_tspace_max, (qspace_min >> 1), size_t) -#endif -CTL_RO_NL_GEN(arenas_qspace_min, qspace_min, size_t) -CTL_RO_NL_GEN(arenas_qspace_max, qspace_max, size_t) -CTL_RO_NL_GEN(arenas_cspace_min, cspace_min, size_t) -CTL_RO_NL_GEN(arenas_cspace_max, cspace_max, size_t) -CTL_RO_NL_GEN(arenas_sspace_min, sspace_min, size_t) -CTL_RO_NL_GEN(arenas_sspace_max, sspace_max, size_t) -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -#endif -CTL_RO_NL_GEN(arenas_ntbins, ntbins, unsigned) -CTL_RO_NL_GEN(arenas_nqbins, nqbins, unsigned) -CTL_RO_NL_GEN(arenas_ncbins, ncbins, unsigned) -CTL_RO_NL_GEN(arenas_nsbins, nsbins, unsigned) -CTL_RO_NL_GEN(arenas_nbins, nbins, unsigned) -#ifdef JEMALLOC_TCACHE -CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) -#endif -CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) - -static int -arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - unsigned arena; - - WRITEONLY(); - arena = UINT_MAX; - WRITE(arena, unsigned); - if (newp != NULL && arena >= narenas) { - ret = EFAULT; - goto RETURN; - } else { - arena_t *tarenas[narenas]; - - malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); - malloc_mutex_unlock(&arenas_lock); - - if (arena == UINT_MAX) { - unsigned i; - for (i = 0; i < narenas; i++) { - if (tarenas[i] != NULL) - arena_purge_all(tarenas[i]); - } - } else { - assert(arena < narenas); - if (tarenas[arena] != NULL) - arena_purge_all(tarenas[arena]); - } - } - - ret = 0; -RETURN: - return (ret); -} - -/******************************************************************************/ - -#ifdef JEMALLOC_PROF -static int -prof_active_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - bool oldval; - - malloc_mutex_lock(&ctl_mtx); /* Protect opt_prof_active. */ - oldval = opt_prof_active; - if (newp != NULL) { - /* - * The memory barriers will tend to make opt_prof_active - * propagate faster on systems with weak memory ordering. - */ - mb_write(); - WRITE(opt_prof_active, bool); - mb_write(); - } - READ(oldval, bool); - - ret = 0; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -static int -prof_dump_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - const char *filename = NULL; - - WRITEONLY(); - WRITE(filename, const char *); - - if (prof_mdump(filename)) { - ret = EFAULT; - goto RETURN; - } - - ret = 0; -RETURN: - return (ret); -} - -CTL_RO_NL_GEN(prof_interval, prof_interval, uint64_t) -#endif - -/******************************************************************************/ - -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_chunks_current, ctl_stats.chunks.current, size_t) -CTL_RO_GEN(stats_chunks_total, ctl_stats.chunks.total, uint64_t) -CTL_RO_GEN(stats_chunks_high, ctl_stats.chunks.high, size_t) -CTL_RO_GEN(stats_huge_allocated, huge_allocated, size_t) -CTL_RO_GEN(stats_huge_nmalloc, huge_nmalloc, uint64_t) -CTL_RO_GEN(stats_huge_ndalloc, huge_ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_allocated, - ctl_stats.arenas[mib[2]].allocated_small, size_t) -CTL_RO_GEN(stats_arenas_i_small_nmalloc, - ctl_stats.arenas[mib[2]].nmalloc_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_ndalloc, - ctl_stats.arenas[mib[2]].ndalloc_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_small_nrequests, - ctl_stats.arenas[mib[2]].nrequests_small, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_allocated, - ctl_stats.arenas[mib[2]].astats.allocated_large, size_t) -CTL_RO_GEN(stats_arenas_i_large_nmalloc, - ctl_stats.arenas[mib[2]].astats.nmalloc_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_ndalloc, - ctl_stats.arenas[mib[2]].astats.ndalloc_large, uint64_t) -CTL_RO_GEN(stats_arenas_i_large_nrequests, - ctl_stats.arenas[mib[2]].astats.nrequests_large, uint64_t) - -CTL_RO_GEN(stats_arenas_i_bins_j_allocated, - ctl_stats.arenas[mib[2]].bstats[mib[4]].allocated, size_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nmalloc, - ctl_stats.arenas[mib[2]].bstats[mib[4]].nmalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_ndalloc, - ctl_stats.arenas[mib[2]].bstats[mib[4]].ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nrequests, - ctl_stats.arenas[mib[2]].bstats[mib[4]].nrequests, uint64_t) -#ifdef JEMALLOC_TCACHE -CTL_RO_GEN(stats_arenas_i_bins_j_nfills, - ctl_stats.arenas[mib[2]].bstats[mib[4]].nfills, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nflushes, - ctl_stats.arenas[mib[2]].bstats[mib[4]].nflushes, uint64_t) -#endif -CTL_RO_GEN(stats_arenas_i_bins_j_nruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].nruns, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_nreruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].reruns, uint64_t) -CTL_RO_GEN(stats_arenas_i_bins_j_highruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].highruns, size_t) -CTL_RO_GEN(stats_arenas_i_bins_j_curruns, - ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) - -const ctl_node_t * -stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) -{ - - if (j > nbins) - return (NULL); - return (super_stats_arenas_i_bins_j_node); -} - -CTL_RO_GEN(stats_arenas_i_lruns_j_nmalloc, - ctl_stats.arenas[mib[2]].lstats[mib[4]].nmalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_ndalloc, - ctl_stats.arenas[mib[2]].lstats[mib[4]].ndalloc, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_nrequests, - ctl_stats.arenas[mib[2]].lstats[mib[4]].nrequests, uint64_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_curruns, - ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) -CTL_RO_GEN(stats_arenas_i_lruns_j_highruns, - ctl_stats.arenas[mib[2]].lstats[mib[4]].highruns, size_t) - -const ctl_node_t * -stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) -{ - - if (j > nlclasses) - return (NULL); - return (super_stats_arenas_i_lruns_j_node); -} - -#endif -CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) -CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) -CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_arenas_i_mapped, ctl_stats.arenas[mib[2]].astats.mapped, - size_t) -CTL_RO_GEN(stats_arenas_i_npurge, ctl_stats.arenas[mib[2]].astats.npurge, - uint64_t) -CTL_RO_GEN(stats_arenas_i_nmadvise, ctl_stats.arenas[mib[2]].astats.nmadvise, - uint64_t) -CTL_RO_GEN(stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, - uint64_t) -#endif - -const ctl_node_t * -stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) -{ - const ctl_node_t * ret; - - malloc_mutex_lock(&ctl_mtx); - if (ctl_stats.arenas[i].initialized == false) { - ret = NULL; - goto RETURN; - } - - ret = super_stats_arenas_i_node; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -#ifdef JEMALLOC_STATS -CTL_RO_GEN(stats_cactive, &stats_cactive, size_t *) -CTL_RO_GEN(stats_allocated, ctl_stats.allocated, size_t) -CTL_RO_GEN(stats_active, ctl_stats.active, size_t) -CTL_RO_GEN(stats_mapped, ctl_stats.mapped, size_t) -#endif - -/******************************************************************************/ - -#ifdef JEMALLOC_SWAP -# ifdef JEMALLOC_STATS -CTL_RO_GEN(swap_avail, ctl_stats.swap_avail, size_t) -# endif - -static int -swap_prezeroed_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - int ret; - - malloc_mutex_lock(&ctl_mtx); - if (swap_enabled) { - READONLY(); - } else { - /* - * swap_prezeroed isn't actually used by the swap code until it - * is set during a successful chunk_swap_enabled() call. We - * use it here to store the value that we'll pass to - * chunk_swap_enable() in a swap.fds mallctl(). This is not - * very clean, but the obvious alternatives are even worse. - */ - WRITE(swap_prezeroed, bool); - } - - READ(swap_prezeroed, bool); - - ret = 0; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} - -CTL_RO_GEN(swap_nfds, swap_nfds, size_t) - -static int -swap_fds_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) -{ - int ret; - - malloc_mutex_lock(&ctl_mtx); - if (swap_enabled) { - READONLY(); - } else if (newp != NULL) { - size_t nfds = newlen / sizeof(int); - - { - int fds[nfds]; - - memcpy(fds, newp, nfds * sizeof(int)); - if (chunk_swap_enable(fds, nfds, swap_prezeroed)) { - ret = EFAULT; - goto RETURN; - } - } - } - - if (oldp != NULL && oldlenp != NULL) { - if (*oldlenp != swap_nfds * sizeof(int)) { - size_t copylen = (swap_nfds * sizeof(int) <= *oldlenp) - ? swap_nfds * sizeof(int) : *oldlenp; - - memcpy(oldp, swap_fds, copylen); - ret = EINVAL; - goto RETURN; - } else - memcpy(oldp, swap_fds, *oldlenp); - } - - ret = 0; -RETURN: - malloc_mutex_unlock(&ctl_mtx); - return (ret); -} -#endif diff --git a/deps/jemalloc.orig/src/extent.c b/deps/jemalloc.orig/src/extent.c deleted file mode 100644 index 3c04d3aa5d1..00000000000 --- a/deps/jemalloc.orig/src/extent.c +++ /dev/null @@ -1,41 +0,0 @@ -#define JEMALLOC_EXTENT_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ - -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) -static inline int -extent_szad_comp(extent_node_t *a, extent_node_t *b) -{ - int ret; - size_t a_size = a->size; - size_t b_size = b->size; - - ret = (a_size > b_size) - (a_size < b_size); - if (ret == 0) { - uintptr_t a_addr = (uintptr_t)a->addr; - uintptr_t b_addr = (uintptr_t)b->addr; - - ret = (a_addr > b_addr) - (a_addr < b_addr); - } - - return (ret); -} - -/* Generate red-black tree functions. */ -rb_gen(, extent_tree_szad_, extent_tree_t, extent_node_t, link_szad, - extent_szad_comp) -#endif - -static inline int -extent_ad_comp(extent_node_t *a, extent_node_t *b) -{ - uintptr_t a_addr = (uintptr_t)a->addr; - uintptr_t b_addr = (uintptr_t)b->addr; - - return ((a_addr > b_addr) - (a_addr < b_addr)); -} - -/* Generate red-black tree functions. */ -rb_gen(, extent_tree_ad_, extent_tree_t, extent_node_t, link_ad, - extent_ad_comp) diff --git a/deps/jemalloc.orig/src/hash.c b/deps/jemalloc.orig/src/hash.c deleted file mode 100644 index cfa4da0275c..00000000000 --- a/deps/jemalloc.orig/src/hash.c +++ /dev/null @@ -1,2 +0,0 @@ -#define JEMALLOC_HASH_C_ -#include "jemalloc/internal/jemalloc_internal.h" diff --git a/deps/jemalloc.orig/src/huge.c b/deps/jemalloc.orig/src/huge.c deleted file mode 100644 index a4f9b054ed5..00000000000 --- a/deps/jemalloc.orig/src/huge.c +++ /dev/null @@ -1,386 +0,0 @@ -#define JEMALLOC_HUGE_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -#ifdef JEMALLOC_STATS -uint64_t huge_nmalloc; -uint64_t huge_ndalloc; -size_t huge_allocated; -#endif - -malloc_mutex_t huge_mtx; - -/******************************************************************************/ - -/* Tree of chunks that are stand-alone huge allocations. */ -static extent_tree_t huge; - -void * -huge_malloc(size_t size, bool zero) -{ - void *ret; - size_t csize; - extent_node_t *node; - - /* Allocate one or more contiguous chunks for this request. */ - - csize = CHUNK_CEILING(size); - if (csize == 0) { - /* size is large enough to cause size_t wrap-around. */ - return (NULL); - } - - /* Allocate an extent node with which to track the chunk. */ - node = base_node_alloc(); - if (node == NULL) - return (NULL); - - ret = chunk_alloc(csize, false, &zero); - if (ret == NULL) { - base_node_dealloc(node); - return (NULL); - } - - /* Insert node into huge. */ - node->addr = ret; - node->size = csize; - - malloc_mutex_lock(&huge_mtx); - extent_tree_ad_insert(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_add(csize); - huge_nmalloc++; - huge_allocated += csize; -#endif - malloc_mutex_unlock(&huge_mtx); - -#ifdef JEMALLOC_FILL - if (zero == false) { - if (opt_junk) - memset(ret, 0xa5, csize); - else if (opt_zero) - memset(ret, 0, csize); - } -#endif - - return (ret); -} - -/* Only handles large allocations that require more than chunk alignment. */ -void * -huge_palloc(size_t size, size_t alignment, bool zero) -{ - void *ret; - size_t alloc_size, chunk_size, offset; - extent_node_t *node; - - /* - * This allocation requires alignment that is even larger than chunk - * alignment. This means that huge_malloc() isn't good enough. - * - * Allocate almost twice as many chunks as are demanded by the size or - * alignment, in order to assure the alignment can be achieved, then - * unmap leading and trailing chunks. - */ - assert(alignment > chunksize); - - chunk_size = CHUNK_CEILING(size); - - if (size >= alignment) - alloc_size = chunk_size + alignment - chunksize; - else - alloc_size = (alignment << 1) - chunksize; - - /* Allocate an extent node with which to track the chunk. */ - node = base_node_alloc(); - if (node == NULL) - return (NULL); - - ret = chunk_alloc(alloc_size, false, &zero); - if (ret == NULL) { - base_node_dealloc(node); - return (NULL); - } - - offset = (uintptr_t)ret & (alignment - 1); - assert((offset & chunksize_mask) == 0); - assert(offset < alloc_size); - if (offset == 0) { - /* Trim trailing space. */ - chunk_dealloc((void *)((uintptr_t)ret + chunk_size), alloc_size - - chunk_size, true); - } else { - size_t trailsize; - - /* Trim leading space. */ - chunk_dealloc(ret, alignment - offset, true); - - ret = (void *)((uintptr_t)ret + (alignment - offset)); - - trailsize = alloc_size - (alignment - offset) - chunk_size; - if (trailsize != 0) { - /* Trim trailing space. */ - assert(trailsize < alloc_size); - chunk_dealloc((void *)((uintptr_t)ret + chunk_size), - trailsize, true); - } - } - - /* Insert node into huge. */ - node->addr = ret; - node->size = chunk_size; - - malloc_mutex_lock(&huge_mtx); - extent_tree_ad_insert(&huge, node); -#ifdef JEMALLOC_STATS - stats_cactive_add(chunk_size); - huge_nmalloc++; - huge_allocated += chunk_size; -#endif - malloc_mutex_unlock(&huge_mtx); - -#ifdef JEMALLOC_FILL - if (zero == false) { - if (opt_junk) - memset(ret, 0xa5, chunk_size); - else if (opt_zero) - memset(ret, 0, chunk_size); - } -#endif - - return (ret); -} - -void * -huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) -{ - - /* - * Avoid moving the allocation if the size class can be left the same. - */ - if (oldsize > arena_maxclass - && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) - && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { - assert(CHUNK_CEILING(oldsize) == oldsize); -#ifdef JEMALLOC_FILL - if (opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), 0x5a, - oldsize - size); - } -#endif - return (ptr); - } - - /* Reallocation would require a move. */ - return (NULL); -} - -void * -huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) -{ - void *ret; - size_t copysize; - - /* Try to avoid moving the allocation. */ - ret = huge_ralloc_no_move(ptr, oldsize, size, extra); - if (ret != NULL) - return (ret); - - /* - * size and oldsize are different enough that we need to use a - * different size class. In that case, fall back to allocating new - * space and copying. - */ - if (alignment > chunksize) - ret = huge_palloc(size + extra, alignment, zero); - else - ret = huge_malloc(size + extra, zero); - - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, this time without extra. */ - if (alignment > chunksize) - ret = huge_palloc(size, alignment, zero); - else - ret = huge_malloc(size, zero); - - if (ret == NULL) - return (NULL); - } - - /* - * Copy at most size bytes (not size+extra), since the caller has no - * expectation that the extra bytes will be reliably preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - - /* - * Use mremap(2) if this is a huge-->huge reallocation, and neither the - * source nor the destination are in swap or dss. - */ -#ifdef JEMALLOC_MREMAP_FIXED - if (oldsize >= chunksize -# ifdef JEMALLOC_SWAP - && (swap_enabled == false || (chunk_in_swap(ptr) == false && - chunk_in_swap(ret) == false)) -# endif -# ifdef JEMALLOC_DSS - && chunk_in_dss(ptr) == false && chunk_in_dss(ret) == false -# endif - ) { - size_t newsize = huge_salloc(ret); - - /* - * Remove ptr from the tree of huge allocations before - * performing the remap operation, in order to avoid the - * possibility of another thread acquiring that mapping before - * this one removes it from the tree. - */ - huge_dalloc(ptr, false); - if (mremap(ptr, oldsize, newsize, MREMAP_MAYMOVE|MREMAP_FIXED, - ret) == MAP_FAILED) { - /* - * Assuming no chunk management bugs in the allocator, - * the only documented way an error can occur here is - * if the application changed the map type for a - * portion of the old allocation. This is firmly in - * undefined behavior territory, so write a diagnostic - * message, and optionally abort. - */ - char buf[BUFERROR_BUF]; - - buferror(errno, buf, sizeof(buf)); - malloc_write(": Error in mremap(): "); - malloc_write(buf); - malloc_write("\n"); - if (opt_abort) - abort(); - memcpy(ret, ptr, copysize); - chunk_dealloc_mmap(ptr, oldsize); - } - } else -#endif - { - memcpy(ret, ptr, copysize); - idalloc(ptr); - } - return (ret); -} - -void -huge_dalloc(void *ptr, bool unmap) -{ - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = ptr; - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - assert(node->addr == ptr); - extent_tree_ad_remove(&huge, node); - -#ifdef JEMALLOC_STATS - stats_cactive_sub(node->size); - huge_ndalloc++; - huge_allocated -= node->size; -#endif - - malloc_mutex_unlock(&huge_mtx); - - if (unmap) { - /* Unmap chunk. */ -#ifdef JEMALLOC_FILL -#if (defined(JEMALLOC_SWAP) || defined(JEMALLOC_DSS)) - if (opt_junk) - memset(node->addr, 0x5a, node->size); -#endif -#endif - } - - chunk_dealloc(node->addr, node->size, unmap); - - base_node_dealloc(node); -} - -size_t -huge_salloc(const void *ptr) -{ - size_t ret; - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - ret = node->size; - - malloc_mutex_unlock(&huge_mtx); - - return (ret); -} - -#ifdef JEMALLOC_PROF -prof_ctx_t * -huge_prof_ctx_get(const void *ptr) -{ - prof_ctx_t *ret; - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - ret = node->prof_ctx; - - malloc_mutex_unlock(&huge_mtx); - - return (ret); -} - -void -huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) -{ - extent_node_t *node, key; - - malloc_mutex_lock(&huge_mtx); - - /* Extract from tree of huge allocations. */ - key.addr = __DECONST(void *, ptr); - node = extent_tree_ad_search(&huge, &key); - assert(node != NULL); - - node->prof_ctx = ctx; - - malloc_mutex_unlock(&huge_mtx); -} -#endif - -bool -huge_boot(void) -{ - - /* Initialize chunks data. */ - if (malloc_mutex_init(&huge_mtx)) - return (true); - extent_tree_ad_new(&huge); - -#ifdef JEMALLOC_STATS - huge_nmalloc = 0; - huge_ndalloc = 0; - huge_allocated = 0; -#endif - - return (false); -} diff --git a/deps/jemalloc.orig/src/jemalloc.c b/deps/jemalloc.orig/src/jemalloc.c deleted file mode 100644 index a161c2e26e1..00000000000 --- a/deps/jemalloc.orig/src/jemalloc.c +++ /dev/null @@ -1,1881 +0,0 @@ -#define JEMALLOC_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -malloc_mutex_t arenas_lock; -arena_t **arenas; -unsigned narenas; - -pthread_key_t arenas_tsd; -#ifndef NO_TLS -__thread arena_t *arenas_tls JEMALLOC_ATTR(tls_model("initial-exec")); -#endif - -#ifdef JEMALLOC_STATS -# ifndef NO_TLS -__thread thread_allocated_t thread_allocated_tls; -# else -pthread_key_t thread_allocated_tsd; -# endif -#endif - -/* Set to true once the allocator has been initialized. */ -static bool malloc_initialized = false; - -/* Used to let the initializing thread recursively allocate. */ -static pthread_t malloc_initializer = (unsigned long)0; - -/* Used to avoid initialization races. */ -static malloc_mutex_t init_lock = -#ifdef JEMALLOC_OSSPIN - 0 -#else - MALLOC_MUTEX_INITIALIZER -#endif - ; - -#ifdef DYNAMIC_PAGE_SHIFT -size_t pagesize; -size_t pagesize_mask; -size_t lg_pagesize; -#endif - -unsigned ncpus; - -/* Runtime configuration options. */ -const char *JEMALLOC_P(malloc_conf) JEMALLOC_ATTR(visibility("default")); -#ifdef JEMALLOC_DEBUG -bool opt_abort = true; -# ifdef JEMALLOC_FILL -bool opt_junk = true; -# endif -#else -bool opt_abort = false; -# ifdef JEMALLOC_FILL -bool opt_junk = false; -# endif -#endif -#ifdef JEMALLOC_SYSV -bool opt_sysv = false; -#endif -#ifdef JEMALLOC_XMALLOC -bool opt_xmalloc = false; -#endif -#ifdef JEMALLOC_FILL -bool opt_zero = false; -#endif -size_t opt_narenas = 0; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void wrtmessage(void *cbopaque, const char *s); -static void stats_print_atexit(void); -static unsigned malloc_ncpus(void); -static void arenas_cleanup(void *arg); -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -static void thread_allocated_cleanup(void *arg); -#endif -static bool malloc_conf_next(char const **opts_p, char const **k_p, - size_t *klen_p, char const **v_p, size_t *vlen_p); -static void malloc_conf_error(const char *msg, const char *k, size_t klen, - const char *v, size_t vlen); -static void malloc_conf_init(void); -static bool malloc_init_hard(void); -static int imemalign(void **memptr, size_t alignment, size_t size); - -/******************************************************************************/ -/* malloc_message() setup. */ - -#ifdef JEMALLOC_HAVE_ATTR -JEMALLOC_ATTR(visibility("hidden")) -#else -static -#endif -void -wrtmessage(void *cbopaque, const char *s) -{ -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - write(STDERR_FILENO, s, strlen(s)); -#ifdef JEMALLOC_CC_SILENCE - if (result < 0) - result = errno; -#endif -} - -void (*JEMALLOC_P(malloc_message))(void *, const char *s) - JEMALLOC_ATTR(visibility("default")) = wrtmessage; - -/******************************************************************************/ -/* - * Begin miscellaneous support functions. - */ - -/* Create a new arena and insert it into the arenas array at index ind. */ -arena_t * -arenas_extend(unsigned ind) -{ - arena_t *ret; - - /* Allocate enough space for trailing bins. */ - ret = (arena_t *)base_alloc(offsetof(arena_t, bins) - + (sizeof(arena_bin_t) * nbins)); - if (ret != NULL && arena_new(ret, ind) == false) { - arenas[ind] = ret; - return (ret); - } - /* Only reached if there is an OOM error. */ - - /* - * OOM here is quite inconvenient to propagate, since dealing with it - * would require a check for failure in the fast path. Instead, punt - * by using arenas[0]. In practice, this is an extremely unlikely - * failure. - */ - malloc_write(": Error initializing arena\n"); - if (opt_abort) - abort(); - - return (arenas[0]); -} - -/* - * Choose an arena based on a per-thread value (slow-path code only, called - * only by choose_arena()). - */ -arena_t * -choose_arena_hard(void) -{ - arena_t *ret; - - if (narenas > 1) { - unsigned i, choose, first_null; - - choose = 0; - first_null = narenas; - malloc_mutex_lock(&arenas_lock); - assert(arenas[0] != NULL); - for (i = 1; i < narenas; i++) { - if (arenas[i] != NULL) { - /* - * Choose the first arena that has the lowest - * number of threads assigned to it. - */ - if (arenas[i]->nthreads < - arenas[choose]->nthreads) - choose = i; - } else if (first_null == narenas) { - /* - * Record the index of the first uninitialized - * arena, in case all extant arenas are in use. - * - * NB: It is possible for there to be - * discontinuities in terms of initialized - * versus uninitialized arenas, due to the - * "thread.arena" mallctl. - */ - first_null = i; - } - } - - if (arenas[choose] == 0 || first_null == narenas) { - /* - * Use an unloaded arena, or the least loaded arena if - * all arenas are already initialized. - */ - ret = arenas[choose]; - } else { - /* Initialize a new arena. */ - ret = arenas_extend(first_null); - } - ret->nthreads++; - malloc_mutex_unlock(&arenas_lock); - } else { - ret = arenas[0]; - malloc_mutex_lock(&arenas_lock); - ret->nthreads++; - malloc_mutex_unlock(&arenas_lock); - } - - ARENA_SET(ret); - - return (ret); -} - -/* - * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so - * provide a wrapper. - */ -int -buferror(int errnum, char *buf, size_t buflen) -{ -#ifdef _GNU_SOURCE - char *b = strerror_r(errno, buf, buflen); - if (b != buf) { - strncpy(buf, b, buflen); - buf[buflen-1] = '\0'; - } - return (0); -#else - return (strerror_r(errno, buf, buflen)); -#endif -} - -static void -stats_print_atexit(void) -{ - -#if (defined(JEMALLOC_TCACHE) && defined(JEMALLOC_STATS)) - unsigned i; - - /* - * Merge stats from extant threads. This is racy, since individual - * threads do not lock when recording tcache stats events. As a - * consequence, the final stats may be slightly out of date by the time - * they are reported, if other threads continue to allocate. - */ - for (i = 0; i < narenas; i++) { - arena_t *arena = arenas[i]; - if (arena != NULL) { - tcache_t *tcache; - - /* - * tcache_stats_merge() locks bins, so if any code is - * introduced that acquires both arena and bin locks in - * the opposite order, deadlocks may result. - */ - malloc_mutex_lock(&arena->lock); - ql_foreach(tcache, &arena->tcache_ql, link) { - tcache_stats_merge(tcache, arena); - } - malloc_mutex_unlock(&arena->lock); - } - } -#endif - JEMALLOC_P(malloc_stats_print)(NULL, NULL, NULL); -} - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -thread_allocated_t * -thread_allocated_get_hard(void) -{ - thread_allocated_t *thread_allocated = (thread_allocated_t *) - imalloc(sizeof(thread_allocated_t)); - if (thread_allocated == NULL) { - static thread_allocated_t static_thread_allocated = {0, 0}; - malloc_write(": Error allocating TSD;" - " mallctl(\"thread.{de,}allocated[p]\", ...)" - " will be inaccurate\n"); - if (opt_abort) - abort(); - return (&static_thread_allocated); - } - pthread_setspecific(thread_allocated_tsd, thread_allocated); - thread_allocated->allocated = 0; - thread_allocated->deallocated = 0; - return (thread_allocated); -} -#endif - -/* - * End miscellaneous support functions. - */ -/******************************************************************************/ -/* - * Begin initialization functions. - */ - -static unsigned -malloc_ncpus(void) -{ - unsigned ret; - long result; - - result = sysconf(_SC_NPROCESSORS_ONLN); - if (result == -1) { - /* Error. */ - ret = 1; - } - ret = (unsigned)result; - - return (ret); -} - -static void -arenas_cleanup(void *arg) -{ - arena_t *arena = (arena_t *)arg; - - malloc_mutex_lock(&arenas_lock); - arena->nthreads--; - malloc_mutex_unlock(&arenas_lock); -} - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) -static void -thread_allocated_cleanup(void *arg) -{ - uint64_t *allocated = (uint64_t *)arg; - - if (allocated != NULL) - idalloc(allocated); -} -#endif - -/* - * FreeBSD's pthreads implementation calls malloc(3), so the malloc - * implementation has to take pains to avoid infinite recursion during - * initialization. - */ -static inline bool -malloc_init(void) -{ - - if (malloc_initialized == false) - return (malloc_init_hard()); - - return (false); -} - -static bool -malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, - char const **v_p, size_t *vlen_p) -{ - bool accept; - const char *opts = *opts_p; - - *k_p = opts; - - for (accept = false; accept == false;) { - switch (*opts) { - case 'A': case 'B': case 'C': case 'D': case 'E': - case 'F': case 'G': case 'H': case 'I': case 'J': - case 'K': case 'L': case 'M': case 'N': case 'O': - case 'P': case 'Q': case 'R': case 'S': case 'T': - case 'U': case 'V': case 'W': case 'X': case 'Y': - case 'Z': - case 'a': case 'b': case 'c': case 'd': case 'e': - case 'f': case 'g': case 'h': case 'i': case 'j': - case 'k': case 'l': case 'm': case 'n': case 'o': - case 'p': case 'q': case 'r': case 's': case 't': - case 'u': case 'v': case 'w': case 'x': case 'y': - case 'z': - case '0': case '1': case '2': case '3': case '4': - case '5': case '6': case '7': case '8': case '9': - case '_': - opts++; - break; - case ':': - opts++; - *klen_p = (uintptr_t)opts - 1 - (uintptr_t)*k_p; - *v_p = opts; - accept = true; - break; - case '\0': - if (opts != *opts_p) { - malloc_write(": Conf string " - "ends with key\n"); - } - return (true); - default: - malloc_write(": Malformed conf " - "string\n"); - return (true); - } - } - - for (accept = false; accept == false;) { - switch (*opts) { - case ',': - opts++; - /* - * Look ahead one character here, because the - * next time this function is called, it will - * assume that end of input has been cleanly - * reached if no input remains, but we have - * optimistically already consumed the comma if - * one exists. - */ - if (*opts == '\0') { - malloc_write(": Conf string " - "ends with comma\n"); - } - *vlen_p = (uintptr_t)opts - 1 - (uintptr_t)*v_p; - accept = true; - break; - case '\0': - *vlen_p = (uintptr_t)opts - (uintptr_t)*v_p; - accept = true; - break; - default: - opts++; - break; - } - } - - *opts_p = opts; - return (false); -} - -static void -malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, - size_t vlen) -{ - char buf[PATH_MAX + 1]; - - malloc_write(": "); - malloc_write(msg); - malloc_write(": "); - memcpy(buf, k, klen); - memcpy(&buf[klen], ":", 1); - memcpy(&buf[klen+1], v, vlen); - buf[klen+1+vlen] = '\0'; - malloc_write(buf); - malloc_write("\n"); -} - -static void -malloc_conf_init(void) -{ - unsigned i; - char buf[PATH_MAX + 1]; - const char *opts, *k, *v; - size_t klen, vlen; - - for (i = 0; i < 3; i++) { - /* Get runtime configuration. */ - switch (i) { - case 0: - if (JEMALLOC_P(malloc_conf) != NULL) { - /* - * Use options that were compiled into the - * program. - */ - opts = JEMALLOC_P(malloc_conf); - } else { - /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; - } - break; - case 1: { - int linklen; - const char *linkname = -#ifdef JEMALLOC_PREFIX - "/etc/"JEMALLOC_PREFIX"malloc.conf" -#else - "/etc/malloc.conf" -#endif - ; - - if ((linklen = readlink(linkname, buf, - sizeof(buf) - 1)) != -1) { - /* - * Use the contents of the "/etc/malloc.conf" - * symbolic link's name. - */ - buf[linklen] = '\0'; - opts = buf; - } else { - /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; - } - break; - } - case 2: { - const char *envname = -#ifdef JEMALLOC_PREFIX - JEMALLOC_CPREFIX"MALLOC_CONF" -#else - "MALLOC_CONF" -#endif - ; - - if ((opts = getenv(envname)) != NULL) { - /* - * Do nothing; opts is already initialized to - * the value of the MALLOC_CONF environment - * variable. - */ - } else { - /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; - } - break; - } - default: - /* NOTREACHED */ - assert(false); - buf[0] = '\0'; - opts = buf; - } - - while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, - &vlen) == false) { -#define CONF_HANDLE_BOOL(n) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ - klen) == 0) { \ - if (strncmp("true", v, vlen) == 0 && \ - vlen == sizeof("true")-1) \ - opt_##n = true; \ - else if (strncmp("false", v, vlen) == \ - 0 && vlen == sizeof("false")-1) \ - opt_##n = false; \ - else { \ - malloc_conf_error( \ - "Invalid conf value", \ - k, klen, v, vlen); \ - } \ - continue; \ - } -#define CONF_HANDLE_SIZE_T(n, min, max) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ - klen) == 0) { \ - unsigned long ul; \ - char *end; \ - \ - errno = 0; \ - ul = strtoul(v, &end, 0); \ - if (errno != 0 || (uintptr_t)end - \ - (uintptr_t)v != vlen) { \ - malloc_conf_error( \ - "Invalid conf value", \ - k, klen, v, vlen); \ - } else if (ul < min || ul > max) { \ - malloc_conf_error( \ - "Out-of-range conf value", \ - k, klen, v, vlen); \ - } else \ - opt_##n = ul; \ - continue; \ - } -#define CONF_HANDLE_SSIZE_T(n, min, max) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ - klen) == 0) { \ - long l; \ - char *end; \ - \ - errno = 0; \ - l = strtol(v, &end, 0); \ - if (errno != 0 || (uintptr_t)end - \ - (uintptr_t)v != vlen) { \ - malloc_conf_error( \ - "Invalid conf value", \ - k, klen, v, vlen); \ - } else if (l < (ssize_t)min || l > \ - (ssize_t)max) { \ - malloc_conf_error( \ - "Out-of-range conf value", \ - k, klen, v, vlen); \ - } else \ - opt_##n = l; \ - continue; \ - } -#define CONF_HANDLE_CHAR_P(n, d) \ - if (sizeof(#n)-1 == klen && strncmp(#n, k, \ - klen) == 0) { \ - size_t cpylen = (vlen <= \ - sizeof(opt_##n)-1) ? vlen : \ - sizeof(opt_##n)-1; \ - strncpy(opt_##n, v, cpylen); \ - opt_##n[cpylen] = '\0'; \ - continue; \ - } - - CONF_HANDLE_BOOL(abort) - CONF_HANDLE_SIZE_T(lg_qspace_max, LG_QUANTUM, - PAGE_SHIFT-1) - CONF_HANDLE_SIZE_T(lg_cspace_max, LG_QUANTUM, - PAGE_SHIFT-1) - /* - * Chunks always require at least one * header page, - * plus one data page. - */ - CONF_HANDLE_SIZE_T(lg_chunk, PAGE_SHIFT+1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SIZE_T(narenas, 1, SIZE_T_MAX) - CONF_HANDLE_SSIZE_T(lg_dirty_mult, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_BOOL(stats_print) -#ifdef JEMALLOC_FILL - CONF_HANDLE_BOOL(junk) - CONF_HANDLE_BOOL(zero) -#endif -#ifdef JEMALLOC_SYSV - CONF_HANDLE_BOOL(sysv) -#endif -#ifdef JEMALLOC_XMALLOC - CONF_HANDLE_BOOL(xmalloc) -#endif -#ifdef JEMALLOC_TCACHE - CONF_HANDLE_BOOL(tcache) - CONF_HANDLE_SSIZE_T(lg_tcache_gc_sweep, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SSIZE_T(lg_tcache_max, -1, - (sizeof(size_t) << 3) - 1) -#endif -#ifdef JEMALLOC_PROF - CONF_HANDLE_BOOL(prof) - CONF_HANDLE_CHAR_P(prof_prefix, "jeprof") - CONF_HANDLE_SIZE_T(lg_prof_bt_max, 0, LG_PROF_BT_MAX) - CONF_HANDLE_BOOL(prof_active) - CONF_HANDLE_SSIZE_T(lg_prof_sample, 0, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(prof_accum) - CONF_HANDLE_SSIZE_T(lg_prof_tcmax, -1, - (sizeof(size_t) << 3) - 1) - CONF_HANDLE_SSIZE_T(lg_prof_interval, -1, - (sizeof(uint64_t) << 3) - 1) - CONF_HANDLE_BOOL(prof_gdump) - CONF_HANDLE_BOOL(prof_leak) -#endif -#ifdef JEMALLOC_SWAP - CONF_HANDLE_BOOL(overcommit) -#endif - malloc_conf_error("Invalid conf pair", k, klen, v, - vlen); -#undef CONF_HANDLE_BOOL -#undef CONF_HANDLE_SIZE_T -#undef CONF_HANDLE_SSIZE_T -#undef CONF_HANDLE_CHAR_P - } - - /* Validate configuration of options that are inter-related. */ - if (opt_lg_qspace_max+1 >= opt_lg_cspace_max) { - malloc_write(": Invalid lg_[qc]space_max " - "relationship; restoring defaults\n"); - opt_lg_qspace_max = LG_QSPACE_MAX_DEFAULT; - opt_lg_cspace_max = LG_CSPACE_MAX_DEFAULT; - } - } -} - -static bool -malloc_init_hard(void) -{ - arena_t *init_arenas[1]; - - malloc_mutex_lock(&init_lock); - if (malloc_initialized || malloc_initializer == pthread_self()) { - /* - * Another thread initialized the allocator before this one - * acquired init_lock, or this thread is the initializing - * thread, and it is recursively allocating. - */ - malloc_mutex_unlock(&init_lock); - return (false); - } - if (malloc_initializer != (unsigned long)0) { - /* Busy-wait until the initializing thread completes. */ - do { - malloc_mutex_unlock(&init_lock); - CPU_SPINWAIT; - malloc_mutex_lock(&init_lock); - } while (malloc_initialized == false); - malloc_mutex_unlock(&init_lock); - return (false); - } - -#ifdef DYNAMIC_PAGE_SHIFT - /* Get page size. */ - { - long result; - - result = sysconf(_SC_PAGESIZE); - assert(result != -1); - pagesize = (size_t)result; - - /* - * We assume that pagesize is a power of 2 when calculating - * pagesize_mask and lg_pagesize. - */ - assert(((result - 1) & result) == 0); - pagesize_mask = result - 1; - lg_pagesize = ffs((int)result) - 1; - } -#endif - -#ifdef JEMALLOC_PROF - prof_boot0(); -#endif - - malloc_conf_init(); - - /* Register fork handlers. */ - if (pthread_atfork(jemalloc_prefork, jemalloc_postfork, - jemalloc_postfork) != 0) { - malloc_write(": Error in pthread_atfork()\n"); - if (opt_abort) - abort(); - } - - if (ctl_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - if (opt_stats_print) { - /* Print statistics at exit. */ - if (atexit(stats_print_atexit) != 0) { - malloc_write(": Error in atexit()\n"); - if (opt_abort) - abort(); - } - } - - if (chunk_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - if (base_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - -#ifdef JEMALLOC_PROF - prof_boot1(); -#endif - - if (arena_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - -#ifdef JEMALLOC_TCACHE - if (tcache_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } -#endif - - if (huge_boot()) { - malloc_mutex_unlock(&init_lock); - return (true); - } - -#if (defined(JEMALLOC_STATS) && defined(NO_TLS)) - /* Initialize allocation counters before any allocations can occur. */ - if (pthread_key_create(&thread_allocated_tsd, thread_allocated_cleanup) - != 0) { - malloc_mutex_unlock(&init_lock); - return (true); - } -#endif - - if (malloc_mutex_init(&arenas_lock)) - return (true); - - if (pthread_key_create(&arenas_tsd, arenas_cleanup) != 0) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - /* - * Create enough scaffolding to allow recursive allocation in - * malloc_ncpus(). - */ - narenas = 1; - arenas = init_arenas; - memset(arenas, 0, sizeof(arena_t *) * narenas); - - /* - * Initialize one arena here. The rest are lazily created in - * choose_arena_hard(). - */ - arenas_extend(0); - if (arenas[0] == NULL) { - malloc_mutex_unlock(&init_lock); - return (true); - } - - /* - * Assign the initial arena to the initial thread, in order to avoid - * spurious creation of an extra arena if the application switches to - * threaded mode. - */ - ARENA_SET(arenas[0]); - arenas[0]->nthreads++; - -#ifdef JEMALLOC_PROF - if (prof_boot2()) { - malloc_mutex_unlock(&init_lock); - return (true); - } -#endif - - /* Get number of CPUs. */ - malloc_initializer = pthread_self(); - malloc_mutex_unlock(&init_lock); - ncpus = malloc_ncpus(); - malloc_mutex_lock(&init_lock); - - if (opt_narenas == 0) { - /* - * For SMP systems, create more than one arena per CPU by - * default. - */ - if (ncpus > 1) - opt_narenas = ncpus << 2; - else - opt_narenas = 1; - } - narenas = opt_narenas; - /* - * Make sure that the arenas array can be allocated. In practice, this - * limit is enough to allow the allocator to function, but the ctl - * machinery will fail to allocate memory at far lower limits. - */ - if (narenas > chunksize / sizeof(arena_t *)) { - char buf[UMAX2S_BUFSIZE]; - - narenas = chunksize / sizeof(arena_t *); - malloc_write(": Reducing narenas to limit ("); - malloc_write(u2s(narenas, 10, buf)); - malloc_write(")\n"); - } - - /* Allocate and initialize arenas. */ - arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); - if (arenas == NULL) { - malloc_mutex_unlock(&init_lock); - return (true); - } - /* - * Zero the array. In practice, this should always be pre-zeroed, - * since it was just mmap()ed, but let's be sure. - */ - memset(arenas, 0, sizeof(arena_t *) * narenas); - /* Copy the pointer to the one arena that was already initialized. */ - arenas[0] = init_arenas[0]; - -#ifdef JEMALLOC_ZONE - /* Register the custom zone. */ - malloc_zone_register(create_zone()); - - /* - * Convert the default szone to an "overlay zone" that is capable of - * deallocating szone-allocated objects, but allocating new objects - * from jemalloc. - */ - szone2ozone(malloc_default_zone()); -#endif - - malloc_initialized = true; - malloc_mutex_unlock(&init_lock); - return (false); -} - -#ifdef JEMALLOC_ZONE -JEMALLOC_ATTR(constructor) -void -jemalloc_darwin_init(void) -{ - - if (malloc_init_hard()) - abort(); -} -#endif - -/* - * End initialization functions. - */ -/******************************************************************************/ -/* - * Begin malloc(3)-compatible functions. - */ - -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) -void * -JEMALLOC_P(malloc)(size_t size) -{ - void *ret; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif - - if (malloc_init()) { - ret = NULL; - goto OOM; - } - - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { -# ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in malloc(): " - "invalid size 0\n"); - abort(); - } -# endif - ret = NULL; - goto RETURN; - } -#endif - } - -#ifdef JEMALLOC_PROF - if (opt_prof) { - usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - ret = NULL; - goto OOM; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { - ret = imalloc(small_maxclass+1); - if (ret != NULL) - arena_prof_promoted(ret, usize); - } else - ret = imalloc(size); - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif - ret = imalloc(size); - } - -OOM: - if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in malloc(): " - "out of memory\n"); - abort(); - } -#endif - errno = ENOMEM; - } - -#ifdef JEMALLOC_SYSV -RETURN: -#endif -#ifdef JEMALLOC_PROF - if (opt_prof && ret != NULL) - prof_malloc(ret, usize, cnt); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, 0); - } -#endif - return (ret); -} - -JEMALLOC_ATTR(nonnull(1)) -#ifdef JEMALLOC_PROF -/* - * Avoid any uncertainty as to how many backtrace frames to ignore in - * PROF_ALLOC_PREP(). - */ -JEMALLOC_ATTR(noinline) -#endif -static int -imemalign(void **memptr, size_t alignment, size_t size) -{ - int ret; - size_t usize -#ifdef JEMALLOC_CC_SILENCE - = 0 -#endif - ; - void *result; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif - - if (malloc_init()) - result = NULL; - else { - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { -# ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in " - "posix_memalign(): invalid size " - "0\n"); - abort(); - } -# endif - result = NULL; - *memptr = NULL; - ret = 0; - goto RETURN; - } -#endif - } - - /* Make sure that alignment is a large enough power of 2. */ - if (((alignment - 1) & alignment) != 0 - || alignment < sizeof(void *)) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in " - "posix_memalign(): invalid alignment\n"); - abort(); - } -#endif - result = NULL; - ret = EINVAL; - goto RETURN; - } - - usize = sa2u(size, alignment, NULL); - if (usize == 0) { - result = NULL; - ret = ENOMEM; - goto RETURN; - } - -#ifdef JEMALLOC_PROF - if (opt_prof) { - PROF_ALLOC_PREP(2, usize, cnt); - if (cnt == NULL) { - result = NULL; - ret = EINVAL; - } else { - if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= small_maxclass) { - assert(sa2u(small_maxclass+1, - alignment, NULL) != 0); - result = ipalloc(sa2u(small_maxclass+1, - alignment, NULL), alignment, false); - if (result != NULL) { - arena_prof_promoted(result, - usize); - } - } else { - result = ipalloc(usize, alignment, - false); - } - } - } else -#endif - result = ipalloc(usize, alignment, false); - } - - if (result == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in posix_memalign(): " - "out of memory\n"); - abort(); - } -#endif - ret = ENOMEM; - goto RETURN; - } - - *memptr = result; - ret = 0; - -RETURN: -#ifdef JEMALLOC_STATS - if (result != NULL) { - assert(usize == isalloc(result)); - ALLOCATED_ADD(usize, 0); - } -#endif -#ifdef JEMALLOC_PROF - if (opt_prof && result != NULL) - prof_malloc(result, usize, cnt); -#endif - return (ret); -} - -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(posix_memalign)(void **memptr, size_t alignment, size_t size) -{ - - return imemalign(memptr, alignment, size); -} - -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) -void * -JEMALLOC_P(calloc)(size_t num, size_t size) -{ - void *ret; - size_t num_size; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif - - if (malloc_init()) { - num_size = 0; - ret = NULL; - goto RETURN; - } - - num_size = num * size; - if (num_size == 0) { -#ifdef JEMALLOC_SYSV - if ((opt_sysv == false) && ((num == 0) || (size == 0))) -#endif - num_size = 1; -#ifdef JEMALLOC_SYSV - else { - ret = NULL; - goto RETURN; - } -#endif - /* - * Try to avoid division here. We know that it isn't possible to - * overflow during multiplication if neither operand uses any of the - * most significant half of the bits in a size_t. - */ - } else if (((num | size) & (SIZE_T_MAX << (sizeof(size_t) << 2))) - && (num_size / size != num)) { - /* size_t overflow. */ - ret = NULL; - goto RETURN; - } - -#ifdef JEMALLOC_PROF - if (opt_prof) { - usize = s2u(num_size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - ret = NULL; - goto RETURN; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize - <= small_maxclass) { - ret = icalloc(small_maxclass+1); - if (ret != NULL) - arena_prof_promoted(ret, usize); - } else - ret = icalloc(num_size); - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(num_size); -#endif - ret = icalloc(num_size); - } - -RETURN: - if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in calloc(): out of " - "memory\n"); - abort(); - } -#endif - errno = ENOMEM; - } - -#ifdef JEMALLOC_PROF - if (opt_prof && ret != NULL) - prof_malloc(ret, usize, cnt); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, 0); - } -#endif - return (ret); -} - -JEMALLOC_ATTR(visibility("default")) -void * -JEMALLOC_P(realloc)(void *ptr, size_t size) -{ - void *ret; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize -# ifdef JEMALLOC_CC_SILENCE - = 0 -# endif - ; - size_t old_size = 0; -#endif -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; - prof_ctx_t *old_ctx -# ifdef JEMALLOC_CC_SILENCE - = NULL -# endif - ; -#endif - - if (size == 0) { -#ifdef JEMALLOC_SYSV - if (opt_sysv == false) -#endif - size = 1; -#ifdef JEMALLOC_SYSV - else { - if (ptr != NULL) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - old_size = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { - old_ctx = prof_ctx_get(ptr); - cnt = NULL; - } -#endif - idalloc(ptr); - } -#ifdef JEMALLOC_PROF - else if (opt_prof) { - old_ctx = NULL; - cnt = NULL; - } -#endif - ret = NULL; - goto RETURN; - } -#endif - } - - if (ptr != NULL) { - assert(malloc_initialized || malloc_initializer == - pthread_self()); - -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - old_size = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { - usize = s2u(size); - old_ctx = prof_ctx_get(ptr); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - old_ctx = NULL; - ret = NULL; - goto OOM; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && - usize <= small_maxclass) { - ret = iralloc(ptr, small_maxclass+1, 0, 0, - false, false); - if (ret != NULL) - arena_prof_promoted(ret, usize); - else - old_ctx = NULL; - } else { - ret = iralloc(ptr, size, 0, 0, false, false); - if (ret == NULL) - old_ctx = NULL; - } - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif - ret = iralloc(ptr, size, 0, 0, false, false); - } - -#ifdef JEMALLOC_PROF -OOM: -#endif - if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in realloc(): " - "out of memory\n"); - abort(); - } -#endif - errno = ENOMEM; - } - } else { -#ifdef JEMALLOC_PROF - if (opt_prof) - old_ctx = NULL; -#endif - if (malloc_init()) { -#ifdef JEMALLOC_PROF - if (opt_prof) - cnt = NULL; -#endif - ret = NULL; - } else { -#ifdef JEMALLOC_PROF - if (opt_prof) { - usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) - ret = NULL; - else { - if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= - small_maxclass) { - ret = imalloc(small_maxclass+1); - if (ret != NULL) { - arena_prof_promoted(ret, - usize); - } - } else - ret = imalloc(size); - } - } else -#endif - { -#ifdef JEMALLOC_STATS - usize = s2u(size); -#endif - ret = imalloc(size); - } - } - - if (ret == NULL) { -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in realloc(): " - "out of memory\n"); - abort(); - } -#endif - errno = ENOMEM; - } - } - -#ifdef JEMALLOC_SYSV -RETURN: -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) - prof_realloc(ret, usize, cnt, old_size, old_ctx); -#endif -#ifdef JEMALLOC_STATS - if (ret != NULL) { - assert(usize == isalloc(ret)); - ALLOCATED_ADD(usize, old_size); - } -#endif - return (ret); -} - -JEMALLOC_ATTR(visibility("default")) -void -JEMALLOC_P(free)(void *ptr) -{ - - if (ptr != NULL) { -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize; -#endif - - assert(malloc_initialized || malloc_initializer == - pthread_self()); - -#ifdef JEMALLOC_STATS - usize = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { -# ifndef JEMALLOC_STATS - usize = isalloc(ptr); -# endif - prof_free(ptr, usize); - } -#endif -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(0, usize); -#endif - idalloc(ptr); - } -} - -/* - * End malloc(3)-compatible functions. - */ -/******************************************************************************/ -/* - * Begin non-standard override functions. - * - * These overrides are omitted if the JEMALLOC_PREFIX is defined, since the - * entire point is to avoid accidental mixed allocator usage. - */ -#ifndef JEMALLOC_PREFIX - -#ifdef JEMALLOC_OVERRIDE_MEMALIGN -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) -void * -JEMALLOC_P(memalign)(size_t alignment, size_t size) -{ - void *ret; -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - imemalign(&ret, alignment, size); -#ifdef JEMALLOC_CC_SILENCE - if (result != 0) - return (NULL); -#endif - return (ret); -} -#endif - -#ifdef JEMALLOC_OVERRIDE_VALLOC -JEMALLOC_ATTR(malloc) -JEMALLOC_ATTR(visibility("default")) -void * -JEMALLOC_P(valloc)(size_t size) -{ - void *ret; -#ifdef JEMALLOC_CC_SILENCE - int result = -#endif - imemalign(&ret, PAGE_SIZE, size); -#ifdef JEMALLOC_CC_SILENCE - if (result != 0) - return (NULL); -#endif - return (ret); -} -#endif - -#endif /* JEMALLOC_PREFIX */ -/* - * End non-standard override functions. - */ -/******************************************************************************/ -/* - * Begin non-standard functions. - */ - -JEMALLOC_ATTR(visibility("default")) -size_t -JEMALLOC_P(malloc_usable_size)(const void *ptr) -{ - size_t ret; - - assert(malloc_initialized || malloc_initializer == pthread_self()); - -#ifdef JEMALLOC_IVSALLOC - ret = ivsalloc(ptr); -#else - assert(ptr != NULL); - ret = isalloc(ptr); -#endif - - return (ret); -} - -JEMALLOC_ATTR(visibility("default")) -void -JEMALLOC_P(malloc_stats_print)(void (*write_cb)(void *, const char *), - void *cbopaque, const char *opts) -{ - - stats_print(write_cb, cbopaque, opts); -} - -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(mallctl)(const char *name, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) -{ - - if (malloc_init()) - return (EAGAIN); - - return (ctl_byname(name, oldp, oldlenp, newp, newlen)); -} - -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(mallctlnametomib)(const char *name, size_t *mibp, size_t *miblenp) -{ - - if (malloc_init()) - return (EAGAIN); - - return (ctl_nametomib(name, mibp, miblenp)); -} - -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(mallctlbymib)(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - - if (malloc_init()) - return (EAGAIN); - - return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); -} - -JEMALLOC_INLINE void * -iallocm(size_t usize, size_t alignment, bool zero) -{ - - assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment, - NULL))); - - if (alignment != 0) - return (ipalloc(usize, alignment, zero)); - else if (zero) - return (icalloc(usize)); - else - return (imalloc(usize)); -} - -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(allocm)(void **ptr, size_t *rsize, size_t size, int flags) -{ - void *p; - size_t usize; - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) - & (SIZE_T_MAX-1)); - bool zero = flags & ALLOCM_ZERO; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt; -#endif - - assert(ptr != NULL); - assert(size != 0); - - if (malloc_init()) - goto OOM; - - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment, NULL); - if (usize == 0) - goto OOM; - -#ifdef JEMALLOC_PROF - if (opt_prof) { - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) - goto OOM; - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - small_maxclass) { - size_t usize_promoted = (alignment == 0) ? - s2u(small_maxclass+1) : sa2u(small_maxclass+1, - alignment, NULL); - assert(usize_promoted != 0); - p = iallocm(usize_promoted, alignment, zero); - if (p == NULL) - goto OOM; - arena_prof_promoted(p, usize); - } else { - p = iallocm(usize, alignment, zero); - if (p == NULL) - goto OOM; - } - prof_malloc(p, usize, cnt); - if (rsize != NULL) - *rsize = usize; - } else -#endif - { - p = iallocm(usize, alignment, zero); - if (p == NULL) - goto OOM; -#ifndef JEMALLOC_STATS - if (rsize != NULL) -#endif - { -#ifdef JEMALLOC_STATS - if (rsize != NULL) -#endif - *rsize = usize; - } - } - - *ptr = p; -#ifdef JEMALLOC_STATS - assert(usize == isalloc(p)); - ALLOCATED_ADD(usize, 0); -#endif - return (ALLOCM_SUCCESS); -OOM: -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in allocm(): " - "out of memory\n"); - abort(); - } -#endif - *ptr = NULL; - return (ALLOCM_ERR_OOM); -} - -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(rallocm)(void **ptr, size_t *rsize, size_t size, size_t extra, - int flags) -{ - void *p, *q; - size_t usize; -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t old_size; -#endif - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) - & (SIZE_T_MAX-1)); - bool zero = flags & ALLOCM_ZERO; - bool no_move = flags & ALLOCM_NO_MOVE; -#ifdef JEMALLOC_PROF - prof_thr_cnt_t *cnt; -#endif - - assert(ptr != NULL); - assert(*ptr != NULL); - assert(size != 0); - assert(SIZE_T_MAX - size >= extra); - assert(malloc_initialized || malloc_initializer == pthread_self()); - - p = *ptr; -#ifdef JEMALLOC_PROF - if (opt_prof) { - /* - * usize isn't knowable before iralloc() returns when extra is - * non-zero. Therefore, compute its maximum possible value and - * use that in PROF_ALLOC_PREP() to decide whether to capture a - * backtrace. prof_realloc() will use the actual usize to - * decide whether to sample. - */ - size_t max_usize = (alignment == 0) ? s2u(size+extra) : - sa2u(size+extra, alignment, NULL); - prof_ctx_t *old_ctx = prof_ctx_get(p); - old_size = isalloc(p); - PROF_ALLOC_PREP(1, max_usize, cnt); - if (cnt == NULL) - goto OOM; - /* - * Use minimum usize to determine whether promotion may happen. - */ - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U - && ((alignment == 0) ? s2u(size) : sa2u(size, - alignment, NULL)) <= small_maxclass) { - q = iralloc(p, small_maxclass+1, (small_maxclass+1 >= - size+extra) ? 0 : size+extra - (small_maxclass+1), - alignment, zero, no_move); - if (q == NULL) - goto ERR; - if (max_usize < PAGE_SIZE) { - usize = max_usize; - arena_prof_promoted(q, usize); - } else - usize = isalloc(q); - } else { - q = iralloc(p, size, extra, alignment, zero, no_move); - if (q == NULL) - goto ERR; - usize = isalloc(q); - } - prof_realloc(q, usize, cnt, old_size, old_ctx); - if (rsize != NULL) - *rsize = usize; - } else -#endif - { -#ifdef JEMALLOC_STATS - old_size = isalloc(p); -#endif - q = iralloc(p, size, extra, alignment, zero, no_move); - if (q == NULL) - goto ERR; -#ifndef JEMALLOC_STATS - if (rsize != NULL) -#endif - { - usize = isalloc(q); -#ifdef JEMALLOC_STATS - if (rsize != NULL) -#endif - *rsize = usize; - } - } - - *ptr = q; -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(usize, old_size); -#endif - return (ALLOCM_SUCCESS); -ERR: - if (no_move) - return (ALLOCM_ERR_NOT_MOVED); -#ifdef JEMALLOC_PROF -OOM: -#endif -#ifdef JEMALLOC_XMALLOC - if (opt_xmalloc) { - malloc_write(": Error in rallocm(): " - "out of memory\n"); - abort(); - } -#endif - return (ALLOCM_ERR_OOM); -} - -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(sallocm)(const void *ptr, size_t *rsize, int flags) -{ - size_t sz; - - assert(malloc_initialized || malloc_initializer == pthread_self()); - -#ifdef JEMALLOC_IVSALLOC - sz = ivsalloc(ptr); -#else - assert(ptr != NULL); - sz = isalloc(ptr); -#endif - assert(rsize != NULL); - *rsize = sz; - - return (ALLOCM_SUCCESS); -} - -JEMALLOC_ATTR(nonnull(1)) -JEMALLOC_ATTR(visibility("default")) -int -JEMALLOC_P(dallocm)(void *ptr, int flags) -{ -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - size_t usize; -#endif - - assert(ptr != NULL); - assert(malloc_initialized || malloc_initializer == pthread_self()); - -#ifdef JEMALLOC_STATS - usize = isalloc(ptr); -#endif -#ifdef JEMALLOC_PROF - if (opt_prof) { -# ifndef JEMALLOC_STATS - usize = isalloc(ptr); -# endif - prof_free(ptr, usize); - } -#endif -#ifdef JEMALLOC_STATS - ALLOCATED_ADD(0, usize); -#endif - idalloc(ptr); - - return (ALLOCM_SUCCESS); -} - -/* - * End non-standard functions. - */ -/******************************************************************************/ - -/* - * The following functions are used by threading libraries for protection of - * malloc during fork(). - */ - -void -jemalloc_prefork(void) -{ - unsigned i; - - /* Acquire all mutexes in a safe order. */ - - malloc_mutex_lock(&arenas_lock); - for (i = 0; i < narenas; i++) { - if (arenas[i] != NULL) - malloc_mutex_lock(&arenas[i]->lock); - } - - malloc_mutex_lock(&base_mtx); - - malloc_mutex_lock(&huge_mtx); - -#ifdef JEMALLOC_DSS - malloc_mutex_lock(&dss_mtx); -#endif - -#ifdef JEMALLOC_SWAP - malloc_mutex_lock(&swap_mtx); -#endif -} - -void -jemalloc_postfork(void) -{ - unsigned i; - - /* Release all mutexes, now that fork() has completed. */ - -#ifdef JEMALLOC_SWAP - malloc_mutex_unlock(&swap_mtx); -#endif - -#ifdef JEMALLOC_DSS - malloc_mutex_unlock(&dss_mtx); -#endif - - malloc_mutex_unlock(&huge_mtx); - - malloc_mutex_unlock(&base_mtx); - - for (i = 0; i < narenas; i++) { - if (arenas[i] != NULL) - malloc_mutex_unlock(&arenas[i]->lock); - } - malloc_mutex_unlock(&arenas_lock); -} - -/******************************************************************************/ diff --git a/deps/jemalloc.orig/src/mb.c b/deps/jemalloc.orig/src/mb.c deleted file mode 100644 index dc2c0a256fd..00000000000 --- a/deps/jemalloc.orig/src/mb.c +++ /dev/null @@ -1,2 +0,0 @@ -#define JEMALLOC_MB_C_ -#include "jemalloc/internal/jemalloc_internal.h" diff --git a/deps/jemalloc.orig/src/mutex.c b/deps/jemalloc.orig/src/mutex.c deleted file mode 100644 index ca89ef1c962..00000000000 --- a/deps/jemalloc.orig/src/mutex.c +++ /dev/null @@ -1,90 +0,0 @@ -#define JEMALLOC_MUTEX_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -/******************************************************************************/ -/* Data. */ - -#ifdef JEMALLOC_LAZY_LOCK -bool isthreaded = false; -#endif - -#ifdef JEMALLOC_LAZY_LOCK -static void pthread_create_once(void); -#endif - -/******************************************************************************/ -/* - * We intercept pthread_create() calls in order to toggle isthreaded if the - * process goes multi-threaded. - */ - -#ifdef JEMALLOC_LAZY_LOCK -static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, - void *(*)(void *), void *__restrict); - -static void -pthread_create_once(void) -{ - - pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); - if (pthread_create_fptr == NULL) { - malloc_write(": Error in dlsym(RTLD_NEXT, " - "\"pthread_create\")\n"); - abort(); - } - - isthreaded = true; -} - -JEMALLOC_ATTR(visibility("default")) -int -pthread_create(pthread_t *__restrict thread, - const pthread_attr_t *__restrict attr, void *(*start_routine)(void *), - void *__restrict arg) -{ - static pthread_once_t once_control = PTHREAD_ONCE_INIT; - - pthread_once(&once_control, pthread_create_once); - - return (pthread_create_fptr(thread, attr, start_routine, arg)); -} -#endif - -/******************************************************************************/ - -bool -malloc_mutex_init(malloc_mutex_t *mutex) -{ -#ifdef JEMALLOC_OSSPIN - *mutex = 0; -#else - pthread_mutexattr_t attr; - - if (pthread_mutexattr_init(&attr) != 0) - return (true); -#ifdef PTHREAD_MUTEX_ADAPTIVE_NP - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ADAPTIVE_NP); -#else - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); -#endif - if (pthread_mutex_init(mutex, &attr) != 0) { - pthread_mutexattr_destroy(&attr); - return (true); - } - pthread_mutexattr_destroy(&attr); - -#endif - return (false); -} - -void -malloc_mutex_destroy(malloc_mutex_t *mutex) -{ - -#ifndef JEMALLOC_OSSPIN - if (pthread_mutex_destroy(mutex) != 0) { - malloc_write(": Error in pthread_mutex_destroy()\n"); - abort(); - } -#endif -} diff --git a/deps/jemalloc.orig/src/prof.c b/deps/jemalloc.orig/src/prof.c deleted file mode 100644 index 8a144b4e46c..00000000000 --- a/deps/jemalloc.orig/src/prof.c +++ /dev/null @@ -1,1244 +0,0 @@ -#define JEMALLOC_PROF_C_ -#include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_PROF -/******************************************************************************/ - -#ifdef JEMALLOC_PROF_LIBUNWIND -#define UNW_LOCAL_ONLY -#include -#endif - -#ifdef JEMALLOC_PROF_LIBGCC -#include -#endif - -/******************************************************************************/ -/* Data. */ - -bool opt_prof = false; -bool opt_prof_active = true; -size_t opt_lg_prof_bt_max = LG_PROF_BT_MAX_DEFAULT; -size_t opt_lg_prof_sample = LG_PROF_SAMPLE_DEFAULT; -ssize_t opt_lg_prof_interval = LG_PROF_INTERVAL_DEFAULT; -bool opt_prof_gdump = false; -bool opt_prof_leak = false; -bool opt_prof_accum = true; -ssize_t opt_lg_prof_tcmax = LG_PROF_TCMAX_DEFAULT; -char opt_prof_prefix[PATH_MAX + 1]; - -uint64_t prof_interval; -bool prof_promote; - -unsigned prof_bt_max; - -#ifndef NO_TLS -__thread prof_tdata_t *prof_tdata_tls - JEMALLOC_ATTR(tls_model("initial-exec")); -#endif -pthread_key_t prof_tdata_tsd; - -/* - * Global hash of (prof_bt_t *)-->(prof_ctx_t *). This is the master data - * structure that knows about all backtraces currently captured. - */ -static ckh_t bt2ctx; -static malloc_mutex_t bt2ctx_mtx; - -static malloc_mutex_t prof_dump_seq_mtx; -static uint64_t prof_dump_seq; -static uint64_t prof_dump_iseq; -static uint64_t prof_dump_mseq; -static uint64_t prof_dump_useq; - -/* - * This buffer is rather large for stack allocation, so use a single buffer for - * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since - * it must be locked anyway during dumping. - */ -static char prof_dump_buf[PROF_DUMP_BUF_SIZE]; -static unsigned prof_dump_buf_end; -static int prof_dump_fd; - -/* Do not dump any profiles until bootstrapping is complete. */ -static bool prof_booted = false; - -static malloc_mutex_t enq_mtx; -static bool enq; -static bool enq_idump; -static bool enq_gdump; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static prof_bt_t *bt_dup(prof_bt_t *bt); -static void bt_destroy(prof_bt_t *bt); -#ifdef JEMALLOC_PROF_LIBGCC -static _Unwind_Reason_Code prof_unwind_init_callback( - struct _Unwind_Context *context, void *arg); -static _Unwind_Reason_Code prof_unwind_callback( - struct _Unwind_Context *context, void *arg); -#endif -static bool prof_flush(bool propagate_err); -static bool prof_write(const char *s, bool propagate_err); -static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, - size_t *leak_nctx); -static void prof_ctx_destroy(prof_ctx_t *ctx); -static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); -static bool prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, - bool propagate_err); -static bool prof_dump_maps(bool propagate_err); -static bool prof_dump(const char *filename, bool leakcheck, - bool propagate_err); -static void prof_dump_filename(char *filename, char v, int64_t vseq); -static void prof_fdump(void); -static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); -static bool prof_bt_keycomp(const void *k1, const void *k2); -static void prof_tdata_cleanup(void *arg); - -/******************************************************************************/ - -void -bt_init(prof_bt_t *bt, void **vec) -{ - - bt->vec = vec; - bt->len = 0; -} - -static void -bt_destroy(prof_bt_t *bt) -{ - - idalloc(bt); -} - -static prof_bt_t * -bt_dup(prof_bt_t *bt) -{ - prof_bt_t *ret; - - /* - * Create a single allocation that has space for vec immediately - * following the prof_bt_t structure. The backtraces that get - * stored in the backtrace caches are copied from stack-allocated - * temporary variables, so size is known at creation time. Making this - * a contiguous object improves cache locality. - */ - ret = (prof_bt_t *)imalloc(QUANTUM_CEILING(sizeof(prof_bt_t)) + - (bt->len * sizeof(void *))); - if (ret == NULL) - return (NULL); - ret->vec = (void **)((uintptr_t)ret + - QUANTUM_CEILING(sizeof(prof_bt_t))); - memcpy(ret->vec, bt->vec, bt->len * sizeof(void *)); - ret->len = bt->len; - - return (ret); -} - -static inline void -prof_enter(void) -{ - - malloc_mutex_lock(&enq_mtx); - enq = true; - malloc_mutex_unlock(&enq_mtx); - - malloc_mutex_lock(&bt2ctx_mtx); -} - -static inline void -prof_leave(void) -{ - bool idump, gdump; - - malloc_mutex_unlock(&bt2ctx_mtx); - - malloc_mutex_lock(&enq_mtx); - enq = false; - idump = enq_idump; - enq_idump = false; - gdump = enq_gdump; - enq_gdump = false; - malloc_mutex_unlock(&enq_mtx); - - if (idump) - prof_idump(); - if (gdump) - prof_gdump(); -} - -#ifdef JEMALLOC_PROF_LIBUNWIND -void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) -{ - unw_context_t uc; - unw_cursor_t cursor; - unsigned i; - int err; - - assert(bt->len == 0); - assert(bt->vec != NULL); - assert(max <= (1U << opt_lg_prof_bt_max)); - - unw_getcontext(&uc); - unw_init_local(&cursor, &uc); - - /* Throw away (nignore+1) stack frames, if that many exist. */ - for (i = 0; i < nignore + 1; i++) { - err = unw_step(&cursor); - if (err <= 0) - return; - } - - /* - * Iterate over stack frames until there are no more, or until no space - * remains in bt. - */ - for (i = 0; i < max; i++) { - unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *)&bt->vec[i]); - bt->len++; - err = unw_step(&cursor); - if (err <= 0) - break; - } -} -#endif -#ifdef JEMALLOC_PROF_LIBGCC -static _Unwind_Reason_Code -prof_unwind_init_callback(struct _Unwind_Context *context, void *arg) -{ - - return (_URC_NO_REASON); -} - -static _Unwind_Reason_Code -prof_unwind_callback(struct _Unwind_Context *context, void *arg) -{ - prof_unwind_data_t *data = (prof_unwind_data_t *)arg; - - if (data->nignore > 0) - data->nignore--; - else { - data->bt->vec[data->bt->len] = (void *)_Unwind_GetIP(context); - data->bt->len++; - if (data->bt->len == data->max) - return (_URC_END_OF_STACK); - } - - return (_URC_NO_REASON); -} - -void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) -{ - prof_unwind_data_t data = {bt, nignore, max}; - - _Unwind_Backtrace(prof_unwind_callback, &data); -} -#endif -#ifdef JEMALLOC_PROF_GCC -void -prof_backtrace(prof_bt_t *bt, unsigned nignore, unsigned max) -{ -#define BT_FRAME(i) \ - if ((i) < nignore + max) { \ - void *p; \ - if (__builtin_frame_address(i) == 0) \ - return; \ - p = __builtin_return_address(i); \ - if (p == NULL) \ - return; \ - if (i >= nignore) { \ - bt->vec[(i) - nignore] = p; \ - bt->len = (i) - nignore + 1; \ - } \ - } else \ - return; - - assert(nignore <= 3); - assert(max <= (1U << opt_lg_prof_bt_max)); - - BT_FRAME(0) - BT_FRAME(1) - BT_FRAME(2) - BT_FRAME(3) - BT_FRAME(4) - BT_FRAME(5) - BT_FRAME(6) - BT_FRAME(7) - BT_FRAME(8) - BT_FRAME(9) - - BT_FRAME(10) - BT_FRAME(11) - BT_FRAME(12) - BT_FRAME(13) - BT_FRAME(14) - BT_FRAME(15) - BT_FRAME(16) - BT_FRAME(17) - BT_FRAME(18) - BT_FRAME(19) - - BT_FRAME(20) - BT_FRAME(21) - BT_FRAME(22) - BT_FRAME(23) - BT_FRAME(24) - BT_FRAME(25) - BT_FRAME(26) - BT_FRAME(27) - BT_FRAME(28) - BT_FRAME(29) - - BT_FRAME(30) - BT_FRAME(31) - BT_FRAME(32) - BT_FRAME(33) - BT_FRAME(34) - BT_FRAME(35) - BT_FRAME(36) - BT_FRAME(37) - BT_FRAME(38) - BT_FRAME(39) - - BT_FRAME(40) - BT_FRAME(41) - BT_FRAME(42) - BT_FRAME(43) - BT_FRAME(44) - BT_FRAME(45) - BT_FRAME(46) - BT_FRAME(47) - BT_FRAME(48) - BT_FRAME(49) - - BT_FRAME(50) - BT_FRAME(51) - BT_FRAME(52) - BT_FRAME(53) - BT_FRAME(54) - BT_FRAME(55) - BT_FRAME(56) - BT_FRAME(57) - BT_FRAME(58) - BT_FRAME(59) - - BT_FRAME(60) - BT_FRAME(61) - BT_FRAME(62) - BT_FRAME(63) - BT_FRAME(64) - BT_FRAME(65) - BT_FRAME(66) - BT_FRAME(67) - BT_FRAME(68) - BT_FRAME(69) - - BT_FRAME(70) - BT_FRAME(71) - BT_FRAME(72) - BT_FRAME(73) - BT_FRAME(74) - BT_FRAME(75) - BT_FRAME(76) - BT_FRAME(77) - BT_FRAME(78) - BT_FRAME(79) - - BT_FRAME(80) - BT_FRAME(81) - BT_FRAME(82) - BT_FRAME(83) - BT_FRAME(84) - BT_FRAME(85) - BT_FRAME(86) - BT_FRAME(87) - BT_FRAME(88) - BT_FRAME(89) - - BT_FRAME(90) - BT_FRAME(91) - BT_FRAME(92) - BT_FRAME(93) - BT_FRAME(94) - BT_FRAME(95) - BT_FRAME(96) - BT_FRAME(97) - BT_FRAME(98) - BT_FRAME(99) - - BT_FRAME(100) - BT_FRAME(101) - BT_FRAME(102) - BT_FRAME(103) - BT_FRAME(104) - BT_FRAME(105) - BT_FRAME(106) - BT_FRAME(107) - BT_FRAME(108) - BT_FRAME(109) - - BT_FRAME(110) - BT_FRAME(111) - BT_FRAME(112) - BT_FRAME(113) - BT_FRAME(114) - BT_FRAME(115) - BT_FRAME(116) - BT_FRAME(117) - BT_FRAME(118) - BT_FRAME(119) - - BT_FRAME(120) - BT_FRAME(121) - BT_FRAME(122) - BT_FRAME(123) - BT_FRAME(124) - BT_FRAME(125) - BT_FRAME(126) - BT_FRAME(127) - - /* Extras to compensate for nignore. */ - BT_FRAME(128) - BT_FRAME(129) - BT_FRAME(130) -#undef BT_FRAME -} -#endif - -prof_thr_cnt_t * -prof_lookup(prof_bt_t *bt) -{ - union { - prof_thr_cnt_t *p; - void *v; - } ret; - prof_tdata_t *prof_tdata; - - prof_tdata = PROF_TCACHE_GET(); - if (prof_tdata == NULL) { - prof_tdata = prof_tdata_init(); - if (prof_tdata == NULL) - return (NULL); - } - - if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { - union { - prof_bt_t *p; - void *v; - } btkey; - union { - prof_ctx_t *p; - void *v; - } ctx; - bool new_ctx; - - /* - * This thread's cache lacks bt. Look for it in the global - * cache. - */ - prof_enter(); - if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { - /* bt has never been seen before. Insert it. */ - ctx.v = imalloc(sizeof(prof_ctx_t)); - if (ctx.v == NULL) { - prof_leave(); - return (NULL); - } - btkey.p = bt_dup(bt); - if (btkey.v == NULL) { - prof_leave(); - idalloc(ctx.v); - return (NULL); - } - ctx.p->bt = btkey.p; - if (malloc_mutex_init(&ctx.p->lock)) { - prof_leave(); - idalloc(btkey.v); - idalloc(ctx.v); - return (NULL); - } - memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); - ql_new(&ctx.p->cnts_ql); - if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { - /* OOM. */ - prof_leave(); - malloc_mutex_destroy(&ctx.p->lock); - idalloc(btkey.v); - idalloc(ctx.v); - return (NULL); - } - /* - * Artificially raise curobjs, in order to avoid a race - * condition with prof_ctx_merge()/prof_ctx_destroy(). - * - * No locking is necessary for ctx here because no other - * threads have had the opportunity to fetch it from - * bt2ctx yet. - */ - ctx.p->cnt_merged.curobjs++; - new_ctx = true; - } else { - /* - * Artificially raise curobjs, in order to avoid a race - * condition with prof_ctx_merge()/prof_ctx_destroy(). - */ - malloc_mutex_lock(&ctx.p->lock); - ctx.p->cnt_merged.curobjs++; - malloc_mutex_unlock(&ctx.p->lock); - new_ctx = false; - } - prof_leave(); - - /* Link a prof_thd_cnt_t into ctx for this thread. */ - if (opt_lg_prof_tcmax >= 0 && ckh_count(&prof_tdata->bt2cnt) - == (ZU(1) << opt_lg_prof_tcmax)) { - assert(ckh_count(&prof_tdata->bt2cnt) > 0); - /* - * Flush the least recently used cnt in order to keep - * bt2cnt from becoming too large. - */ - ret.p = ql_last(&prof_tdata->lru_ql, lru_link); - assert(ret.v != NULL); - if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, - NULL, NULL)) - assert(false); - ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); - prof_ctx_merge(ret.p->ctx, ret.p); - /* ret can now be re-used. */ - } else { - assert(opt_lg_prof_tcmax < 0 || - ckh_count(&prof_tdata->bt2cnt) < (ZU(1) << - opt_lg_prof_tcmax)); - /* Allocate and partially initialize a new cnt. */ - ret.v = imalloc(sizeof(prof_thr_cnt_t)); - if (ret.p == NULL) { - if (new_ctx) - prof_ctx_destroy(ctx.p); - return (NULL); - } - ql_elm_new(ret.p, cnts_link); - ql_elm_new(ret.p, lru_link); - } - /* Finish initializing ret. */ - ret.p->ctx = ctx.p; - ret.p->epoch = 0; - memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); - if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) { - if (new_ctx) - prof_ctx_destroy(ctx.p); - idalloc(ret.v); - return (NULL); - } - ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); - malloc_mutex_lock(&ctx.p->lock); - ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); - ctx.p->cnt_merged.curobjs--; - malloc_mutex_unlock(&ctx.p->lock); - } else { - /* Move ret to the front of the LRU. */ - ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); - ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); - } - - return (ret.p); -} - -static bool -prof_flush(bool propagate_err) -{ - bool ret = false; - ssize_t err; - - err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); - if (err == -1) { - if (propagate_err == false) { - malloc_write(": write() failed during heap " - "profile flush\n"); - if (opt_abort) - abort(); - } - ret = true; - } - prof_dump_buf_end = 0; - - return (ret); -} - -static bool -prof_write(const char *s, bool propagate_err) -{ - unsigned i, slen, n; - - i = 0; - slen = strlen(s); - while (i < slen) { - /* Flush the buffer if it is full. */ - if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) - if (prof_flush(propagate_err) && propagate_err) - return (true); - - if (prof_dump_buf_end + slen <= PROF_DUMP_BUF_SIZE) { - /* Finish writing. */ - n = slen - i; - } else { - /* Write as much of s as will fit. */ - n = PROF_DUMP_BUF_SIZE - prof_dump_buf_end; - } - memcpy(&prof_dump_buf[prof_dump_buf_end], &s[i], n); - prof_dump_buf_end += n; - i += n; - } - - return (false); -} - -static void -prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) -{ - prof_thr_cnt_t *thr_cnt; - prof_cnt_t tcnt; - - malloc_mutex_lock(&ctx->lock); - - memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); - ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { - volatile unsigned *epoch = &thr_cnt->epoch; - - while (true) { - unsigned epoch0 = *epoch; - - /* Make sure epoch is even. */ - if (epoch0 & 1U) - continue; - - memcpy(&tcnt, &thr_cnt->cnts, sizeof(prof_cnt_t)); - - /* Terminate if epoch didn't change while reading. */ - if (*epoch == epoch0) - break; - } - - ctx->cnt_summed.curobjs += tcnt.curobjs; - ctx->cnt_summed.curbytes += tcnt.curbytes; - if (opt_prof_accum) { - ctx->cnt_summed.accumobjs += tcnt.accumobjs; - ctx->cnt_summed.accumbytes += tcnt.accumbytes; - } - } - - if (ctx->cnt_summed.curobjs != 0) - (*leak_nctx)++; - - /* Add to cnt_all. */ - cnt_all->curobjs += ctx->cnt_summed.curobjs; - cnt_all->curbytes += ctx->cnt_summed.curbytes; - if (opt_prof_accum) { - cnt_all->accumobjs += ctx->cnt_summed.accumobjs; - cnt_all->accumbytes += ctx->cnt_summed.accumbytes; - } - - malloc_mutex_unlock(&ctx->lock); -} - -static void -prof_ctx_destroy(prof_ctx_t *ctx) -{ - - /* - * Check that ctx is still unused by any thread cache before destroying - * it. prof_lookup() artificially raises ctx->cnt_merge.curobjs in - * order to avoid a race condition with this function, as does - * prof_ctx_merge() in order to avoid a race between the main body of - * prof_ctx_merge() and entry into this function. - */ - prof_enter(); - malloc_mutex_lock(&ctx->lock); - if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 1) { - assert(ctx->cnt_merged.curbytes == 0); - assert(ctx->cnt_merged.accumobjs == 0); - assert(ctx->cnt_merged.accumbytes == 0); - /* Remove ctx from bt2ctx. */ - if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) - assert(false); - prof_leave(); - /* Destroy ctx. */ - malloc_mutex_unlock(&ctx->lock); - bt_destroy(ctx->bt); - malloc_mutex_destroy(&ctx->lock); - idalloc(ctx); - } else { - /* - * Compensate for increment in prof_ctx_merge() or - * prof_lookup(). - */ - ctx->cnt_merged.curobjs--; - malloc_mutex_unlock(&ctx->lock); - prof_leave(); - } -} - -static void -prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) -{ - bool destroy; - - /* Merge cnt stats and detach from ctx. */ - malloc_mutex_lock(&ctx->lock); - ctx->cnt_merged.curobjs += cnt->cnts.curobjs; - ctx->cnt_merged.curbytes += cnt->cnts.curbytes; - ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; - ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; - ql_remove(&ctx->cnts_ql, cnt, cnts_link); - if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && - ctx->cnt_merged.curobjs == 0) { - /* - * Artificially raise ctx->cnt_merged.curobjs in order to keep - * another thread from winning the race to destroy ctx while - * this one has ctx->lock dropped. Without this, it would be - * possible for another thread to: - * - * 1) Sample an allocation associated with ctx. - * 2) Deallocate the sampled object. - * 3) Successfully prof_ctx_destroy(ctx). - * - * The result would be that ctx no longer exists by the time - * this thread accesses it in prof_ctx_destroy(). - */ - ctx->cnt_merged.curobjs++; - destroy = true; - } else - destroy = false; - malloc_mutex_unlock(&ctx->lock); - if (destroy) - prof_ctx_destroy(ctx); -} - -static bool -prof_dump_ctx(prof_ctx_t *ctx, prof_bt_t *bt, bool propagate_err) -{ - char buf[UMAX2S_BUFSIZE]; - unsigned i; - - if (opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) { - assert(ctx->cnt_summed.curbytes == 0); - assert(ctx->cnt_summed.accumobjs == 0); - assert(ctx->cnt_summed.accumbytes == 0); - return (false); - } - - if (prof_write(u2s(ctx->cnt_summed.curobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(ctx->cnt_summed.curbytes, 10, buf), - propagate_err) - || prof_write(" [", propagate_err) - || prof_write(u2s(ctx->cnt_summed.accumobjs, 10, buf), - propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(ctx->cnt_summed.accumbytes, 10, buf), - propagate_err) - || prof_write("] @", propagate_err)) - return (true); - - for (i = 0; i < bt->len; i++) { - if (prof_write(" 0x", propagate_err) - || prof_write(u2s((uintptr_t)bt->vec[i], 16, buf), - propagate_err)) - return (true); - } - - if (prof_write("\n", propagate_err)) - return (true); - - return (false); -} - -static bool -prof_dump_maps(bool propagate_err) -{ - int mfd; - char buf[UMAX2S_BUFSIZE]; - char *s; - unsigned i, slen; - /* /proc//maps\0 */ - char mpath[6 + UMAX2S_BUFSIZE - + 5 + 1]; - - i = 0; - - s = "/proc/"; - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - s = u2s(getpid(), 10, buf); - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - s = "/maps"; - slen = strlen(s); - memcpy(&mpath[i], s, slen); - i += slen; - - mpath[i] = '\0'; - - mfd = open(mpath, O_RDONLY); - if (mfd != -1) { - ssize_t nread; - - if (prof_write("\nMAPPED_LIBRARIES:\n", propagate_err) && - propagate_err) - return (true); - nread = 0; - do { - prof_dump_buf_end += nread; - if (prof_dump_buf_end == PROF_DUMP_BUF_SIZE) { - /* Make space in prof_dump_buf before read(). */ - if (prof_flush(propagate_err) && propagate_err) - return (true); - } - nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], - PROF_DUMP_BUF_SIZE - prof_dump_buf_end); - } while (nread > 0); - close(mfd); - } else - return (true); - - return (false); -} - -static bool -prof_dump(const char *filename, bool leakcheck, bool propagate_err) -{ - prof_cnt_t cnt_all; - size_t tabind; - union { - prof_bt_t *p; - void *v; - } bt; - union { - prof_ctx_t *p; - void *v; - } ctx; - char buf[UMAX2S_BUFSIZE]; - size_t leak_nctx; - - prof_enter(); - prof_dump_fd = creat(filename, 0644); - if (prof_dump_fd == -1) { - if (propagate_err == false) { - malloc_write(": creat(\""); - malloc_write(filename); - malloc_write("\", 0644) failed\n"); - if (opt_abort) - abort(); - } - goto ERROR; - } - - /* Merge per thread profile stats, and sum them in cnt_all. */ - memset(&cnt_all, 0, sizeof(prof_cnt_t)); - leak_nctx = 0; - for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) - prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); - - /* Dump profile header. */ - if (prof_write("heap profile: ", propagate_err) - || prof_write(u2s(cnt_all.curobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(cnt_all.curbytes, 10, buf), propagate_err) - || prof_write(" [", propagate_err) - || prof_write(u2s(cnt_all.accumobjs, 10, buf), propagate_err) - || prof_write(": ", propagate_err) - || prof_write(u2s(cnt_all.accumbytes, 10, buf), propagate_err)) - goto ERROR; - - if (opt_lg_prof_sample == 0) { - if (prof_write("] @ heapprofile\n", propagate_err)) - goto ERROR; - } else { - if (prof_write("] @ heap_v2/", propagate_err) - || prof_write(u2s((uint64_t)1U << opt_lg_prof_sample, 10, - buf), propagate_err) - || prof_write("\n", propagate_err)) - goto ERROR; - } - - /* Dump per ctx profile stats. */ - for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) - == false;) { - if (prof_dump_ctx(ctx.p, bt.p, propagate_err)) - goto ERROR; - } - - /* Dump /proc//maps if possible. */ - if (prof_dump_maps(propagate_err)) - goto ERROR; - - if (prof_flush(propagate_err)) - goto ERROR; - close(prof_dump_fd); - prof_leave(); - - if (leakcheck && cnt_all.curbytes != 0) { - malloc_write(": Leak summary: "); - malloc_write(u2s(cnt_all.curbytes, 10, buf)); - malloc_write((cnt_all.curbytes != 1) ? " bytes, " : " byte, "); - malloc_write(u2s(cnt_all.curobjs, 10, buf)); - malloc_write((cnt_all.curobjs != 1) ? " objects, " : - " object, "); - malloc_write(u2s(leak_nctx, 10, buf)); - malloc_write((leak_nctx != 1) ? " contexts\n" : " context\n"); - malloc_write(": Run pprof on \""); - malloc_write(filename); - malloc_write("\" for leak detail\n"); - } - - return (false); -ERROR: - prof_leave(); - return (true); -} - -#define DUMP_FILENAME_BUFSIZE (PATH_MAX+ UMAX2S_BUFSIZE \ - + 1 \ - + UMAX2S_BUFSIZE \ - + 2 \ - + UMAX2S_BUFSIZE \ - + 5 + 1) -static void -prof_dump_filename(char *filename, char v, int64_t vseq) -{ - char buf[UMAX2S_BUFSIZE]; - char *s; - unsigned i, slen; - - /* - * Construct a filename of the form: - * - * ...v.heap\0 - */ - - i = 0; - - s = opt_prof_prefix; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = u2s(getpid(), 10, buf); - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = u2s(prof_dump_seq, 10, buf); - prof_dump_seq++; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - s = "."; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - filename[i] = v; - i++; - - if (vseq != 0xffffffffffffffffLLU) { - s = u2s(vseq, 10, buf); - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - } - - s = ".heap"; - slen = strlen(s); - memcpy(&filename[i], s, slen); - i += slen; - - filename[i] = '\0'; -} - -static void -prof_fdump(void) -{ - char filename[DUMP_FILENAME_BUFSIZE]; - - if (prof_booted == false) - return; - - if (opt_prof_prefix[0] != '\0') { - malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'f', 0xffffffffffffffffLLU); - malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, opt_prof_leak, false); - } -} - -void -prof_idump(void) -{ - char filename[DUMP_FILENAME_BUFSIZE]; - - if (prof_booted == false) - return; - malloc_mutex_lock(&enq_mtx); - if (enq) { - enq_idump = true; - malloc_mutex_unlock(&enq_mtx); - return; - } - malloc_mutex_unlock(&enq_mtx); - - if (opt_prof_prefix[0] != '\0') { - malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'i', prof_dump_iseq); - prof_dump_iseq++; - malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, false, false); - } -} - -bool -prof_mdump(const char *filename) -{ - char filename_buf[DUMP_FILENAME_BUFSIZE]; - - if (opt_prof == false || prof_booted == false) - return (true); - - if (filename == NULL) { - /* No filename specified, so automatically generate one. */ - if (opt_prof_prefix[0] == '\0') - return (true); - malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename_buf, 'm', prof_dump_mseq); - prof_dump_mseq++; - malloc_mutex_unlock(&prof_dump_seq_mtx); - filename = filename_buf; - } - return (prof_dump(filename, false, true)); -} - -void -prof_gdump(void) -{ - char filename[DUMP_FILENAME_BUFSIZE]; - - if (prof_booted == false) - return; - malloc_mutex_lock(&enq_mtx); - if (enq) { - enq_gdump = true; - malloc_mutex_unlock(&enq_mtx); - return; - } - malloc_mutex_unlock(&enq_mtx); - - if (opt_prof_prefix[0] != '\0') { - malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'u', prof_dump_useq); - prof_dump_useq++; - malloc_mutex_unlock(&prof_dump_seq_mtx); - prof_dump(filename, false, false); - } -} - -static void -prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) -{ - size_t ret1, ret2; - uint64_t h; - prof_bt_t *bt = (prof_bt_t *)key; - - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - - h = hash(bt->vec, bt->len * sizeof(void *), 0x94122f335b332aeaLLU); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - ret1 = h; - ret2 = hash(bt->vec, bt->len * sizeof(void *), - 0x8432a476666bbc13LLU); - } - - *hash1 = ret1; - *hash2 = ret2; -} - -static bool -prof_bt_keycomp(const void *k1, const void *k2) -{ - const prof_bt_t *bt1 = (prof_bt_t *)k1; - const prof_bt_t *bt2 = (prof_bt_t *)k2; - - if (bt1->len != bt2->len) - return (false); - return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); -} - -prof_tdata_t * -prof_tdata_init(void) -{ - prof_tdata_t *prof_tdata; - - /* Initialize an empty cache for this thread. */ - prof_tdata = (prof_tdata_t *)imalloc(sizeof(prof_tdata_t)); - if (prof_tdata == NULL) - return (NULL); - - if (ckh_new(&prof_tdata->bt2cnt, PROF_CKH_MINITEMS, - prof_bt_hash, prof_bt_keycomp)) { - idalloc(prof_tdata); - return (NULL); - } - ql_new(&prof_tdata->lru_ql); - - prof_tdata->vec = imalloc(sizeof(void *) * prof_bt_max); - if (prof_tdata->vec == NULL) { - ckh_delete(&prof_tdata->bt2cnt); - idalloc(prof_tdata); - return (NULL); - } - - prof_tdata->prn_state = 0; - prof_tdata->threshold = 0; - prof_tdata->accum = 0; - - PROF_TCACHE_SET(prof_tdata); - - return (prof_tdata); -} - -static void -prof_tdata_cleanup(void *arg) -{ - prof_thr_cnt_t *cnt; - prof_tdata_t *prof_tdata = (prof_tdata_t *)arg; - - /* - * Delete the hash table. All of its contents can still be iterated - * over via the LRU. - */ - ckh_delete(&prof_tdata->bt2cnt); - - /* Iteratively merge cnt's into the global stats and delete them. */ - while ((cnt = ql_last(&prof_tdata->lru_ql, lru_link)) != NULL) { - ql_remove(&prof_tdata->lru_ql, cnt, lru_link); - prof_ctx_merge(cnt->ctx, cnt); - idalloc(cnt); - } - - idalloc(prof_tdata->vec); - - idalloc(prof_tdata); - PROF_TCACHE_SET(NULL); -} - -void -prof_boot0(void) -{ - - memcpy(opt_prof_prefix, PROF_PREFIX_DEFAULT, - sizeof(PROF_PREFIX_DEFAULT)); -} - -void -prof_boot1(void) -{ - - /* - * opt_prof and prof_promote must be in their final state before any - * arenas are initialized, so this function must be executed early. - */ - - if (opt_prof_leak && opt_prof == false) { - /* - * Enable opt_prof, but in such a way that profiles are never - * automatically dumped. - */ - opt_prof = true; - opt_prof_gdump = false; - prof_interval = 0; - } else if (opt_prof) { - if (opt_lg_prof_interval >= 0) { - prof_interval = (((uint64_t)1U) << - opt_lg_prof_interval); - } else - prof_interval = 0; - } - - prof_promote = (opt_prof && opt_lg_prof_sample > PAGE_SHIFT); -} - -bool -prof_boot2(void) -{ - - if (opt_prof) { - if (ckh_new(&bt2ctx, PROF_CKH_MINITEMS, prof_bt_hash, - prof_bt_keycomp)) - return (true); - if (malloc_mutex_init(&bt2ctx_mtx)) - return (true); - if (pthread_key_create(&prof_tdata_tsd, prof_tdata_cleanup) - != 0) { - malloc_write( - ": Error in pthread_key_create()\n"); - abort(); - } - - prof_bt_max = (1U << opt_lg_prof_bt_max); - if (malloc_mutex_init(&prof_dump_seq_mtx)) - return (true); - - if (malloc_mutex_init(&enq_mtx)) - return (true); - enq = false; - enq_idump = false; - enq_gdump = false; - - if (atexit(prof_fdump) != 0) { - malloc_write(": Error in atexit()\n"); - if (opt_abort) - abort(); - } - } - -#ifdef JEMALLOC_PROF_LIBGCC - /* - * Cause the backtracing machinery to allocate its internal state - * before enabling profiling. - */ - _Unwind_Backtrace(prof_unwind_init_callback, NULL); -#endif - - prof_booted = true; - - return (false); -} - -/******************************************************************************/ -#endif /* JEMALLOC_PROF */ diff --git a/deps/jemalloc.orig/src/rtree.c b/deps/jemalloc.orig/src/rtree.c deleted file mode 100644 index eb0ff1e24af..00000000000 --- a/deps/jemalloc.orig/src/rtree.c +++ /dev/null @@ -1,46 +0,0 @@ -#define JEMALLOC_RTREE_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -rtree_t * -rtree_new(unsigned bits) -{ - rtree_t *ret; - unsigned bits_per_level, height, i; - - bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; - height = bits / bits_per_level; - if (height * bits_per_level != bits) - height++; - assert(height * bits_per_level >= bits); - - ret = (rtree_t*)base_alloc(offsetof(rtree_t, level2bits) + - (sizeof(unsigned) * height)); - if (ret == NULL) - return (NULL); - memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) * - height)); - - if (malloc_mutex_init(&ret->mutex)) { - /* Leak the rtree. */ - return (NULL); - } - ret->height = height; - if (bits_per_level * height > bits) - ret->level2bits[0] = bits % bits_per_level; - else - ret->level2bits[0] = bits_per_level; - for (i = 1; i < height; i++) - ret->level2bits[i] = bits_per_level; - - ret->root = (void**)base_alloc(sizeof(void *) << ret->level2bits[0]); - if (ret->root == NULL) { - /* - * We leak the rtree here, since there's no generic base - * deallocation. - */ - return (NULL); - } - memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]); - - return (ret); -} diff --git a/deps/jemalloc.orig/src/stats.c b/deps/jemalloc.orig/src/stats.c deleted file mode 100644 index dc172e425c0..00000000000 --- a/deps/jemalloc.orig/src/stats.c +++ /dev/null @@ -1,790 +0,0 @@ -#define JEMALLOC_STATS_C_ -#include "jemalloc/internal/jemalloc_internal.h" - -#define CTL_GET(n, v, t) do { \ - size_t sz = sizeof(t); \ - xmallctl(n, v, &sz, NULL, 0); \ -} while (0) - -#define CTL_I_GET(n, v, t) do { \ - size_t mib[6]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[2] = i; \ - xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ -} while (0) - -#define CTL_J_GET(n, v, t) do { \ - size_t mib[6]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[2] = j; \ - xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ -} while (0) - -#define CTL_IJ_GET(n, v, t) do { \ - size_t mib[6]; \ - size_t miblen = sizeof(mib) / sizeof(size_t); \ - size_t sz = sizeof(t); \ - xmallctlnametomib(n, mib, &miblen); \ - mib[2] = i; \ - mib[4] = j; \ - xmallctlbymib(mib, miblen, v, &sz, NULL, 0); \ -} while (0) - -/******************************************************************************/ -/* Data. */ - -bool opt_stats_print = false; - -#ifdef JEMALLOC_STATS -size_t stats_cactive = 0; -#endif - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -#ifdef JEMALLOC_STATS -static void malloc_vcprintf(void (*write_cb)(void *, const char *), - void *cbopaque, const char *format, va_list ap); -static void stats_arena_bins_print(void (*write_cb)(void *, const char *), - void *cbopaque, unsigned i); -static void stats_arena_lruns_print(void (*write_cb)(void *, const char *), - void *cbopaque, unsigned i); -static void stats_arena_print(void (*write_cb)(void *, const char *), - void *cbopaque, unsigned i); -#endif - -/******************************************************************************/ - -/* - * We don't want to depend on vsnprintf() for production builds, since that can - * cause unnecessary bloat for static binaries. u2s() provides minimal integer - * printing functionality, so that malloc_printf() use can be limited to - * JEMALLOC_STATS code. - */ -char * -u2s(uint64_t x, unsigned base, char *s) -{ - unsigned i; - - i = UMAX2S_BUFSIZE - 1; - s[i] = '\0'; - switch (base) { - case 10: - do { - i--; - s[i] = "0123456789"[x % (uint64_t)10]; - x /= (uint64_t)10; - } while (x > 0); - break; - case 16: - do { - i--; - s[i] = "0123456789abcdef"[x & 0xf]; - x >>= 4; - } while (x > 0); - break; - default: - do { - i--; - s[i] = "0123456789abcdefghijklmnopqrstuvwxyz"[x % - (uint64_t)base]; - x /= (uint64_t)base; - } while (x > 0); - } - - return (&s[i]); -} - -#ifdef JEMALLOC_STATS -static void -malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, - const char *format, va_list ap) -{ - char buf[4096]; - - if (write_cb == NULL) { - /* - * The caller did not provide an alternate write_cb callback - * function, so use the default one. malloc_write() is an - * inline function, so use malloc_message() directly here. - */ - write_cb = JEMALLOC_P(malloc_message); - cbopaque = NULL; - } - - vsnprintf(buf, sizeof(buf), format, ap); - write_cb(cbopaque, buf); -} - -/* - * Print to a callback function in such a way as to (hopefully) avoid memory - * allocation. - */ -JEMALLOC_ATTR(format(printf, 3, 4)) -void -malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, - const char *format, ...) -{ - va_list ap; - - va_start(ap, format); - malloc_vcprintf(write_cb, cbopaque, format, ap); - va_end(ap); -} - -/* - * Print to stderr in such a way as to (hopefully) avoid memory allocation. - */ -JEMALLOC_ATTR(format(printf, 1, 2)) -void -malloc_printf(const char *format, ...) -{ - va_list ap; - - va_start(ap, format); - malloc_vcprintf(NULL, NULL, format, ap); - va_end(ap); -} -#endif - -#ifdef JEMALLOC_STATS -static void -stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, - unsigned i) -{ - size_t pagesize; - bool config_tcache; - unsigned nbins, j, gap_start; - - CTL_GET("arenas.pagesize", &pagesize, size_t); - - CTL_GET("config.tcache", &config_tcache, bool); - if (config_tcache) { - malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" - " ndalloc nrequests nfills nflushes" - " newruns reruns maxruns curruns\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "bins: bin size regs pgs allocated nmalloc" - " ndalloc newruns reruns maxruns" - " curruns\n"); - } - CTL_GET("arenas.nbins", &nbins, unsigned); - for (j = 0, gap_start = UINT_MAX; j < nbins; j++) { - uint64_t nruns; - - CTL_IJ_GET("stats.arenas.0.bins.0.nruns", &nruns, uint64_t); - if (nruns == 0) { - if (gap_start == UINT_MAX) - gap_start = j; - } else { - unsigned ntbins_, nqbins, ncbins, nsbins; - size_t reg_size, run_size, allocated; - uint32_t nregs; - uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; - uint64_t reruns; - size_t highruns, curruns; - - if (gap_start != UINT_MAX) { - if (j > gap_start + 1) { - /* Gap of more than one size class. */ - malloc_cprintf(write_cb, cbopaque, - "[%u..%u]\n", gap_start, - j - 1); - } else { - /* Gap of one size class. */ - malloc_cprintf(write_cb, cbopaque, - "[%u]\n", gap_start); - } - gap_start = UINT_MAX; - } - CTL_GET("arenas.ntbins", &ntbins_, unsigned); - CTL_GET("arenas.nqbins", &nqbins, unsigned); - CTL_GET("arenas.ncbins", &ncbins, unsigned); - CTL_GET("arenas.nsbins", &nsbins, unsigned); - CTL_J_GET("arenas.bin.0.size", ®_size, size_t); - CTL_J_GET("arenas.bin.0.nregs", &nregs, uint32_t); - CTL_J_GET("arenas.bin.0.run_size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.allocated", - &allocated, size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nmalloc", - &nmalloc, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.ndalloc", - &ndalloc, uint64_t); - if (config_tcache) { - CTL_IJ_GET("stats.arenas.0.bins.0.nrequests", - &nrequests, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nfills", - &nfills, uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.nflushes", - &nflushes, uint64_t); - } - CTL_IJ_GET("stats.arenas.0.bins.0.nreruns", &reruns, - uint64_t); - CTL_IJ_GET("stats.arenas.0.bins.0.highruns", &highruns, - size_t); - CTL_IJ_GET("stats.arenas.0.bins.0.curruns", &curruns, - size_t); - if (config_tcache) { - malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 - " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - j, - j < ntbins_ ? "T" : j < ntbins_ + nqbins ? - "Q" : j < ntbins_ + nqbins + ncbins ? "C" : - "S", - reg_size, nregs, run_size / pagesize, - allocated, nmalloc, ndalloc, nrequests, - nfills, nflushes, nruns, reruns, highruns, - curruns); - } else { - malloc_cprintf(write_cb, cbopaque, - "%13u %1s %5zu %4u %3zu %12zu %12"PRIu64 - " %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - j, - j < ntbins_ ? "T" : j < ntbins_ + nqbins ? - "Q" : j < ntbins_ + nqbins + ncbins ? "C" : - "S", - reg_size, nregs, run_size / pagesize, - allocated, nmalloc, ndalloc, nruns, reruns, - highruns, curruns); - } - } - } - if (gap_start != UINT_MAX) { - if (j > gap_start + 1) { - /* Gap of more than one size class. */ - malloc_cprintf(write_cb, cbopaque, "[%u..%u]\n", - gap_start, j - 1); - } else { - /* Gap of one size class. */ - malloc_cprintf(write_cb, cbopaque, "[%u]\n", gap_start); - } - } -} - -static void -stats_arena_lruns_print(void (*write_cb)(void *, const char *), void *cbopaque, - unsigned i) -{ - size_t pagesize, nlruns, j; - ssize_t gap_start; - - CTL_GET("arenas.pagesize", &pagesize, size_t); - - malloc_cprintf(write_cb, cbopaque, - "large: size pages nmalloc ndalloc nrequests" - " maxruns curruns\n"); - CTL_GET("arenas.nlruns", &nlruns, size_t); - for (j = 0, gap_start = -1; j < nlruns; j++) { - uint64_t nmalloc, ndalloc, nrequests; - size_t run_size, highruns, curruns; - - CTL_IJ_GET("stats.arenas.0.lruns.0.nmalloc", &nmalloc, - uint64_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.ndalloc", &ndalloc, - uint64_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.nrequests", &nrequests, - uint64_t); - if (nrequests == 0) { - if (gap_start == -1) - gap_start = j; - } else { - CTL_J_GET("arenas.lrun.0.size", &run_size, size_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.highruns", &highruns, - size_t); - CTL_IJ_GET("stats.arenas.0.lruns.0.curruns", &curruns, - size_t); - if (gap_start != -1) { - malloc_cprintf(write_cb, cbopaque, "[%zu]\n", - j - gap_start); - gap_start = -1; - } - malloc_cprintf(write_cb, cbopaque, - "%13zu %5zu %12"PRIu64" %12"PRIu64" %12"PRIu64 - " %12zu %12zu\n", - run_size, run_size / pagesize, nmalloc, ndalloc, - nrequests, highruns, curruns); - } - } - if (gap_start != -1) - malloc_cprintf(write_cb, cbopaque, "[%zu]\n", j - gap_start); -} - -static void -stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, - unsigned i) -{ - unsigned nthreads; - size_t pagesize, pactive, pdirty, mapped; - uint64_t npurge, nmadvise, purged; - size_t small_allocated; - uint64_t small_nmalloc, small_ndalloc, small_nrequests; - size_t large_allocated; - uint64_t large_nmalloc, large_ndalloc, large_nrequests; - - CTL_GET("arenas.pagesize", &pagesize, size_t); - - CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned); - malloc_cprintf(write_cb, cbopaque, - "assigned threads: %u\n", nthreads); - CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t); - CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t); - CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t); - CTL_I_GET("stats.arenas.0.nmadvise", &nmadvise, uint64_t); - CTL_I_GET("stats.arenas.0.purged", &purged, uint64_t); - malloc_cprintf(write_cb, cbopaque, - "dirty pages: %zu:%zu active:dirty, %"PRIu64" sweep%s," - " %"PRIu64" madvise%s, %"PRIu64" purged\n", - pactive, pdirty, npurge, npurge == 1 ? "" : "s", - nmadvise, nmadvise == 1 ? "" : "s", purged); - - malloc_cprintf(write_cb, cbopaque, - " allocated nmalloc ndalloc nrequests\n"); - CTL_I_GET("stats.arenas.0.small.allocated", &small_allocated, size_t); - CTL_I_GET("stats.arenas.0.small.nmalloc", &small_nmalloc, uint64_t); - CTL_I_GET("stats.arenas.0.small.ndalloc", &small_ndalloc, uint64_t); - CTL_I_GET("stats.arenas.0.small.nrequests", &small_nrequests, uint64_t); - malloc_cprintf(write_cb, cbopaque, - "small: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", - small_allocated, small_nmalloc, small_ndalloc, small_nrequests); - CTL_I_GET("stats.arenas.0.large.allocated", &large_allocated, size_t); - CTL_I_GET("stats.arenas.0.large.nmalloc", &large_nmalloc, uint64_t); - CTL_I_GET("stats.arenas.0.large.ndalloc", &large_ndalloc, uint64_t); - CTL_I_GET("stats.arenas.0.large.nrequests", &large_nrequests, uint64_t); - malloc_cprintf(write_cb, cbopaque, - "large: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", - large_allocated, large_nmalloc, large_ndalloc, large_nrequests); - malloc_cprintf(write_cb, cbopaque, - "total: %12zu %12"PRIu64" %12"PRIu64" %12"PRIu64"\n", - small_allocated + large_allocated, - small_nmalloc + large_nmalloc, - small_ndalloc + large_ndalloc, - small_nrequests + large_nrequests); - malloc_cprintf(write_cb, cbopaque, "active: %12zu\n", - pactive * pagesize ); - CTL_I_GET("stats.arenas.0.mapped", &mapped, size_t); - malloc_cprintf(write_cb, cbopaque, "mapped: %12zu\n", mapped); - - stats_arena_bins_print(write_cb, cbopaque, i); - stats_arena_lruns_print(write_cb, cbopaque, i); -} -#endif - -void -stats_print(void (*write_cb)(void *, const char *), void *cbopaque, - const char *opts) -{ - int err; - uint64_t epoch; - size_t u64sz; - char s[UMAX2S_BUFSIZE]; - bool general = true; - bool merged = true; - bool unmerged = true; - bool bins = true; - bool large = true; - - /* - * Refresh stats, in case mallctl() was called by the application. - * - * Check for OOM here, since refreshing the ctl cache can trigger - * allocation. In practice, none of the subsequent mallctl()-related - * calls in this function will cause OOM if this one succeeds. - * */ - epoch = 1; - u64sz = sizeof(uint64_t); - err = JEMALLOC_P(mallctl)("epoch", &epoch, &u64sz, &epoch, - sizeof(uint64_t)); - if (err != 0) { - if (err == EAGAIN) { - malloc_write(": Memory allocation failure in " - "mallctl(\"epoch\", ...)\n"); - return; - } - malloc_write(": Failure in mallctl(\"epoch\", " - "...)\n"); - abort(); - } - - if (write_cb == NULL) { - /* - * The caller did not provide an alternate write_cb callback - * function, so use the default one. malloc_write() is an - * inline function, so use malloc_message() directly here. - */ - write_cb = JEMALLOC_P(malloc_message); - cbopaque = NULL; - } - - if (opts != NULL) { - unsigned i; - - for (i = 0; opts[i] != '\0'; i++) { - switch (opts[i]) { - case 'g': - general = false; - break; - case 'm': - merged = false; - break; - case 'a': - unmerged = false; - break; - case 'b': - bins = false; - break; - case 'l': - large = false; - break; - default:; - } - } - } - - write_cb(cbopaque, "___ Begin jemalloc statistics ___\n"); - if (general) { - int err; - const char *cpv; - bool bv; - unsigned uv; - ssize_t ssv; - size_t sv, bsz, ssz, sssz, cpsz; - - bsz = sizeof(bool); - ssz = sizeof(size_t); - sssz = sizeof(ssize_t); - cpsz = sizeof(const char *); - - CTL_GET("version", &cpv, const char *); - write_cb(cbopaque, "Version: "); - write_cb(cbopaque, cpv); - write_cb(cbopaque, "\n"); - CTL_GET("config.debug", &bv, bool); - write_cb(cbopaque, "Assertions "); - write_cb(cbopaque, bv ? "enabled" : "disabled"); - write_cb(cbopaque, "\n"); - -#define OPT_WRITE_BOOL(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &bv, &bsz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, bv ? "true" : "false"); \ - write_cb(cbopaque, "\n"); \ - } -#define OPT_WRITE_SIZE_T(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &sv, &ssz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, u2s(sv, 10, s)); \ - write_cb(cbopaque, "\n"); \ - } -#define OPT_WRITE_SSIZE_T(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &ssv, &sssz, \ - NULL, 0)) == 0) { \ - if (ssv >= 0) { \ - write_cb(cbopaque, " opt."#n": "); \ - write_cb(cbopaque, u2s(ssv, 10, s)); \ - } else { \ - write_cb(cbopaque, " opt."#n": -"); \ - write_cb(cbopaque, u2s(-ssv, 10, s)); \ - } \ - write_cb(cbopaque, "\n"); \ - } -#define OPT_WRITE_CHAR_P(n) \ - if ((err = JEMALLOC_P(mallctl)("opt."#n, &cpv, &cpsz, \ - NULL, 0)) == 0) { \ - write_cb(cbopaque, " opt."#n": \""); \ - write_cb(cbopaque, cpv); \ - write_cb(cbopaque, "\"\n"); \ - } - - write_cb(cbopaque, "Run-time option settings:\n"); - OPT_WRITE_BOOL(abort) - OPT_WRITE_SIZE_T(lg_qspace_max) - OPT_WRITE_SIZE_T(lg_cspace_max) - OPT_WRITE_SIZE_T(lg_chunk) - OPT_WRITE_SIZE_T(narenas) - OPT_WRITE_SSIZE_T(lg_dirty_mult) - OPT_WRITE_BOOL(stats_print) - OPT_WRITE_BOOL(junk) - OPT_WRITE_BOOL(zero) - OPT_WRITE_BOOL(sysv) - OPT_WRITE_BOOL(xmalloc) - OPT_WRITE_BOOL(tcache) - OPT_WRITE_SSIZE_T(lg_tcache_gc_sweep) - OPT_WRITE_SSIZE_T(lg_tcache_max) - OPT_WRITE_BOOL(prof) - OPT_WRITE_CHAR_P(prof_prefix) - OPT_WRITE_SIZE_T(lg_prof_bt_max) - OPT_WRITE_BOOL(prof_active) - OPT_WRITE_SSIZE_T(lg_prof_sample) - OPT_WRITE_BOOL(prof_accum) - OPT_WRITE_SSIZE_T(lg_prof_tcmax) - OPT_WRITE_SSIZE_T(lg_prof_interval) - OPT_WRITE_BOOL(prof_gdump) - OPT_WRITE_BOOL(prof_leak) - OPT_WRITE_BOOL(overcommit) - -#undef OPT_WRITE_BOOL -#undef OPT_WRITE_SIZE_T -#undef OPT_WRITE_SSIZE_T -#undef OPT_WRITE_CHAR_P - - write_cb(cbopaque, "CPUs: "); - write_cb(cbopaque, u2s(ncpus, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.narenas", &uv, unsigned); - write_cb(cbopaque, "Max arenas: "); - write_cb(cbopaque, u2s(uv, 10, s)); - write_cb(cbopaque, "\n"); - - write_cb(cbopaque, "Pointer size: "); - write_cb(cbopaque, u2s(sizeof(void *), 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.quantum", &sv, size_t); - write_cb(cbopaque, "Quantum size: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.cacheline", &sv, size_t); - write_cb(cbopaque, "Cacheline size (assumed): "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("arenas.subpage", &sv, size_t); - write_cb(cbopaque, "Subpage spacing: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - - if ((err = JEMALLOC_P(mallctl)("arenas.tspace_min", &sv, &ssz, - NULL, 0)) == 0) { - write_cb(cbopaque, "Tiny 2^n-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - - CTL_GET("arenas.tspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - } - - CTL_GET("arenas.qspace_min", &sv, size_t); - write_cb(cbopaque, "Quantum-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.qspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("arenas.cspace_min", &sv, size_t); - write_cb(cbopaque, "Cacheline-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.cspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("arenas.sspace_min", &sv, size_t); - write_cb(cbopaque, "Subpage-spaced sizes: ["); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ".."); - CTL_GET("arenas.sspace_max", &sv, size_t); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "]\n"); - - CTL_GET("opt.lg_dirty_mult", &ssv, ssize_t); - if (ssv >= 0) { - write_cb(cbopaque, - "Min active:dirty page ratio per arena: "); - write_cb(cbopaque, u2s((1U << ssv), 10, s)); - write_cb(cbopaque, ":1\n"); - } else { - write_cb(cbopaque, - "Min active:dirty page ratio per arena: N/A\n"); - } - if ((err = JEMALLOC_P(mallctl)("arenas.tcache_max", &sv, - &ssz, NULL, 0)) == 0) { - write_cb(cbopaque, - "Maximum thread-cached size class: "); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, "\n"); - } - if ((err = JEMALLOC_P(mallctl)("opt.lg_tcache_gc_sweep", &ssv, - &ssz, NULL, 0)) == 0) { - size_t tcache_gc_sweep = (1U << ssv); - bool tcache_enabled; - CTL_GET("opt.tcache", &tcache_enabled, bool); - write_cb(cbopaque, "Thread cache GC sweep interval: "); - write_cb(cbopaque, tcache_enabled && ssv >= 0 ? - u2s(tcache_gc_sweep, 10, s) : "N/A"); - write_cb(cbopaque, "\n"); - } - if ((err = JEMALLOC_P(mallctl)("opt.prof", &bv, &bsz, NULL, 0)) - == 0 && bv) { - CTL_GET("opt.lg_prof_bt_max", &sv, size_t); - write_cb(cbopaque, "Maximum profile backtrace depth: "); - write_cb(cbopaque, u2s((1U << sv), 10, s)); - write_cb(cbopaque, "\n"); - - CTL_GET("opt.lg_prof_tcmax", &ssv, ssize_t); - write_cb(cbopaque, - "Maximum per thread backtrace cache: "); - if (ssv >= 0) { - write_cb(cbopaque, u2s((1U << ssv), 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(ssv, 10, s)); - write_cb(cbopaque, ")\n"); - } else - write_cb(cbopaque, "N/A\n"); - - CTL_GET("opt.lg_prof_sample", &sv, size_t); - write_cb(cbopaque, "Average profile sample interval: "); - write_cb(cbopaque, u2s((((uint64_t)1U) << sv), 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ")\n"); - - CTL_GET("opt.lg_prof_interval", &ssv, ssize_t); - write_cb(cbopaque, "Average profile dump interval: "); - if (ssv >= 0) { - write_cb(cbopaque, u2s((((uint64_t)1U) << ssv), - 10, s)); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(ssv, 10, s)); - write_cb(cbopaque, ")\n"); - } else - write_cb(cbopaque, "N/A\n"); - } - CTL_GET("arenas.chunksize", &sv, size_t); - write_cb(cbopaque, "Chunk size: "); - write_cb(cbopaque, u2s(sv, 10, s)); - CTL_GET("opt.lg_chunk", &sv, size_t); - write_cb(cbopaque, " (2^"); - write_cb(cbopaque, u2s(sv, 10, s)); - write_cb(cbopaque, ")\n"); - } - -#ifdef JEMALLOC_STATS - { - int err; - size_t sszp, ssz; - size_t *cactive; - size_t allocated, active, mapped; - size_t chunks_current, chunks_high, swap_avail; - uint64_t chunks_total; - size_t huge_allocated; - uint64_t huge_nmalloc, huge_ndalloc; - - sszp = sizeof(size_t *); - ssz = sizeof(size_t); - - CTL_GET("stats.cactive", &cactive, size_t *); - CTL_GET("stats.allocated", &allocated, size_t); - CTL_GET("stats.active", &active, size_t); - CTL_GET("stats.mapped", &mapped, size_t); - malloc_cprintf(write_cb, cbopaque, - "Allocated: %zu, active: %zu, mapped: %zu\n", - allocated, active, mapped); - malloc_cprintf(write_cb, cbopaque, - "Current active ceiling: %zu\n", atomic_read_z(cactive)); - - /* Print chunk stats. */ - CTL_GET("stats.chunks.total", &chunks_total, uint64_t); - CTL_GET("stats.chunks.high", &chunks_high, size_t); - CTL_GET("stats.chunks.current", &chunks_current, size_t); - if ((err = JEMALLOC_P(mallctl)("swap.avail", &swap_avail, &ssz, - NULL, 0)) == 0) { - size_t lg_chunk; - - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks swap_avail\n"); - CTL_GET("opt.lg_chunk", &lg_chunk, size_t); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64"%13zu%13zu%13zu\n", - chunks_total, chunks_high, chunks_current, - swap_avail << lg_chunk); - } else { - malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " - "highchunks curchunks\n"); - malloc_cprintf(write_cb, cbopaque, - " %13"PRIu64"%13zu%13zu\n", - chunks_total, chunks_high, chunks_current); - } - - /* Print huge stats. */ - CTL_GET("stats.huge.nmalloc", &huge_nmalloc, uint64_t); - CTL_GET("stats.huge.ndalloc", &huge_ndalloc, uint64_t); - CTL_GET("stats.huge.allocated", &huge_allocated, size_t); - malloc_cprintf(write_cb, cbopaque, - "huge: nmalloc ndalloc allocated\n"); - malloc_cprintf(write_cb, cbopaque, - " %12"PRIu64" %12"PRIu64" %12zu\n", - huge_nmalloc, huge_ndalloc, huge_allocated); - - if (merged) { - unsigned narenas; - - CTL_GET("arenas.narenas", &narenas, unsigned); - { - bool initialized[narenas]; - size_t isz; - unsigned i, ninitialized; - - isz = sizeof(initialized); - xmallctl("arenas.initialized", initialized, - &isz, NULL, 0); - for (i = ninitialized = 0; i < narenas; i++) { - if (initialized[i]) - ninitialized++; - } - - if (ninitialized > 1 || unmerged == false) { - /* Print merged arena stats. */ - malloc_cprintf(write_cb, cbopaque, - "\nMerged arenas stats:\n"); - stats_arena_print(write_cb, cbopaque, - narenas); - } - } - } - - if (unmerged) { - unsigned narenas; - - /* Print stats for each arena. */ - - CTL_GET("arenas.narenas", &narenas, unsigned); - { - bool initialized[narenas]; - size_t isz; - unsigned i; - - isz = sizeof(initialized); - xmallctl("arenas.initialized", initialized, - &isz, NULL, 0); - - for (i = 0; i < narenas; i++) { - if (initialized[i]) { - malloc_cprintf(write_cb, - cbopaque, - "\narenas[%u]:\n", i); - stats_arena_print(write_cb, - cbopaque, i); - } - } - } - } - } -#endif /* #ifdef JEMALLOC_STATS */ - write_cb(cbopaque, "--- End jemalloc statistics ---\n"); -} diff --git a/deps/jemalloc.orig/src/tcache.c b/deps/jemalloc.orig/src/tcache.c deleted file mode 100644 index 31c329e1613..00000000000 --- a/deps/jemalloc.orig/src/tcache.c +++ /dev/null @@ -1,480 +0,0 @@ -#define JEMALLOC_TCACHE_C_ -#include "jemalloc/internal/jemalloc_internal.h" -#ifdef JEMALLOC_TCACHE -/******************************************************************************/ -/* Data. */ - -bool opt_tcache = true; -ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; -ssize_t opt_lg_tcache_gc_sweep = LG_TCACHE_GC_SWEEP_DEFAULT; - -tcache_bin_info_t *tcache_bin_info; -static unsigned stack_nelms; /* Total stack elms per tcache. */ - -/* Map of thread-specific caches. */ -#ifndef NO_TLS -__thread tcache_t *tcache_tls JEMALLOC_ATTR(tls_model("initial-exec")); -#endif - -/* - * Same contents as tcache, but initialized such that the TSD destructor is - * called when a thread exits, so that the cache can be cleaned up. - */ -pthread_key_t tcache_tsd; - -size_t nhbins; -size_t tcache_maxclass; -unsigned tcache_gc_incr; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void tcache_thread_cleanup(void *arg); - -/******************************************************************************/ - -void * -tcache_alloc_small_hard(tcache_t *tcache, tcache_bin_t *tbin, size_t binind) -{ - void *ret; - - arena_tcache_fill_small(tcache->arena, tbin, binind -#ifdef JEMALLOC_PROF - , tcache->prof_accumbytes -#endif - ); -#ifdef JEMALLOC_PROF - tcache->prof_accumbytes = 0; -#endif - ret = tcache_alloc_easy(tbin); - - return (ret); -} - -void -tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ) -{ - void *ptr; - unsigned i, nflush, ndeferred; -#ifdef JEMALLOC_STATS - bool merged_stats = false; -#endif - - assert(binind < nbins); - assert(rem <= tbin->ncached); - - for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { - /* Lock the arena bin associated with the first object. */ - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( - tbin->avail[0]); - arena_t *arena = chunk->arena; - arena_bin_t *bin = &arena->bins[binind]; - -#ifdef JEMALLOC_PROF - if (arena == tcache->arena) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, tcache->prof_accumbytes); - malloc_mutex_unlock(&arena->lock); - tcache->prof_accumbytes = 0; - } -#endif - - malloc_mutex_lock(&bin->lock); -#ifdef JEMALLOC_STATS - if (arena == tcache->arena) { - assert(merged_stats == false); - merged_stats = true; - bin->stats.nflushes++; - bin->stats.nrequests += tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; - } -#endif - ndeferred = 0; - for (i = 0; i < nflush; i++) { - ptr = tbin->avail[i]; - assert(ptr != NULL); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == arena) { - size_t pageind = ((uintptr_t)ptr - - (uintptr_t)chunk) >> PAGE_SHIFT; - arena_chunk_map_t *mapelm = - &chunk->map[pageind-map_bias]; - arena_dalloc_bin(arena, chunk, ptr, mapelm); - } else { - /* - * This object was allocated via a different - * arena bin than the one that is currently - * locked. Stash the object, so that it can be - * handled in a future pass. - */ - tbin->avail[ndeferred] = ptr; - ndeferred++; - } - } - malloc_mutex_unlock(&bin->lock); - } -#ifdef JEMALLOC_STATS - if (merged_stats == false) { - /* - * The flush loop didn't happen to flush to this thread's - * arena, so the stats didn't get merged. Manually do so now. - */ - arena_bin_t *bin = &tcache->arena->bins[binind]; - malloc_mutex_lock(&bin->lock); - bin->stats.nflushes++; - bin->stats.nrequests += tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; - malloc_mutex_unlock(&bin->lock); - } -#endif - - memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], - rem * sizeof(void *)); - tbin->ncached = rem; - if ((int)tbin->ncached < tbin->low_water) - tbin->low_water = tbin->ncached; -} - -void -tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache_t *tcache -#endif - ) -{ - void *ptr; - unsigned i, nflush, ndeferred; -#ifdef JEMALLOC_STATS - bool merged_stats = false; -#endif - - assert(binind < nhbins); - assert(rem <= tbin->ncached); - - for (nflush = tbin->ncached - rem; nflush > 0; nflush = ndeferred) { - /* Lock the arena associated with the first object. */ - arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( - tbin->avail[0]); - arena_t *arena = chunk->arena; - - malloc_mutex_lock(&arena->lock); -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - if (arena == tcache->arena) { -#endif -#ifdef JEMALLOC_PROF - arena_prof_accum(arena, tcache->prof_accumbytes); - tcache->prof_accumbytes = 0; -#endif -#ifdef JEMALLOC_STATS - merged_stats = true; - arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[binind - nbins].nrequests += - tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; -#endif -#if (defined(JEMALLOC_PROF) || defined(JEMALLOC_STATS)) - } -#endif - ndeferred = 0; - for (i = 0; i < nflush; i++) { - ptr = tbin->avail[i]; - assert(ptr != NULL); - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); - if (chunk->arena == arena) - arena_dalloc_large(arena, chunk, ptr); - else { - /* - * This object was allocated via a different - * arena than the one that is currently locked. - * Stash the object, so that it can be handled - * in a future pass. - */ - tbin->avail[ndeferred] = ptr; - ndeferred++; - } - } - malloc_mutex_unlock(&arena->lock); - } -#ifdef JEMALLOC_STATS - if (merged_stats == false) { - /* - * The flush loop didn't happen to flush to this thread's - * arena, so the stats didn't get merged. Manually do so now. - */ - arena_t *arena = tcache->arena; - malloc_mutex_lock(&arena->lock); - arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[binind - nbins].nrequests += - tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; - malloc_mutex_unlock(&arena->lock); - } -#endif - - memmove(tbin->avail, &tbin->avail[tbin->ncached - rem], - rem * sizeof(void *)); - tbin->ncached = rem; - if ((int)tbin->ncached < tbin->low_water) - tbin->low_water = tbin->ncached; -} - -tcache_t * -tcache_create(arena_t *arena) -{ - tcache_t *tcache; - size_t size, stack_offset; - unsigned i; - - size = offsetof(tcache_t, tbins) + (sizeof(tcache_bin_t) * nhbins); - /* Naturally align the pointer stacks. */ - size = PTR_CEILING(size); - stack_offset = size; - size += stack_nelms * sizeof(void *); - /* - * Round up to the nearest multiple of the cacheline size, in order to - * avoid the possibility of false cacheline sharing. - * - * That this works relies on the same logic as in ipalloc(), but we - * cannot directly call ipalloc() here due to tcache bootstrapping - * issues. - */ - size = (size + CACHELINE_MASK) & (-CACHELINE); - - if (size <= small_maxclass) - tcache = (tcache_t *)arena_malloc_small(arena, size, true); - else if (size <= tcache_maxclass) - tcache = (tcache_t *)arena_malloc_large(arena, size, true); - else - tcache = (tcache_t *)icalloc(size); - - if (tcache == NULL) - return (NULL); - -#ifdef JEMALLOC_STATS - /* Link into list of extant tcaches. */ - malloc_mutex_lock(&arena->lock); - ql_elm_new(tcache, link); - ql_tail_insert(&arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&arena->lock); -#endif - - tcache->arena = arena; - assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); - for (i = 0; i < nhbins; i++) { - tcache->tbins[i].lg_fill_div = 1; - tcache->tbins[i].avail = (void **)((uintptr_t)tcache + - (uintptr_t)stack_offset); - stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); - } - - TCACHE_SET(tcache); - - return (tcache); -} - -void -tcache_destroy(tcache_t *tcache) -{ - unsigned i; - size_t tcache_size; - -#ifdef JEMALLOC_STATS - /* Unlink from list of extant tcaches. */ - malloc_mutex_lock(&tcache->arena->lock); - ql_remove(&tcache->arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&tcache->arena->lock); - tcache_stats_merge(tcache, tcache->arena); -#endif - - for (i = 0; i < nbins; i++) { - tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_small(tbin, i, 0 -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - -#ifdef JEMALLOC_STATS - if (tbin->tstats.nrequests != 0) { - arena_t *arena = tcache->arena; - arena_bin_t *bin = &arena->bins[i]; - malloc_mutex_lock(&bin->lock); - bin->stats.nrequests += tbin->tstats.nrequests; - malloc_mutex_unlock(&bin->lock); - } -#endif - } - - for (; i < nhbins; i++) { - tcache_bin_t *tbin = &tcache->tbins[i]; - tcache_bin_flush_large(tbin, i, 0 -#if (defined(JEMALLOC_STATS) || defined(JEMALLOC_PROF)) - , tcache -#endif - ); - -#ifdef JEMALLOC_STATS - if (tbin->tstats.nrequests != 0) { - arena_t *arena = tcache->arena; - malloc_mutex_lock(&arena->lock); - arena->stats.nrequests_large += tbin->tstats.nrequests; - arena->stats.lstats[i - nbins].nrequests += - tbin->tstats.nrequests; - malloc_mutex_unlock(&arena->lock); - } -#endif - } - -#ifdef JEMALLOC_PROF - if (tcache->prof_accumbytes > 0) { - malloc_mutex_lock(&tcache->arena->lock); - arena_prof_accum(tcache->arena, tcache->prof_accumbytes); - malloc_mutex_unlock(&tcache->arena->lock); - } -#endif - - tcache_size = arena_salloc(tcache); - if (tcache_size <= small_maxclass) { - arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); - arena_t *arena = chunk->arena; - size_t pageind = ((uintptr_t)tcache - (uintptr_t)chunk) >> - PAGE_SHIFT; - arena_chunk_map_t *mapelm = &chunk->map[pageind-map_bias]; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)((pageind - (mapelm->bits >> PAGE_SHIFT)) << - PAGE_SHIFT)); - arena_bin_t *bin = run->bin; - - malloc_mutex_lock(&bin->lock); - arena_dalloc_bin(arena, chunk, tcache, mapelm); - malloc_mutex_unlock(&bin->lock); - } else if (tcache_size <= tcache_maxclass) { - arena_chunk_t *chunk = CHUNK_ADDR2BASE(tcache); - arena_t *arena = chunk->arena; - - malloc_mutex_lock(&arena->lock); - arena_dalloc_large(arena, chunk, tcache); - malloc_mutex_unlock(&arena->lock); - } else - idalloc(tcache); -} - -static void -tcache_thread_cleanup(void *arg) -{ - tcache_t *tcache = (tcache_t *)arg; - - if (tcache == (void *)(uintptr_t)1) { - /* - * The previous time this destructor was called, we set the key - * to 1 so that other destructors wouldn't cause re-creation of - * the tcache. This time, do nothing, so that the destructor - * will not be called again. - */ - } else if (tcache == (void *)(uintptr_t)2) { - /* - * Another destructor called an allocator function after this - * destructor was called. Reset tcache to 1 in order to - * receive another callback. - */ - TCACHE_SET((uintptr_t)1); - } else if (tcache != NULL) { - assert(tcache != (void *)(uintptr_t)1); - tcache_destroy(tcache); - TCACHE_SET((uintptr_t)1); - } -} - -#ifdef JEMALLOC_STATS -void -tcache_stats_merge(tcache_t *tcache, arena_t *arena) -{ - unsigned i; - - /* Merge and reset tcache stats. */ - for (i = 0; i < nbins; i++) { - arena_bin_t *bin = &arena->bins[i]; - tcache_bin_t *tbin = &tcache->tbins[i]; - malloc_mutex_lock(&bin->lock); - bin->stats.nrequests += tbin->tstats.nrequests; - malloc_mutex_unlock(&bin->lock); - tbin->tstats.nrequests = 0; - } - - for (; i < nhbins; i++) { - malloc_large_stats_t *lstats = &arena->stats.lstats[i - nbins]; - tcache_bin_t *tbin = &tcache->tbins[i]; - arena->stats.nrequests_large += tbin->tstats.nrequests; - lstats->nrequests += tbin->tstats.nrequests; - tbin->tstats.nrequests = 0; - } -} -#endif - -bool -tcache_boot(void) -{ - - if (opt_tcache) { - unsigned i; - - /* - * If necessary, clamp opt_lg_tcache_max, now that - * small_maxclass and arena_maxclass are known. - */ - if (opt_lg_tcache_max < 0 || (1U << - opt_lg_tcache_max) < small_maxclass) - tcache_maxclass = small_maxclass; - else if ((1U << opt_lg_tcache_max) > arena_maxclass) - tcache_maxclass = arena_maxclass; - else - tcache_maxclass = (1U << opt_lg_tcache_max); - - nhbins = nbins + (tcache_maxclass >> PAGE_SHIFT); - - /* Initialize tcache_bin_info. */ - tcache_bin_info = (tcache_bin_info_t *)base_alloc(nhbins * - sizeof(tcache_bin_info_t)); - if (tcache_bin_info == NULL) - return (true); - stack_nelms = 0; - for (i = 0; i < nbins; i++) { - if ((arena_bin_info[i].nregs << 1) <= - TCACHE_NSLOTS_SMALL_MAX) { - tcache_bin_info[i].ncached_max = - (arena_bin_info[i].nregs << 1); - } else { - tcache_bin_info[i].ncached_max = - TCACHE_NSLOTS_SMALL_MAX; - } - stack_nelms += tcache_bin_info[i].ncached_max; - } - for (; i < nhbins; i++) { - tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_LARGE; - stack_nelms += tcache_bin_info[i].ncached_max; - } - - /* Compute incremental GC event threshold. */ - if (opt_lg_tcache_gc_sweep >= 0) { - tcache_gc_incr = ((1U << opt_lg_tcache_gc_sweep) / - nbins) + (((1U << opt_lg_tcache_gc_sweep) % nbins == - 0) ? 0 : 1); - } else - tcache_gc_incr = 0; - - if (pthread_key_create(&tcache_tsd, tcache_thread_cleanup) != - 0) { - malloc_write( - ": Error in pthread_key_create()\n"); - abort(); - } - } - - return (false); -} -/******************************************************************************/ -#endif /* JEMALLOC_TCACHE */ diff --git a/deps/jemalloc.orig/src/zone.c b/deps/jemalloc.orig/src/zone.c deleted file mode 100644 index 2c1b2318ef4..00000000000 --- a/deps/jemalloc.orig/src/zone.c +++ /dev/null @@ -1,354 +0,0 @@ -#include "jemalloc/internal/jemalloc_internal.h" -#ifndef JEMALLOC_ZONE -# error "This source file is for zones on Darwin (OS X)." -#endif - -/******************************************************************************/ -/* Data. */ - -static malloc_zone_t zone, szone; -static struct malloc_introspection_t zone_introspect, ozone_introspect; - -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static size_t zone_size(malloc_zone_t *zone, void *ptr); -static void *zone_malloc(malloc_zone_t *zone, size_t size); -static void *zone_calloc(malloc_zone_t *zone, size_t num, size_t size); -static void *zone_valloc(malloc_zone_t *zone, size_t size); -static void zone_free(malloc_zone_t *zone, void *ptr); -static void *zone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -#if (JEMALLOC_ZONE_VERSION >= 6) -static void *zone_memalign(malloc_zone_t *zone, size_t alignment, - size_t size); -static void zone_free_definite_size(malloc_zone_t *zone, void *ptr, - size_t size); -#endif -static void *zone_destroy(malloc_zone_t *zone); -static size_t zone_good_size(malloc_zone_t *zone, size_t size); -static void zone_force_lock(malloc_zone_t *zone); -static void zone_force_unlock(malloc_zone_t *zone); -static size_t ozone_size(malloc_zone_t *zone, void *ptr); -static void ozone_free(malloc_zone_t *zone, void *ptr); -static void *ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size); -static unsigned ozone_batch_malloc(malloc_zone_t *zone, size_t size, - void **results, unsigned num_requested); -static void ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, - unsigned num); -#if (JEMALLOC_ZONE_VERSION >= 6) -static void ozone_free_definite_size(malloc_zone_t *zone, void *ptr, - size_t size); -#endif -static void ozone_force_lock(malloc_zone_t *zone); -static void ozone_force_unlock(malloc_zone_t *zone); - -/******************************************************************************/ -/* - * Functions. - */ - -static size_t -zone_size(malloc_zone_t *zone, void *ptr) -{ - - /* - * There appear to be places within Darwin (such as setenv(3)) that - * cause calls to this function with pointers that *no* zone owns. If - * we knew that all pointers were owned by *some* zone, we could split - * our zone into two parts, and use one as the default allocator and - * the other as the default deallocator/reallocator. Since that will - * not work in practice, we must check all pointers to assure that they - * reside within a mapped chunk before determining size. - */ - return (ivsalloc(ptr)); -} - -static void * -zone_malloc(malloc_zone_t *zone, size_t size) -{ - - return (JEMALLOC_P(malloc)(size)); -} - -static void * -zone_calloc(malloc_zone_t *zone, size_t num, size_t size) -{ - - return (JEMALLOC_P(calloc)(num, size)); -} - -static void * -zone_valloc(malloc_zone_t *zone, size_t size) -{ - void *ret = NULL; /* Assignment avoids useless compiler warning. */ - - JEMALLOC_P(posix_memalign)(&ret, PAGE_SIZE, size); - - return (ret); -} - -static void -zone_free(malloc_zone_t *zone, void *ptr) -{ - - JEMALLOC_P(free)(ptr); -} - -static void * -zone_realloc(malloc_zone_t *zone, void *ptr, size_t size) -{ - - return (JEMALLOC_P(realloc)(ptr, size)); -} - -#if (JEMALLOC_ZONE_VERSION >= 6) -static void * -zone_memalign(malloc_zone_t *zone, size_t alignment, size_t size) -{ - void *ret = NULL; /* Assignment avoids useless compiler warning. */ - - JEMALLOC_P(posix_memalign)(&ret, alignment, size); - - return (ret); -} - -static void -zone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) -{ - - assert(ivsalloc(ptr) == size); - JEMALLOC_P(free)(ptr); -} -#endif - -static void * -zone_destroy(malloc_zone_t *zone) -{ - - /* This function should never be called. */ - assert(false); - return (NULL); -} - -static size_t -zone_good_size(malloc_zone_t *zone, size_t size) -{ - size_t ret; - void *p; - - /* - * Actually create an object of the appropriate size, then find out - * how large it could have been without moving up to the next size - * class. - */ - p = JEMALLOC_P(malloc)(size); - if (p != NULL) { - ret = isalloc(p); - JEMALLOC_P(free)(p); - } else - ret = size; - - return (ret); -} - -static void -zone_force_lock(malloc_zone_t *zone) -{ - - if (isthreaded) - jemalloc_prefork(); -} - -static void -zone_force_unlock(malloc_zone_t *zone) -{ - - if (isthreaded) - jemalloc_postfork(); -} - -malloc_zone_t * -create_zone(void) -{ - - zone.size = (void *)zone_size; - zone.malloc = (void *)zone_malloc; - zone.calloc = (void *)zone_calloc; - zone.valloc = (void *)zone_valloc; - zone.free = (void *)zone_free; - zone.realloc = (void *)zone_realloc; - zone.destroy = (void *)zone_destroy; - zone.zone_name = "jemalloc_zone"; - zone.batch_malloc = NULL; - zone.batch_free = NULL; - zone.introspect = &zone_introspect; - zone.version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 6) - zone.memalign = zone_memalign; - zone.free_definite_size = zone_free_definite_size; -#endif - - zone_introspect.enumerator = NULL; - zone_introspect.good_size = (void *)zone_good_size; - zone_introspect.check = NULL; - zone_introspect.print = NULL; - zone_introspect.log = NULL; - zone_introspect.force_lock = (void *)zone_force_lock; - zone_introspect.force_unlock = (void *)zone_force_unlock; - zone_introspect.statistics = NULL; -#if (JEMALLOC_ZONE_VERSION >= 6) - zone_introspect.zone_locked = NULL; -#endif - - return (&zone); -} - -static size_t -ozone_size(malloc_zone_t *zone, void *ptr) -{ - size_t ret; - - ret = ivsalloc(ptr); - if (ret == 0) - ret = szone.size(zone, ptr); - - return (ret); -} - -static void -ozone_free(malloc_zone_t *zone, void *ptr) -{ - - if (ivsalloc(ptr) != 0) - JEMALLOC_P(free)(ptr); - else { - size_t size = szone.size(zone, ptr); - if (size != 0) - (szone.free)(zone, ptr); - } -} - -static void * -ozone_realloc(malloc_zone_t *zone, void *ptr, size_t size) -{ - size_t oldsize; - - if (ptr == NULL) - return (JEMALLOC_P(malloc)(size)); - - oldsize = ivsalloc(ptr); - if (oldsize != 0) - return (JEMALLOC_P(realloc)(ptr, size)); - else { - oldsize = szone.size(zone, ptr); - if (oldsize == 0) - return (JEMALLOC_P(malloc)(size)); - else { - void *ret = JEMALLOC_P(malloc)(size); - if (ret != NULL) { - memcpy(ret, ptr, (oldsize < size) ? oldsize : - size); - (szone.free)(zone, ptr); - } - return (ret); - } - } -} - -static unsigned -ozone_batch_malloc(malloc_zone_t *zone, size_t size, void **results, - unsigned num_requested) -{ - - /* Don't bother implementing this interface, since it isn't required. */ - return (0); -} - -static void -ozone_batch_free(malloc_zone_t *zone, void **to_be_freed, unsigned num) -{ - unsigned i; - - for (i = 0; i < num; i++) - ozone_free(zone, to_be_freed[i]); -} - -#if (JEMALLOC_ZONE_VERSION >= 6) -static void -ozone_free_definite_size(malloc_zone_t *zone, void *ptr, size_t size) -{ - - if (ivsalloc(ptr) != 0) { - assert(ivsalloc(ptr) == size); - JEMALLOC_P(free)(ptr); - } else { - assert(size == szone.size(zone, ptr)); - szone.free_definite_size(zone, ptr, size); - } -} -#endif - -static void -ozone_force_lock(malloc_zone_t *zone) -{ - - /* jemalloc locking is taken care of by the normal jemalloc zone. */ - szone.introspect->force_lock(zone); -} - -static void -ozone_force_unlock(malloc_zone_t *zone) -{ - - /* jemalloc locking is taken care of by the normal jemalloc zone. */ - szone.introspect->force_unlock(zone); -} - -/* - * Overlay the default scalable zone (szone) such that existing allocations are - * drained, and further allocations come from jemalloc. This is necessary - * because Core Foundation directly accesses and uses the szone before the - * jemalloc library is even loaded. - */ -void -szone2ozone(malloc_zone_t *zone) -{ - - /* - * Stash a copy of the original szone so that we can call its - * functions as needed. Note that the internally, the szone stores its - * bookkeeping data structures immediately following the malloc_zone_t - * header, so when calling szone functions, we need to pass a pointer - * to the original zone structure. - */ - memcpy(&szone, zone, sizeof(malloc_zone_t)); - - zone->size = (void *)ozone_size; - zone->malloc = (void *)zone_malloc; - zone->calloc = (void *)zone_calloc; - zone->valloc = (void *)zone_valloc; - zone->free = (void *)ozone_free; - zone->realloc = (void *)ozone_realloc; - zone->destroy = (void *)zone_destroy; - zone->zone_name = "jemalloc_ozone"; - zone->batch_malloc = ozone_batch_malloc; - zone->batch_free = ozone_batch_free; - zone->introspect = &ozone_introspect; - zone->version = JEMALLOC_ZONE_VERSION; -#if (JEMALLOC_ZONE_VERSION >= 6) - zone->memalign = zone_memalign; - zone->free_definite_size = ozone_free_definite_size; -#endif - - ozone_introspect.enumerator = NULL; - ozone_introspect.good_size = (void *)zone_good_size; - ozone_introspect.check = NULL; - ozone_introspect.print = NULL; - ozone_introspect.log = NULL; - ozone_introspect.force_lock = (void *)ozone_force_lock; - ozone_introspect.force_unlock = (void *)ozone_force_unlock; - ozone_introspect.statistics = NULL; -#if (JEMALLOC_ZONE_VERSION >= 6) - ozone_introspect.zone_locked = NULL; -#endif -} diff --git a/deps/jemalloc.orig/test/allocated.c b/deps/jemalloc.orig/test/allocated.c deleted file mode 100644 index b1e40e4720e..00000000000 --- a/deps/jemalloc.orig/test/allocated.c +++ /dev/null @@ -1,142 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -void * -thread_start(void *arg) -{ - int err; - void *p; - uint64_t a0, a1, d0, d1; - uint64_t *ap0, *ap1, *dp0, *dp1; - size_t sz, usize; - - sz = sizeof(a0); - if ((err = JEMALLOC_P(mallctl)("thread.allocated", &a0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - sz = sizeof(ap0); - if ((err = JEMALLOC_P(mallctl)("thread.allocatedp", &ap0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - assert(*ap0 == a0); - - sz = sizeof(d0); - if ((err = JEMALLOC_P(mallctl)("thread.deallocated", &d0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - sz = sizeof(dp0); - if ((err = JEMALLOC_P(mallctl)("thread.deallocatedp", &dp0, &sz, NULL, - 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto RETURN; - } - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - assert(*dp0 == d0); - - p = JEMALLOC_P(malloc)(1); - if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); - exit(1); - } - - sz = sizeof(a1); - JEMALLOC_P(mallctl)("thread.allocated", &a1, &sz, NULL, 0); - sz = sizeof(ap1); - JEMALLOC_P(mallctl)("thread.allocatedp", &ap1, &sz, NULL, 0); - assert(*ap1 == a1); - assert(ap0 == ap1); - - usize = JEMALLOC_P(malloc_usable_size)(p); - assert(a0 + usize <= a1); - - JEMALLOC_P(free)(p); - - sz = sizeof(d1); - JEMALLOC_P(mallctl)("thread.deallocated", &d1, &sz, NULL, 0); - sz = sizeof(dp1); - JEMALLOC_P(mallctl)("thread.deallocatedp", &dp1, &sz, NULL, 0); - assert(*dp1 == d1); - assert(dp0 == dp1); - - assert(d0 + usize <= d1); - -RETURN: - return (NULL); -} - -int -main(void) -{ - int ret = 0; - pthread_t thread; - - fprintf(stderr, "Test begin\n"); - - thread_start(NULL); - - if (pthread_create(&thread, NULL, thread_start, NULL) - != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); - ret = 1; - goto RETURN; - } - pthread_join(thread, (void *)&ret); - - thread_start(NULL); - - if (pthread_create(&thread, NULL, thread_start, NULL) - != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", __func__); - ret = 1; - goto RETURN; - } - pthread_join(thread, (void *)&ret); - - thread_start(NULL); - -RETURN: - fprintf(stderr, "Test end\n"); - return (ret); -} diff --git a/deps/jemalloc.orig/test/allocated.exp b/deps/jemalloc.orig/test/allocated.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc.orig/test/allocated.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc.orig/test/allocm.c b/deps/jemalloc.orig/test/allocm.c deleted file mode 100644 index 59d0002e7be..00000000000 --- a/deps/jemalloc.orig/test/allocm.c +++ /dev/null @@ -1,133 +0,0 @@ -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)0x80000000000LLU) */ -#define MAXALIGN ((size_t)0x2000000LLU) -#define NITER 4 - -int -main(void) -{ - int r; - void *p; - size_t sz, alignment, total, tsz; - unsigned i; - void *ps[NITER]; - - fprintf(stderr, "Test begin\n"); - - sz = 0; - r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - if (sz < 42) - fprintf(stderr, "Real size smaller than expected\n"); - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); - - r = JEMALLOC_P(allocm)(&p, NULL, 42, 0); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); - - r = JEMALLOC_P(allocm)(&p, NULL, 42, ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - if (JEMALLOC_P(dallocm)(p, 0) != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected dallocm() error\n"); - -#if LG_SIZEOF_PTR == 3 - alignment = 0x8000000000000000LLU; - sz = 0x8000000000000000LLU; -#else - alignment = 0x80000000LU; - sz = 0x80000000LU; -#endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - -#if LG_SIZEOF_PTR == 3 - alignment = 0x4000000000000000LLU; - sz = 0x8400000000000001LLU; -#else - alignment = 0x40000000LU; - sz = 0x84000001LU; -#endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - - alignment = 0x10LLU; -#if LG_SIZEOF_PTR == 3 - sz = 0xfffffffffffffff0LLU; -#else - sz = 0xfffffff0LU; -#endif - r = JEMALLOC_P(allocm)(&p, NULL, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - fprintf(stderr, - "Expected error for allocm(&p, %zu, 0x%x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - fprintf(stderr, "Alignment: %zu\n", alignment); - for (sz = 1; - sz < 3 * alignment && sz < (1U << 31); - sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - r = JEMALLOC_P(allocm)(&ps[i], NULL, sz, - ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, - "Error for size %zu (0x%zx): %d\n", - sz, sz, r); - exit(1); - } - if ((uintptr_t)p & (alignment-1)) { - fprintf(stderr, - "%p inadequately aligned for" - " alignment: %zu\n", p, alignment); - } - JEMALLOC_P(sallocm)(ps[i], &tsz, 0); - total += tsz; - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - JEMALLOC_P(dallocm)(ps[i], 0); - ps[i] = NULL; - } - } - } - } - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc.orig/test/allocm.exp b/deps/jemalloc.orig/test/allocm.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc.orig/test/allocm.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc.orig/test/bitmap.c b/deps/jemalloc.orig/test/bitmap.c deleted file mode 100644 index adfaacfe52b..00000000000 --- a/deps/jemalloc.orig/test/bitmap.c +++ /dev/null @@ -1,157 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -/* - * Avoid using the assert() from jemalloc_internal.h, since it requires - * internal libjemalloc functionality. - * */ -#include - -/* - * Directly include the bitmap code, since it isn't exposed outside - * libjemalloc. - */ -#include "../src/bitmap.c" - -#if (LG_BITMAP_MAXBITS > 12) -# define MAXBITS 4500 -#else -# define MAXBITS (1U << LG_BITMAP_MAXBITS) -#endif - -static void -test_bitmap_size(void) -{ - size_t i, prev_size; - - prev_size = 0; - for (i = 1; i <= MAXBITS; i++) { - size_t size = bitmap_size(i); - assert(size >= prev_size); - prev_size = size; - } -} - -static void -test_bitmap_init(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - for (j = 0; j < i; j++) - assert(bitmap_get(bitmap, &binfo, j) == false); - - } - } -} - -static void -test_bitmap_set(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - for (j = 0; j < i; j++) - bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); - } - } -} - -static void -test_bitmap_unset(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - size_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - for (j = 0; j < i; j++) - bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); - for (j = 0; j < i; j++) - bitmap_unset(bitmap, &binfo, j); - for (j = 0; j < i; j++) - bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); - } - } -} - -static void -test_bitmap_sfu(void) -{ - size_t i; - - for (i = 1; i <= MAXBITS; i++) { - bitmap_info_t binfo; - bitmap_info_init(&binfo, i); - { - ssize_t j; - bitmap_t bitmap[bitmap_info_ngroups(&binfo)]; - bitmap_init(bitmap, &binfo); - - /* Iteratively set bits starting at the beginning. */ - for (j = 0; j < i; j++) - assert(bitmap_sfu(bitmap, &binfo) == j); - assert(bitmap_full(bitmap, &binfo)); - - /* - * Iteratively unset bits starting at the end, and - * verify that bitmap_sfu() reaches the unset bits. - */ - for (j = i - 1; j >= 0; j--) { - bitmap_unset(bitmap, &binfo, j); - assert(bitmap_sfu(bitmap, &binfo) == j); - bitmap_unset(bitmap, &binfo, j); - } - assert(bitmap_get(bitmap, &binfo, 0) == false); - - /* - * Iteratively set bits starting at the beginning, and - * verify that bitmap_sfu() looks past them. - */ - for (j = 1; j < i; j++) { - bitmap_set(bitmap, &binfo, j - 1); - assert(bitmap_sfu(bitmap, &binfo) == j); - bitmap_unset(bitmap, &binfo, j); - } - assert(bitmap_sfu(bitmap, &binfo) == i - 1); - assert(bitmap_full(bitmap, &binfo)); - } - } -} - -int -main(void) -{ - fprintf(stderr, "Test begin\n"); - - test_bitmap_size(); - test_bitmap_init(); - test_bitmap_set(); - test_bitmap_unset(); - test_bitmap_sfu(); - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc.orig/test/bitmap.exp b/deps/jemalloc.orig/test/bitmap.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc.orig/test/bitmap.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc.orig/test/mremap.c b/deps/jemalloc.orig/test/mremap.c deleted file mode 100644 index 146c66f4212..00000000000 --- a/deps/jemalloc.orig/test/mremap.c +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -int -main(void) -{ - int ret, err; - size_t sz, lg_chunk, chunksize, i; - char *p, *q; - - fprintf(stderr, "Test begin\n"); - - sz = sizeof(lg_chunk); - if ((err = JEMALLOC_P(mallctl)("opt.lg_chunk", &lg_chunk, &sz, NULL, - 0))) { - assert(err != ENOENT); - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - ret = 1; - goto RETURN; - } - chunksize = ((size_t)1U) << lg_chunk; - - p = (char *)malloc(chunksize); - if (p == NULL) { - fprintf(stderr, "malloc(%zu) --> %p\n", chunksize, p); - ret = 1; - goto RETURN; - } - memset(p, 'a', chunksize); - - q = (char *)realloc(p, chunksize * 2); - if (q == NULL) { - fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize * 2, - q); - ret = 1; - goto RETURN; - } - for (i = 0; i < chunksize; i++) { - assert(q[i] == 'a'); - } - - p = q; - - q = (char *)realloc(p, chunksize); - if (q == NULL) { - fprintf(stderr, "realloc(%p, %zu) --> %p\n", p, chunksize, q); - ret = 1; - goto RETURN; - } - for (i = 0; i < chunksize; i++) { - assert(q[i] == 'a'); - } - - free(q); - - ret = 0; -RETURN: - fprintf(stderr, "Test end\n"); - return (ret); -} diff --git a/deps/jemalloc.orig/test/mremap.exp b/deps/jemalloc.orig/test/mremap.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc.orig/test/mremap.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc.orig/test/posix_memalign.c b/deps/jemalloc.orig/test/posix_memalign.c deleted file mode 100644 index 3e306c01dd2..00000000000 --- a/deps/jemalloc.orig/test/posix_memalign.c +++ /dev/null @@ -1,121 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)0x80000000000LLU) */ -#define MAXALIGN ((size_t)0x2000000LLU) -#define NITER 4 - -int -main(void) -{ - size_t alignment, size, total; - unsigned i; - int err; - void *p, *ps[NITER]; - - fprintf(stderr, "Test begin\n"); - - /* Test error conditions. */ - for (alignment = 0; alignment < sizeof(void *); alignment++) { - err = JEMALLOC_P(posix_memalign)(&p, alignment, 1); - if (err != EINVAL) { - fprintf(stderr, - "Expected error for invalid alignment %zu\n", - alignment); - } - } - - for (alignment = sizeof(size_t); alignment < MAXALIGN; - alignment <<= 1) { - err = JEMALLOC_P(posix_memalign)(&p, alignment + 1, 1); - if (err == 0) { - fprintf(stderr, - "Expected error for invalid alignment %zu\n", - alignment + 1); - } - } - -#if LG_SIZEOF_PTR == 3 - alignment = 0x8000000000000000LLU; - size = 0x8000000000000000LLU; -#else - alignment = 0x80000000LU; - size = 0x80000000LU; -#endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); - if (err == 0) { - fprintf(stderr, - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - -#if LG_SIZEOF_PTR == 3 - alignment = 0x4000000000000000LLU; - size = 0x8400000000000001LLU; -#else - alignment = 0x40000000LU; - size = 0x84000001LU; -#endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); - if (err == 0) { - fprintf(stderr, - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - - alignment = 0x10LLU; -#if LG_SIZEOF_PTR == 3 - size = 0xfffffffffffffff0LLU; -#else - size = 0xfffffff0LU; -#endif - err = JEMALLOC_P(posix_memalign)(&p, alignment, size); - if (err == 0) { - fprintf(stderr, - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - fprintf(stderr, "Alignment: %zu\n", alignment); - for (size = 1; - size < 3 * alignment && size < (1U << 31); - size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - err = JEMALLOC_P(posix_memalign)(&ps[i], - alignment, size); - if (err) { - fprintf(stderr, - "Error for size %zu (0x%zx): %s\n", - size, size, strerror(err)); - exit(1); - } - total += JEMALLOC_P(malloc_usable_size)(ps[i]); - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - JEMALLOC_P(free)(ps[i]); - ps[i] = NULL; - } - } - } - } - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc.orig/test/posix_memalign.exp b/deps/jemalloc.orig/test/posix_memalign.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc.orig/test/posix_memalign.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc.orig/test/rallocm.c b/deps/jemalloc.orig/test/rallocm.c deleted file mode 100644 index ccf326bb523..00000000000 --- a/deps/jemalloc.orig/test/rallocm.c +++ /dev/null @@ -1,127 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -int -main(void) -{ - size_t pagesize; - void *p, *q; - size_t sz, tsz; - int r; - - fprintf(stderr, "Test begin\n"); - - /* Get page size. */ - { - long result = sysconf(_SC_PAGESIZE); - assert(result != -1); - pagesize = (size_t)result; - } - - r = JEMALLOC_P(allocm)(&p, &sz, 42, 0); - if (r != ALLOCM_SUCCESS) { - fprintf(stderr, "Unexpected allocm() error\n"); - abort(); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz, 5, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_ERR_NOT_MOVED) - fprintf(stderr, "Unexpected rallocm() result\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz != sz) { - fprintf(stderr, "Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = JEMALLOC_P(rallocm)(&q, &tsz, sz + 5, 0, 0); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q == p) - fprintf(stderr, "Expected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, 0); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q == p) - fprintf(stderr, "Expected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, 0); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*2, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - sz = tsz; - - r = JEMALLOC_P(rallocm)(&q, &tsz, pagesize*4, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - fprintf(stderr, "Unexpected rallocm() error\n"); - if (q != p) - fprintf(stderr, "Unexpected object move\n"); - if (tsz == sz) { - fprintf(stderr, "Expected size change: %zu --> %zu\n", - sz, tsz); - } - sz = tsz; - - JEMALLOC_P(dallocm)(p, 0); - - fprintf(stderr, "Test end\n"); - return (0); -} diff --git a/deps/jemalloc.orig/test/rallocm.exp b/deps/jemalloc.orig/test/rallocm.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc.orig/test/rallocm.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc.orig/test/thread_arena.c b/deps/jemalloc.orig/test/thread_arena.c deleted file mode 100644 index ef8d6817500..00000000000 --- a/deps/jemalloc.orig/test/thread_arena.c +++ /dev/null @@ -1,92 +0,0 @@ -#include -#include -#include -#include -#include - -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define NTHREADS 10 - -void * -thread_start(void *arg) -{ - unsigned main_arena_ind = *(unsigned *)arg; - void *p; - unsigned arena_ind; - size_t size; - int err; - - p = JEMALLOC_P(malloc)(1); - if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); - return (void *)1; - } - - size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, - &main_arena_ind, sizeof(main_arena_ind)))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - return (void *)1; - } - - size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, - 0))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - return (void *)1; - } - assert(arena_ind == main_arena_ind); - - return (NULL); -} - -int -main(void) -{ - int ret = 0; - void *p; - unsigned arena_ind; - size_t size; - int err; - pthread_t threads[NTHREADS]; - unsigned i; - - fprintf(stderr, "Test begin\n"); - - p = JEMALLOC_P(malloc)(1); - if (p == NULL) { - fprintf(stderr, "%s(): Error in malloc()\n", __func__); - ret = 1; - goto RETURN; - } - - size = sizeof(arena_ind); - if ((err = JEMALLOC_P(mallctl)("thread.arena", &arena_ind, &size, NULL, - 0))) { - fprintf(stderr, "%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - ret = 1; - goto RETURN; - } - - for (i = 0; i < NTHREADS; i++) { - if (pthread_create(&threads[i], NULL, thread_start, - (void *)&arena_ind) != 0) { - fprintf(stderr, "%s(): Error in pthread_create()\n", - __func__); - ret = 1; - goto RETURN; - } - } - - for (i = 0; i < NTHREADS; i++) - pthread_join(threads[i], (void *)&ret); - -RETURN: - fprintf(stderr, "Test end\n"); - return (ret); -} diff --git a/deps/jemalloc.orig/test/thread_arena.exp b/deps/jemalloc.orig/test/thread_arena.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc.orig/test/thread_arena.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end From 4dada1b5bce3da69899eb14364142819002cf029 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 May 2012 11:02:38 +0200 Subject: [PATCH 0242/1752] Fixed issue #516 (ZINTERSTORE mixing sets and zsets). Weeks ago trying to fix an harmless GCC warning I introduced a bug in the ziplist-encoded implementations of sorted sets. The bug completely broke zuiNext() iterator, that is used in the ZINTERSTORE and ZUNIONSTORE implementation, so those two commands are no longer reliable starting from Redis version 2.4.12 and latest 2.6.0-RC releases. This commit fixes the problem and adds a regression test. --- src/t_zset.c | 3 ++- tests/unit/type/zset.tcl | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index 50ad8d433d2..4812709e190 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1259,10 +1259,11 @@ int zuiNext(zsetopsrc *op, zsetopval *val) { if (op->type == REDIS_SET) { iterset *it = &op->iter.set; if (op->encoding == REDIS_ENCODING_INTSET) { - int64_t ell = val->ell; + int64_t ell; if (!intsetGet(it->is.is,it->is.ii,&ell)) return 0; + val->ell = ell; val->score = 1.0; /* Move to next element. */ diff --git a/tests/unit/type/zset.tcl b/tests/unit/type/zset.tcl index bab98854db8..806f4c88be8 100644 --- a/tests/unit/type/zset.tcl +++ b/tests/unit/type/zset.tcl @@ -524,6 +524,14 @@ start_server {tags {"zset"}} { r zrange out 0 -1 withscores } {neginf 0} + test {ZINTERSTORE #516 regression, mixed sets and ziplist zsets} { + r sadd one 100 101 102 103 + r sadd two 100 200 201 202 + r zadd three 1 500 1 501 1 502 1 503 1 100 + r zinterstore to_here 3 one two three WEIGHTS 0 0 1 + r zrange to_here 0 -1 + } {100} + proc stressers {encoding} { if {$encoding == "ziplist"} { # Little extra to allow proper fuzzing in the sorting stresser From a5a037bf8134b58b5130168e0eb1526106e5b26c Mon Sep 17 00:00:00 2001 From: jokea Date: Wed, 23 May 2012 17:19:49 +0800 Subject: [PATCH 0243/1752] Set fd to writable when poll(2) detects POLLERR or POLLHUP event. --- src/ae.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ae.c b/src/ae.c index c58c4b74bcc..ba53b456805 100644 --- a/src/ae.c +++ b/src/ae.c @@ -385,6 +385,8 @@ int aeWait(int fd, int mask, long long milliseconds) { if ((retval = poll(&pfd, 1, milliseconds))== 1) { if (pfd.revents & POLLIN) retmask |= AE_READABLE; if (pfd.revents & POLLOUT) retmask |= AE_WRITABLE; + if (pfd.revents & POLLERR) retmask |= AE_WRITABLE; + if (pfd.revents & POLLHUP) retmask |= AE_WRITABLE; return retmask; } else { return retval; From dfa90b596960b64f9dd615154b084768dd11108a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 May 2012 13:13:24 +0200 Subject: [PATCH 0244/1752] Redis test: include bug report on crash. Due to a change in the format of the bug report in case of crash of failed assertion the test suite was no longer able to properly log it. Instead just a protocol error was logged by the Redis TCL client that provided no clue about the actual problem. This commit resolves the issue by logging everything from the first line of the log including the string REDIS BUG REPORT, till the end of the file. --- tests/support/util.tcl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 675d57f78ad..0131c4118ad 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -30,12 +30,16 @@ proc zlistAlikeSort {a b} { proc warnings_from_file {filename} { set lines [split [exec cat $filename] "\n"] set matched 0 + set logall 0 set result {} foreach line $lines { + if {[string match {*REDIS BUG REPORT START*} $line]} { + set logall 1 + } if {[regexp {^\[\d+\]\s+\d+\s+\w+\s+\d{2}:\d{2}:\d{2} \#} $line]} { set matched 1 } - if {$matched} { + if {$logall || $matched} { lappend result $line } } From 27fc5bf582b12a47e7497bc6f7a38a6d033dcb7c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 May 2012 16:50:05 +0200 Subject: [PATCH 0245/1752] Use comments to split aof.c into sections. This makes the code more readable, it is still not the case to split the file itself into three different files, but the logical separation improves the readability especially since new commits are going to introduce an additional section. --- src/aof.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/aof.c b/src/aof.c index 115da29bae2..59b5ab89f1c 100644 --- a/src/aof.c +++ b/src/aof.c @@ -12,6 +12,12 @@ void aofUpdateCurrentSize(void); +/* ---------------------------------------------------------------------------- + * AOF file implementation + * ------------------------------------------------------------------------- */ + +/* Starts a background task that performs fsync() against the specified + * file descriptor (the one of the AOF file) in another thread. */ void aof_background_fsync(int fd) { bioCreateBackgroundJob(REDIS_BIO_AOF_FSYNC,(void*)(long)fd,NULL,NULL); } @@ -280,6 +286,10 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a sdsfree(buf); } +/* ---------------------------------------------------------------------------- + * AOF loading + * ------------------------------------------------------------------------- */ + /* In Redis commands are always executed in the context of a client, so in * order to load the append only file we need to create a fake client. */ struct redisClient *createFakeClient(void) { @@ -424,6 +434,10 @@ int loadAppendOnlyFile(char *filename) { exit(1); } +/* ---------------------------------------------------------------------------- + * AOF rewrite + * ------------------------------------------------------------------------- */ + /* Delegate writing an object to writing a bulk string or bulk long long. * This is not placed in rio.c since that adds the redis.h dependency. */ int rioWriteBulkObject(rio *r, robj *obj) { From 9b6035bc7f17571ebde075e7ecd28796803272dd Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 20 May 2012 23:47:45 +0200 Subject: [PATCH 0246/1752] TODO file removed. The list of things to do is since long time in two places: 1) Github issues. 2) I've a private TOOD list of random ideas, what makes sense is later moved to github issues. So github is anyway the true source of things to do. --- TODO | 45 --------------------------------------------- 1 file changed, 45 deletions(-) delete mode 100644 TODO diff --git a/TODO b/TODO deleted file mode 100644 index 145ec5243e9..00000000000 --- a/TODO +++ /dev/null @@ -1,45 +0,0 @@ -Redis TODO ----------- - -WARNING: are you a possible Redis contributor? - Before implementing what is listed in this file - please drop a message in the Redis google group or chat with - antirez or pietern on irc.freenode.org #redis to check if the work - is already in progress and if the feature is still interesting for - us, and *how* exactly this can be implemented to have good changes - of a merge. Otherwise it is probably wasted work! Thank you - - -CLUSTER -======= - -* Implement rehashing and cluster check in redis-trib. -* Reimplement MIGRATE / RESTORE to use just in memory buffers (no disk at - all). This will require touching a lot of the RDB stuff around, but we may - hand with faster persistence for RDB. -* Implement the slave nodes semantics and election. -* Allow redis-trib to create a cluster-wide snapshot (using SYNC). -* Allow redis-trib to restore a cluster-wide snapshot (implement UPLOAD?). - -SCRIPTING -========= - -* SCRIPT FLUSH or alike to start a fresh interpreter? - -OPTIMIZATIONS -============= - -* SORT: Don't copy the list into a vector when BY argument is constant. -* Write the hash table size of every db in the dump, so that Redis can resize the hash table just one time when loading a big DB. -* Read-only mode for slaves. -* Redis big lists as linked lists of small ziplists? - Possibly a simple heuristic that join near nodes when some node gets smaller than the low_level, and split it into two if gets bigger than high_level. - -KNOWN BUGS -========== - -* #519: Slave may have expired keys that were never read in the master (so a DEL - is not sent in the replication channel) but are already expired since - a lot of time. Maybe after a given delay that is undoubtably greater than - the replication link latency we should expire this key on the slave on - access? From 8152d0c046d319d849ca905822b2da2760a2f136 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 May 2012 11:35:21 +0200 Subject: [PATCH 0247/1752] Dead code removed from replication.c. The user @jokea noticed that the following line of code into replication.c made little sense: addReplySds(slave,sdsempty()); Investigating a bit I found that this was introduced by commit 6208b3a7 three years ago in the early stages of Redis. The code apparently is not useful at all, so I'm removing it. This change will not be backported into 2.4 so that in the rare case this should introduce a bug, we'll have a chance to detect it into the development branch. However following the code path it seems like the code is not useful at all, so the risk is truly small. --- src/replication.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 5c5bc9abfa2..8eb36f837e1 100644 --- a/src/replication.c +++ b/src/replication.c @@ -194,7 +194,6 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { freeClient(slave); return; } - addReplySds(slave,sdsempty()); redisLog(REDIS_NOTICE,"Synchronization with slave succeeded"); } } From edb952532d0f6c267fc0ebf7209404dd65df2e18 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 May 2012 13:03:41 +0200 Subject: [PATCH 0248/1752] Allow an AOF rewrite buffer > 2GB (Fix for issue #504). During the AOF rewrite process, the parent process needs to accumulate the new writes in an in-memory buffer: when the child will terminate the AOF rewriting process this buffer (that ist the difference between the dataset when the rewrite was started, and the current dataset) is flushed to the new AOF file. We used to implement this buffer using an sds.c string, but sds.c has a 2GB limit. Sometimes the dataset can be big enough, the amount of writes so high, and the rewrite process slow enough that we overflow the 2GB limit, causing a crash, documented on github by issue #504. In order to prevent this from happening, this commit introduces a new system to accumulate writes, implemented by a linked list of blocks of 10 MB each, so that we also avoid paying the reallocation cost. Note that theoretically modern operating systems may implement realloc() simply as a remaping of the old pages, thus with very good performances, see for instance the mremap() syscall on Linux. However this is not always true, and jemalloc by default avoids doing this because there are issues with the current implementation of mremap(). For this reason we are using a linked list of blocks instead of a single block that gets reallocated again and again. The changes in this commit lacks testing, that will be performed before merging into the unstable branch. This fix will not enter 2.4 because it is too invasive. However 2.4 will log a warning when the AOF rewrite buffer is near to the 2GB limit. --- src/aof.c | 132 +++++++++++++++++++++++++++++++++++++++++++++------- src/redis.c | 4 +- src/redis.h | 4 +- 3 files changed, 121 insertions(+), 19 deletions(-) diff --git a/src/aof.c b/src/aof.c index 59b5ab89f1c..607bdce8259 100644 --- a/src/aof.c +++ b/src/aof.c @@ -12,6 +12,115 @@ void aofUpdateCurrentSize(void); +/* ---------------------------------------------------------------------------- + * AOF rewrite buffer implementation. + * + * The following code implement a simple buffer used in order to accumulate + * changes while the background process is rewriting the AOF file. + * + * We only need to append, but can't just use realloc with a large block + * because 'huge' reallocs are not always handled as one could expect + * (via remapping of pages at OS level) but may involve copying data. + * + * For this reason we use a list of blocks, every block is + * AOF_RW_BUF_BLOCK_SIZE bytes. + * ------------------------------------------------------------------------- */ + +#define AOF_RW_BUF_BLOCK_SIZE (1024*1024*10) /* 10 MB per block */ + +typedef struct aofrwblock { + unsigned long used, free; + char buf[AOF_RW_BUF_BLOCK_SIZE]; +} aofrwblock; + +/* This function free the old AOF rewrite buffer if needed, and initialize + * a fresh new one. It tests for server.aof_rewrite_buf_blocks equal to NULL + * so can be used for the first initialization as well. */ +void aofRewriteBufferReset(void) { + if (server.aof_rewrite_buf_blocks) + listRelease(server.aof_rewrite_buf_blocks); + + server.aof_rewrite_buf_blocks = listCreate(); + listSetFreeMethod(server.aof_rewrite_buf_blocks,zfree); +} + +/* Return the current size of the AOF rerwite buffer. */ +unsigned long aofRewriteBufferSize(void) { + listNode *ln = listLast(server.aof_rewrite_buf_blocks); + aofrwblock *block = ln ? ln->value : NULL; + + if (block == NULL) return 0; + unsigned long size = + (listLength(server.aof_rewrite_buf_blocks)-1) * AOF_RW_BUF_BLOCK_SIZE; + size += block->used; + return size; +} + +/* Append data to the AOF rewrite buffer, allocating new blocks if needed. */ +void aofRewriteBufferAppend(unsigned char *s, unsigned long len) { + listNode *ln = listLast(server.aof_rewrite_buf_blocks); + aofrwblock *block = ln ? ln->value : NULL; + + while(len) { + /* If we already got at least an allocated block, try appending + * at least some piece into it. */ + if (block) { + unsigned long thislen = (block->free < len) ? block->free : len; + if (thislen) { /* The current block is not already full. */ + memcpy(block->buf+block->used, s, thislen); + block->used += thislen; + block->free -= thislen; + s += thislen; + len -= thislen; + } + } + + if (len) { /* First block to allocate, or need another block. */ + int numblocks; + + block = zmalloc(sizeof(*block)); + block->free = AOF_RW_BUF_BLOCK_SIZE; + block->used = 0; + listAddNodeTail(server.aof_rewrite_buf_blocks,block); + + /* Log every time we cross more 10 or 100 blocks, respectively + * as a notice or warning. */ + numblocks = listLength(server.aof_rewrite_buf_blocks); + if (((numblocks+1) % 10) == 0) { + int level = ((numblocks+1) % 100) == 0 ? REDIS_WARNING : + REDIS_NOTICE; + redisLog(level,"Background AOF buffer size: %lu MB", + aofRewriteBufferSize()/(1024*1024)); + } + } + } +} + +/* Write the buffer (possibly composed of multiple blocks) into the specified + * fd. If no short write or any other error happens -1 is returned, + * otherwise the number of bytes written is returned. */ +ssize_t aofRewriteBufferWrite(int fd) { + listNode *ln; + listIter li; + ssize_t count = 0; + + listRewind(server.aof_rewrite_buf_blocks,&li); + while((ln = listNext(&li))) { + aofrwblock *block = listNodeValue(ln); + ssize_t nwritten; + + if (block->used) { + nwritten = write(fd,block->buf,block->used); + if (nwritten != block->used) { + if (nwritten == 0) errno = EIO; + return -1; + } + count += nwritten; + } + } + return count; +} + /* ---------------------------------------------------------------------------- * AOF file implementation * ------------------------------------------------------------------------- */ @@ -42,8 +151,7 @@ void stopAppendOnly(void) { if (kill(server.aof_child_pid,SIGKILL) != -1) wait3(&statloc,0,NULL); /* reset the buffer accumulating changes while the child saves */ - sdsfree(server.aof_rewrite_buf); - server.aof_rewrite_buf = sdsempty(); + aofRewriteBufferReset(); aofRemoveTempFile(server.aof_child_pid); server.aof_child_pid = -1; } @@ -281,7 +389,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a * in a buffer, so that when the child process will do its work we * can append the differences to the new append only file. */ if (server.aof_child_pid != -1) - server.aof_rewrite_buf = sdscatlen(server.aof_rewrite_buf,buf,sdslen(buf)); + aofRewriteBufferAppend((unsigned char*)buf,sdslen(buf)); sdsfree(buf); } @@ -885,7 +993,6 @@ void aofUpdateCurrentSize(void) { void backgroundRewriteDoneHandler(int exitcode, int bysignal) { if (!bysignal && exitcode == 0) { int newfd, oldfd; - int nwritten; char tmpfile[256]; long long now = ustime(); @@ -903,21 +1010,15 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { goto cleanup; } - nwritten = write(newfd,server.aof_rewrite_buf,sdslen(server.aof_rewrite_buf)); - if (nwritten != (signed)sdslen(server.aof_rewrite_buf)) { - if (nwritten == -1) { - redisLog(REDIS_WARNING, - "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno)); - } else { - redisLog(REDIS_WARNING, - "Short write trying to flush the parent diff to the rewritten AOF: %s", strerror(errno)); - } + if (aofRewriteBufferWrite(newfd) == -1) { + redisLog(REDIS_WARNING, + "Error trying to flush the parent diff to the rewritten AOF: %s", strerror(errno)); close(newfd); goto cleanup; } redisLog(REDIS_NOTICE, - "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", nwritten); + "Parent diff successfully flushed to the rewritten AOF (%lu bytes)", aofRewriteBufferSize()); /* The only remaining thing to do is to rename the temporary file to * the configured file and switch the file descriptor used to do AOF @@ -1009,8 +1110,7 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { } cleanup: - sdsfree(server.aof_rewrite_buf); - server.aof_rewrite_buf = sdsempty(); + aofRewriteBufferReset(); aofRemoveTempFile(server.aof_child_pid); server.aof_child_pid = -1; /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */ diff --git a/src/redis.c b/src/redis.c index e12d27cc339..73d485a1681 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1279,7 +1279,7 @@ void initServer() { server.cronloops = 0; server.rdb_child_pid = -1; server.aof_child_pid = -1; - server.aof_rewrite_buf = sdsempty(); + aofRewriteBufferReset(); server.aof_buf = sdsempty(); server.lastsave = time(NULL); server.dirty = 0; @@ -2134,7 +2134,7 @@ int freeMemoryIfNeeded(void) { } if (server.aof_state != REDIS_AOF_OFF) { mem_used -= sdslen(server.aof_buf); - mem_used -= sdslen(server.aof_rewrite_buf); + mem_used -= aofRewriteBufferSize(); } /* Check if we are over the memory limit. */ diff --git a/src/redis.h b/src/redis.h index 89524160661..5d11799ab84 100644 --- a/src/redis.h +++ b/src/redis.h @@ -505,7 +505,7 @@ struct redisServer { off_t aof_current_size; /* AOF current size. */ int aof_rewrite_scheduled; /* Rewrite once BGSAVE terminates. */ pid_t aof_child_pid; /* PID if rewriting process */ - sds aof_rewrite_buf; /* buffer taken by parent during oppend only rewrite */ + list *aof_rewrite_buf_blocks; /* Hold changes during an AOF rewrite. */ sds aof_buf; /* AOF buffer, written before entering the event loop */ int aof_fd; /* File descriptor of currently selected AOF file */ int aof_selected_db; /* Currently selected DB in AOF */ @@ -843,6 +843,8 @@ int loadAppendOnlyFile(char *filename); void stopAppendOnly(void); int startAppendOnly(void); void backgroundRewriteDoneHandler(int exitcode, int bysignal); +void aofRewriteBufferReset(void); +unsigned long aofRewriteBufferSize(void); /* Sorted sets data type */ From bec74b1b9cd57358f24bd7231f879120a5bf61ab Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 May 2012 15:03:23 +0200 Subject: [PATCH 0249/1752] Add aof_rewrite_buffer_length INFO field. The INFO output, persistence section, already contained the field describing the size of the current AOF buffer to flush on disk. However the other AOF buffer, used to accumulate changes during an AOF rewrite, was not mentioned in the INFO output. This commit introduces a new field called aof_rewrite_buffer_length with the length of the rewrite buffer. --- src/redis.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis.c b/src/redis.c index 73d485a1681..c9a2c5bac39 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1876,12 +1876,14 @@ sds genRedisInfoString(char *section) { "aof_base_size:%lld\r\n" "aof_pending_rewrite:%d\r\n" "aof_buffer_length:%zu\r\n" + "aof_rewrite_buffer_length:%zu\r\n" "aof_pending_bio_fsync:%llu\r\n" "aof_delayed_fsync:%lu\r\n", (long long) server.aof_current_size, (long long) server.aof_rewrite_base_size, server.aof_rewrite_scheduled, sdslen(server.aof_buf), + aofRewriteBufferSize(), bioPendingJobsOfType(REDIS_BIO_AOF_FSYNC), server.aof_delayed_fsync); } From 4a789decaa757e7ceb55cb49248070ba7ae15963 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 16 May 2012 16:23:09 +0200 Subject: [PATCH 0250/1752] New commands: BITOP and BITCOUNT. The motivation for this new commands is to be search in the usage of Redis for real time statistics. See the article "Fast real time metrics using Redis". http://blog.getspool.com/2011/11/29/fast-easy-realtime-metrics-using-redis-bitmaps/ In general Redis strings when used as bitmaps using the SETBIT/GETBIT command provide a very space-efficient and fast way to store statistics. For instance in a web application with users, every user can be associated with a key that shows every day in which the user visited the web service. This information can be really valuable to extract user behaviour information. With Redis bitmaps doing this is very simple just saying that a given day is 0 (the data the service was put online) and all the next days are 1, 2, 3, and so forth. So with SETBIT it is possible to set the bit corresponding to the current day every time the user visits the site. It is possible to take the count of the bit sets on the run, this is extremely easy using a Lua script. However a fast bit count native operation can be useful, especially if it can operate on ranges, or when the string is small like in the case of days (even if you consider many years it is still extremely little data). For this reason BITOP was introduced. The command counts the number of bits set to 1 in a string, with optional range: BITCOUNT key [start end] The start/end parameters are similar to GETRANGE. If omitted the whole string is tested. Population counting is more useful when bit-level operations like AND, OR and XOR are avaialble. For instance I can test multiple users to see the number of days three users visited the site at the same time. To do this we can take the AND of all the bitmaps, and then count the set bits. For this reason the BITOP command was introduced: BITOP [AND|OR|XOR|NOT] dest_key src_key1 src_key2 src_key3 ... src_keyN In the special case of NOT (that inverts the bits) only one source key can be passed. The judicious use of BITCOUNT and BITOP combined can lead to interesting use cases with very space efficient representation of data. The implementation provided is still not tested and optimized for speed, next commits will introduce unit tests. Later the implementation will be profiled to see if it is possible to gain an important amount of speed without making the code much more complex. --- src/redis.c | 4 +- src/redis.h | 2 + src/t_string.c | 149 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index c9a2c5bac39..9937cf56759 100644 --- a/src/redis.c +++ b/src/redis.c @@ -242,7 +242,9 @@ struct redisCommand redisCommandTable[] = { {"evalsha",evalShaCommand,-3,"s",0,zunionInterGetKeys,0,0,0,0,0}, {"slowlog",slowlogCommand,-2,"r",0,NULL,0,0,0,0,0}, {"script",scriptCommand,-2,"ras",0,NULL,0,0,0,0,0}, - {"time",timeCommand,1,"rR",0,NULL,0,0,0,0,0} + {"time",timeCommand,1,"rR",0,NULL,0,0,0,0,0}, + {"bitop",bitopCommand,-4,"wm",0,NULL,2,-1,1,0,0}, + {"bitcount",bitcountCommand,-2,"r",0,NULL,1,1,1,0,0} }; /*============================ Utility functions ============================ */ diff --git a/src/redis.h b/src/redis.h index 5d11799ab84..cf971667c6c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1103,6 +1103,8 @@ void evalCommand(redisClient *c); void evalShaCommand(redisClient *c); void scriptCommand(redisClient *c); void timeCommand(redisClient *c); +void bitopCommand(redisClient *c); +void bitcountCommand(redisClient *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --git a/src/t_string.c b/src/t_string.c index d6143ed27dc..46637d70bb1 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -463,3 +463,152 @@ void strlenCommand(redisClient *c) { addReplyLongLong(c,stringObjectLen(o)); } +#define BITOP_AND 0 +#define BITOP_OR 1 +#define BITOP_XOR 2 +#define BITOP_NOT 3 +/* BITOP op_name target_key src_key1 src_key2 src_key3 ... src_keyN */ +void bitopCommand(redisClient *c) { + char *opname = c->argv[1]->ptr; + robj *o, *targetkey = c->argv[2]; + long op, j, numkeys; + unsigned char **src; /* Array of source strings pointers. */ + long *len, maxlen = 0; /* Array of length of src strings, and max len. */ + unsigned char *res = NULL; /* Resulting string. */ + + /* Parse the operation name. */ + if ((opname[0] == 'a' || opname[0] == 'A') && !strcasecmp(opname,"and")) + op = BITOP_AND; + else if((opname[0] == 'o' || opname[0] == 'O') && !strcasecmp(opname,"or")) + op = BITOP_OR; + else if((opname[0] == 'x' || opname[0] == 'X') && !strcasecmp(opname,"xor")) + op = BITOP_XOR; + else if((opname[0] == 'n' || opname[0] == 'N') && !strcasecmp(opname,"not")) + op = BITOP_NOT; + else { + addReply(c,shared.syntaxerr); + return; + } + + /* Sanity check: NOT accepts only a single key argument. */ + if (op == BITOP_NOT && c->argc != 4) { + addReplyError(c,"BITOP NOT must be called with a single source key."); + return; + } + + /* Lookup keys, and store pointers to the string objects into an array. */ + numkeys = c->argc - 3; + src = zmalloc(sizeof(unsigned char*) * numkeys); + len = zmalloc(sizeof(long) * numkeys); + for (j = 0; j < numkeys; j++) { + o = lookupKeyRead(c->db,c->argv[j+3]); + /* Handle non-existing keys as empty strings. */ + if (o == NULL) { + src[j] = NULL; + len[j] = 0; + continue; + } + /* Return an error if one of the keys is not a string. */ + if (checkType(c,o,REDIS_STRING)) { + zfree(src); + zfree(len); + return; + } + src[j] = o->ptr; + len[j] = sdslen(o->ptr); + if (len[j] > maxlen) maxlen = len[j]; + } + + /* Compute the bit operation, if at least one string is not empty. */ + if (maxlen) { + res = (unsigned char*) sdsnewlen(NULL,maxlen); + unsigned char output, byte; + long i; + + for (j = 0; j < maxlen; j++) { + output = (len[0] <= j) ? 0 : src[0][j]; + if (op == BITOP_NOT) output = ~output; + for (i = 1; i < numkeys; i++) { + byte = (len[i] <= j) ? 0 : src[i][j]; + switch(op) { + case BITOP_AND: output &= byte; break; + case BITOP_OR: output |= byte; break; + case BITOP_XOR: output ^= byte; break; + } + } + res[j] = output; + } + } + zfree(src); + zfree(len); + + /* Store the computed value into the target key */ + if (maxlen) { + o = createObject(REDIS_STRING,res); + setKey(c->db,targetkey,o); + decrRefCount(o); + } else if (dbDelete(c->db,targetkey)) { + signalModifiedKey(c->db,targetkey); + } + server.dirty++; + addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ +} + +/* BITCOUNT key [start end] */ +void bitcountCommand(redisClient *c) { + static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; + robj *o; + long start, end; + unsigned char *p; + char llbuf[32]; + size_t strlen; + + /* Lookup, check for type, and return 0 for non existing keys. */ + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || + checkType(c,o,REDIS_STRING)) return; + + /* Set the 'p' pointer to the string, that can be just a stack allocated + * array if our string was integer encoded. */ + if (o->encoding == REDIS_ENCODING_INT) { + p = (unsigned char*) llbuf; + strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr); + } else { + p = (unsigned char*) o->ptr; + strlen = sdslen(o->ptr); + } + + /* Parse start/end range if any. */ + if (c->argc == 4) { + if (getLongFromObjectOrReply(c,c->argv[2],&start,NULL) != REDIS_OK) + return; + if (getLongFromObjectOrReply(c,c->argv[3],&end,NULL) != REDIS_OK) + return; + /* Convert negative indexes */ + if (start < 0) start = strlen+start; + if (end < 0) end = strlen+end; + if (start < 0) start = 0; + if (end < 0) end = 0; + if ((unsigned)end >= strlen) end = strlen-1; + } else if (c->argc == 2) { + /* The whole string. */ + start = 0; + end = strlen-1; + } else { + /* Syntax error. */ + addReply(c,shared.syntaxerr); + return; + } + + /* Precondition: end >= 0 && end < strlen, so the only condition where + * zero can be returned is: start > end. */ + if (start > end) { + addReply(c,shared.czero); + } else { + long bits = 0, bytes = end-start+1; + + /* We can finally count bits. */ + p += start; + while(bytes--) bits += bitsinbyte[*p++]; + addReplyLongLong(c,bits); + } +} From 1f40cdd0e54394dc2fe007e135ac9b6b9f715950 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 17 May 2012 15:50:44 +0200 Subject: [PATCH 0251/1752] BITOP and BITCOUNT tests. The Redis implementation is tested against Tcl implementations of the same operation. Both fuzzing and testing of specific aspects of the commands behavior are performed. --- tests/test_helper.tcl | 1 + tests/unit/bitops.tcl | 130 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 131 insertions(+) create mode 100644 tests/unit/bitops.tcl diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 7874256da64..853193ccdc1 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -41,6 +41,7 @@ set ::all_tests { unit/limits unit/obuf-limits unit/dump + unit/bitops } # Index to the next test to run in the ::all_tests list. set ::next_test 0 diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl new file mode 100644 index 00000000000..c072f54575a --- /dev/null +++ b/tests/unit/bitops.tcl @@ -0,0 +1,130 @@ +# Compare Redis commadns against Tcl implementations of the same commands. +proc count_bits s { + binary scan $s b* bits + string length [regsub -all {0} $bits {}] +} + +proc simulate_bit_op {op args} { + set maxlen 0 + set j 0 + set count [llength $args] + foreach a $args { + binary scan $a b* bits + set b($j) $bits + if {[string length $bits] > $maxlen} { + set maxlen [string length $bits] + } + incr j + } + for {set j 0} {$j < $count} {incr j} { + if {[string length $b($j)] < $maxlen} { + append b($j) [string repeat 0 [expr $maxlen-[string length $b($j)]]] + } + } + set out {} + for {set x 0} {$x < $maxlen} {incr x} { + set bit [string range $b(0) $x $x] + for {set j 1} {$j < $count} {incr j} { + set bit2 [string range $b($j) $x $x] + switch $op { + and {set bit [expr {$bit & $bit2}]} + or {set bit [expr {$bit | $bit2}]} + xor {set bit [expr {$bit ^ $bit2}]} + not {set bit [expr {!$bit}]} + } + } + append out $bit + } + binary format b* $out +} + +start_server {tags {"bitops"}} { + test {BITCOUNT returns 0 against non existing key} { + r bitcount no-key + } 0 + + catch {unset num} + foreach vec [list \ + "" "\xaa" "\x00\x00\xff" "foobar" \ + [randstring 2000 3000] [randstring 2000 3000] \ + [randstring 2000 3000] \ + ] { + incr num + test "BITCOUNT against test vector #$num" { + r set str $vec + assert {[r bitcount str] == [count_bits $vec]} + } + } + + test {BITCOUNT with start, end} { + r set s "foobar" + assert_equal [r bitcount s 0 -1] [count_bits "foobar"] + assert_equal [r bitcount s 1 -2] [count_bits "ooba"] + assert_equal [r bitcount s -2 1] [count_bits ""] + assert_equal [r bitcount s 0 1000] [count_bits "foobar"] + } + + test {BITCOUNT syntax error #1} { + catch {r bitcount s 0} e + set e + } {ERR*syntax*} + + test {BITOP NOT (empty string)} { + r set s "" + r bitop not dest s + r get dest + } {} + + test {BITOP NOT (known string)} { + r set s "\xaa\x00\xff\x55" + r bitop not dest s + r get dest + } "\x55\xff\x00\xaa" + + test {BITOP where dest and target are the same key} { + r set s "\xaa\x00\xff\x55" + r bitop not s s + r get s + } "\x55\xff\x00\xaa" + + test {BITOP AND|OR|XOR don't change the string with single input key} { + r set a "\x01\x02\xff" + r bitop and res1 a + r bitop or res2 a + r bitop xor res3 a + list [r get res1] [r get res2] [r get res3] + } [list "\x01\x02\xff" "\x01\x02\xff" "\x01\x02\xff"] + + test {BITOP missing key is considered a stream of zero} { + r set a "\x01\x02\xff" + r bitop and res1 no-suck-key a + r bitop or res2 no-suck-key a no-such-key + r bitop xor res3 no-such-key a + list [r get res1] [r get res2] [r get res3] + } [list "\x00\x00\x00" "\x01\x02\xff" "\x01\x02\xff"] + + test {BITOP shorter keys are zero-padded to the key with max length} { + r set a "\x01\x02\xff\xff" + r set b "\x01\x02\xff" + r bitop and res1 a b + r bitop or res2 a b + r bitop xor res3 a b + list [r get res1] [r get res2] [r get res3] + } [list "\x01\x02\xff\x00" "\x01\x02\xff\xff" "\x00\x00\x00\xff"] + + foreach op {and or xor} { + test "BITOP $op fuzzing" { + set vec {} + set veckeys {} + set numvec [expr {[randomInt 10]+1}] + for {set j 0} {$j < $numvec} {incr j} { + set str [randstring 0 1000] + lappend vec $str + lappend veckeys vector_$j + r set vector_$j $str + } + r bitop $op target {*}$veckeys + assert_equal [r get target] [simulate_bit_op $op {*}$vec] + } + } +} From 84ec6706cfe05f699fdbc196ae5506cd99d89863 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 19 May 2012 10:33:20 +0200 Subject: [PATCH 0252/1752] Bit-related string operations moved to bitop.c All the general string operations are implemented in t_string.c, however the bit operations, while targeting the string type, are better served in a specific file where we have the implementations of the following four commands and helper functions: GETBIT SETBIT BITOP BITCOUNT In the future this file will probably contain more code related to making the BITOP and BITCOUNT operations faster. --- src/Makefile | 2 +- src/bitop.c | 259 +++++++++++++++++++++++++++++++++++++++++++++++++ src/t_string.c | 248 ---------------------------------------------- 3 files changed, 260 insertions(+), 249 deletions(-) create mode 100644 src/bitop.c diff --git a/src/Makefile b/src/Makefile index e8543d95732..e8ff5d9838a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -96,7 +96,7 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN endif REDIS_SERVER_NAME= redis-server -REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o +REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitop.o REDIS_CLI_NAME= redis-cli REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o REDIS_BENCHMARK_NAME= redis-benchmark diff --git a/src/bitop.c b/src/bitop.c new file mode 100644 index 00000000000..1eec63ab961 --- /dev/null +++ b/src/bitop.c @@ -0,0 +1,259 @@ +#include "redis.h" + +/* ----------------------------------------------------------------------------- + * Bits related string commands: GETBIT, SETBIT, BITCOUNT, BITOP. + * -------------------------------------------------------------------------- */ + +#define BITOP_AND 0 +#define BITOP_OR 1 +#define BITOP_XOR 2 +#define BITOP_NOT 3 + +/* This helper function used by GETBIT / SETBIT parses the bit offset arguemnt + * making sure an error is returned if it is negative or if it overflows + * Redis 512 MB limit for the string value. */ +static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { + long long loffset; + char *err = "bit offset is not an integer or out of range"; + + if (getLongLongFromObjectOrReply(c,o,&loffset,err) != REDIS_OK) + return REDIS_ERR; + + /* Limit offset to 512MB in bytes */ + if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024)) + { + addReplyError(c,err); + return REDIS_ERR; + } + + *offset = (size_t)loffset; + return REDIS_OK; +} + +/* SETBIT key offset bitvalue */ +void setbitCommand(redisClient *c) { + robj *o; + char *err = "bit is not an integer or out of range"; + size_t bitoffset; + int byte, bit; + int byteval, bitval; + long on; + + if (getBitOffsetFromArgument(c,c->argv[2],&bitoffset) != REDIS_OK) + return; + + if (getLongFromObjectOrReply(c,c->argv[3],&on,err) != REDIS_OK) + return; + + /* Bits can only be set or cleared... */ + if (on & ~1) { + addReplyError(c,err); + return; + } + + o = lookupKeyWrite(c->db,c->argv[1]); + if (o == NULL) { + o = createObject(REDIS_STRING,sdsempty()); + dbAdd(c->db,c->argv[1],o); + } else { + if (checkType(c,o,REDIS_STRING)) return; + + /* Create a copy when the object is shared or encoded. */ + if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) { + robj *decoded = getDecodedObject(o); + o = createStringObject(decoded->ptr, sdslen(decoded->ptr)); + decrRefCount(decoded); + dbOverwrite(c->db,c->argv[1],o); + } + } + + /* Grow sds value to the right length if necessary */ + byte = bitoffset >> 3; + o->ptr = sdsgrowzero(o->ptr,byte+1); + + /* Get current values */ + byteval = ((char*)o->ptr)[byte]; + bit = 7 - (bitoffset & 0x7); + bitval = byteval & (1 << bit); + + /* Update byte with new bit value and return original value */ + byteval &= ~(1 << bit); + byteval |= ((on & 0x1) << bit); + ((char*)o->ptr)[byte] = byteval; + signalModifiedKey(c->db,c->argv[1]); + server.dirty++; + addReply(c, bitval ? shared.cone : shared.czero); +} + +/* GETBIT key offset */ +void getbitCommand(redisClient *c) { + robj *o; + char llbuf[32]; + size_t bitoffset; + size_t byte, bit; + size_t bitval = 0; + + if (getBitOffsetFromArgument(c,c->argv[2],&bitoffset) != REDIS_OK) + return; + + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || + checkType(c,o,REDIS_STRING)) return; + + byte = bitoffset >> 3; + bit = 7 - (bitoffset & 0x7); + if (o->encoding != REDIS_ENCODING_RAW) { + if (byte < (size_t)ll2string(llbuf,sizeof(llbuf),(long)o->ptr)) + bitval = llbuf[byte] & (1 << bit); + } else { + if (byte < sdslen(o->ptr)) + bitval = ((char*)o->ptr)[byte] & (1 << bit); + } + + addReply(c, bitval ? shared.cone : shared.czero); +} + +/* BITOP op_name target_key src_key1 src_key2 src_key3 ... src_keyN */ +void bitopCommand(redisClient *c) { + char *opname = c->argv[1]->ptr; + robj *o, *targetkey = c->argv[2]; + long op, j, numkeys; + unsigned char **src; /* Array of source strings pointers. */ + long *len, maxlen = 0; /* Array of length of src strings, and max len. */ + unsigned char *res = NULL; /* Resulting string. */ + + /* Parse the operation name. */ + if ((opname[0] == 'a' || opname[0] == 'A') && !strcasecmp(opname,"and")) + op = BITOP_AND; + else if((opname[0] == 'o' || opname[0] == 'O') && !strcasecmp(opname,"or")) + op = BITOP_OR; + else if((opname[0] == 'x' || opname[0] == 'X') && !strcasecmp(opname,"xor")) + op = BITOP_XOR; + else if((opname[0] == 'n' || opname[0] == 'N') && !strcasecmp(opname,"not")) + op = BITOP_NOT; + else { + addReply(c,shared.syntaxerr); + return; + } + + /* Sanity check: NOT accepts only a single key argument. */ + if (op == BITOP_NOT && c->argc != 4) { + addReplyError(c,"BITOP NOT must be called with a single source key."); + return; + } + + /* Lookup keys, and store pointers to the string objects into an array. */ + numkeys = c->argc - 3; + src = zmalloc(sizeof(unsigned char*) * numkeys); + len = zmalloc(sizeof(long) * numkeys); + for (j = 0; j < numkeys; j++) { + o = lookupKeyRead(c->db,c->argv[j+3]); + /* Handle non-existing keys as empty strings. */ + if (o == NULL) { + src[j] = NULL; + len[j] = 0; + continue; + } + /* Return an error if one of the keys is not a string. */ + if (checkType(c,o,REDIS_STRING)) { + zfree(src); + zfree(len); + return; + } + src[j] = o->ptr; + len[j] = sdslen(o->ptr); + if (len[j] > maxlen) maxlen = len[j]; + } + + /* Compute the bit operation, if at least one string is not empty. */ + if (maxlen) { + res = (unsigned char*) sdsnewlen(NULL,maxlen); + unsigned char output, byte; + long i; + + for (j = 0; j < maxlen; j++) { + output = (len[0] <= j) ? 0 : src[0][j]; + if (op == BITOP_NOT) output = ~output; + for (i = 1; i < numkeys; i++) { + byte = (len[i] <= j) ? 0 : src[i][j]; + switch(op) { + case BITOP_AND: output &= byte; break; + case BITOP_OR: output |= byte; break; + case BITOP_XOR: output ^= byte; break; + } + } + res[j] = output; + } + } + zfree(src); + zfree(len); + + /* Store the computed value into the target key */ + if (maxlen) { + o = createObject(REDIS_STRING,res); + setKey(c->db,targetkey,o); + decrRefCount(o); + } else if (dbDelete(c->db,targetkey)) { + signalModifiedKey(c->db,targetkey); + } + server.dirty++; + addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ +} + +/* BITCOUNT key [start end] */ +void bitcountCommand(redisClient *c) { + static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; + robj *o; + long start, end; + unsigned char *p; + char llbuf[32]; + size_t strlen; + + /* Lookup, check for type, and return 0 for non existing keys. */ + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || + checkType(c,o,REDIS_STRING)) return; + + /* Set the 'p' pointer to the string, that can be just a stack allocated + * array if our string was integer encoded. */ + if (o->encoding == REDIS_ENCODING_INT) { + p = (unsigned char*) llbuf; + strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr); + } else { + p = (unsigned char*) o->ptr; + strlen = sdslen(o->ptr); + } + + /* Parse start/end range if any. */ + if (c->argc == 4) { + if (getLongFromObjectOrReply(c,c->argv[2],&start,NULL) != REDIS_OK) + return; + if (getLongFromObjectOrReply(c,c->argv[3],&end,NULL) != REDIS_OK) + return; + /* Convert negative indexes */ + if (start < 0) start = strlen+start; + if (end < 0) end = strlen+end; + if (start < 0) start = 0; + if (end < 0) end = 0; + if ((unsigned)end >= strlen) end = strlen-1; + } else if (c->argc == 2) { + /* The whole string. */ + start = 0; + end = strlen-1; + } else { + /* Syntax error. */ + addReply(c,shared.syntaxerr); + return; + } + + /* Precondition: end >= 0 && end < strlen, so the only condition where + * zero can be returned is: start > end. */ + if (start > end) { + addReply(c,shared.czero); + } else { + long bits = 0, bytes = end-start+1; + + /* We can finally count bits. */ + p += start; + while(bytes--) bits += bitsinbyte[*p++]; + addReplyLongLong(c,bits); + } +} diff --git a/src/t_string.c b/src/t_string.c index 46637d70bb1..1e29a613347 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -82,104 +82,6 @@ void getsetCommand(redisClient *c) { server.dirty++; } -static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { - long long loffset; - char *err = "bit offset is not an integer or out of range"; - - if (getLongLongFromObjectOrReply(c,o,&loffset,err) != REDIS_OK) - return REDIS_ERR; - - /* Limit offset to 512MB in bytes */ - if ((loffset < 0) || ((unsigned long long)loffset >> 3) >= (512*1024*1024)) - { - addReplyError(c,err); - return REDIS_ERR; - } - - *offset = (size_t)loffset; - return REDIS_OK; -} - -void setbitCommand(redisClient *c) { - robj *o; - char *err = "bit is not an integer or out of range"; - size_t bitoffset; - int byte, bit; - int byteval, bitval; - long on; - - if (getBitOffsetFromArgument(c,c->argv[2],&bitoffset) != REDIS_OK) - return; - - if (getLongFromObjectOrReply(c,c->argv[3],&on,err) != REDIS_OK) - return; - - /* Bits can only be set or cleared... */ - if (on & ~1) { - addReplyError(c,err); - return; - } - - o = lookupKeyWrite(c->db,c->argv[1]); - if (o == NULL) { - o = createObject(REDIS_STRING,sdsempty()); - dbAdd(c->db,c->argv[1],o); - } else { - if (checkType(c,o,REDIS_STRING)) return; - - /* Create a copy when the object is shared or encoded. */ - if (o->refcount != 1 || o->encoding != REDIS_ENCODING_RAW) { - robj *decoded = getDecodedObject(o); - o = createStringObject(decoded->ptr, sdslen(decoded->ptr)); - decrRefCount(decoded); - dbOverwrite(c->db,c->argv[1],o); - } - } - - /* Grow sds value to the right length if necessary */ - byte = bitoffset >> 3; - o->ptr = sdsgrowzero(o->ptr,byte+1); - - /* Get current values */ - byteval = ((char*)o->ptr)[byte]; - bit = 7 - (bitoffset & 0x7); - bitval = byteval & (1 << bit); - - /* Update byte with new bit value and return original value */ - byteval &= ~(1 << bit); - byteval |= ((on & 0x1) << bit); - ((char*)o->ptr)[byte] = byteval; - signalModifiedKey(c->db,c->argv[1]); - server.dirty++; - addReply(c, bitval ? shared.cone : shared.czero); -} - -void getbitCommand(redisClient *c) { - robj *o; - char llbuf[32]; - size_t bitoffset; - size_t byte, bit; - size_t bitval = 0; - - if (getBitOffsetFromArgument(c,c->argv[2],&bitoffset) != REDIS_OK) - return; - - if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || - checkType(c,o,REDIS_STRING)) return; - - byte = bitoffset >> 3; - bit = 7 - (bitoffset & 0x7); - if (o->encoding != REDIS_ENCODING_RAW) { - if (byte < (size_t)ll2string(llbuf,sizeof(llbuf),(long)o->ptr)) - bitval = llbuf[byte] & (1 << bit); - } else { - if (byte < sdslen(o->ptr)) - bitval = ((char*)o->ptr)[byte] & (1 << bit); - } - - addReply(c, bitval ? shared.cone : shared.czero); -} - void setrangeCommand(redisClient *c) { robj *o; long offset; @@ -462,153 +364,3 @@ void strlenCommand(redisClient *c) { checkType(c,o,REDIS_STRING)) return; addReplyLongLong(c,stringObjectLen(o)); } - -#define BITOP_AND 0 -#define BITOP_OR 1 -#define BITOP_XOR 2 -#define BITOP_NOT 3 -/* BITOP op_name target_key src_key1 src_key2 src_key3 ... src_keyN */ -void bitopCommand(redisClient *c) { - char *opname = c->argv[1]->ptr; - robj *o, *targetkey = c->argv[2]; - long op, j, numkeys; - unsigned char **src; /* Array of source strings pointers. */ - long *len, maxlen = 0; /* Array of length of src strings, and max len. */ - unsigned char *res = NULL; /* Resulting string. */ - - /* Parse the operation name. */ - if ((opname[0] == 'a' || opname[0] == 'A') && !strcasecmp(opname,"and")) - op = BITOP_AND; - else if((opname[0] == 'o' || opname[0] == 'O') && !strcasecmp(opname,"or")) - op = BITOP_OR; - else if((opname[0] == 'x' || opname[0] == 'X') && !strcasecmp(opname,"xor")) - op = BITOP_XOR; - else if((opname[0] == 'n' || opname[0] == 'N') && !strcasecmp(opname,"not")) - op = BITOP_NOT; - else { - addReply(c,shared.syntaxerr); - return; - } - - /* Sanity check: NOT accepts only a single key argument. */ - if (op == BITOP_NOT && c->argc != 4) { - addReplyError(c,"BITOP NOT must be called with a single source key."); - return; - } - - /* Lookup keys, and store pointers to the string objects into an array. */ - numkeys = c->argc - 3; - src = zmalloc(sizeof(unsigned char*) * numkeys); - len = zmalloc(sizeof(long) * numkeys); - for (j = 0; j < numkeys; j++) { - o = lookupKeyRead(c->db,c->argv[j+3]); - /* Handle non-existing keys as empty strings. */ - if (o == NULL) { - src[j] = NULL; - len[j] = 0; - continue; - } - /* Return an error if one of the keys is not a string. */ - if (checkType(c,o,REDIS_STRING)) { - zfree(src); - zfree(len); - return; - } - src[j] = o->ptr; - len[j] = sdslen(o->ptr); - if (len[j] > maxlen) maxlen = len[j]; - } - - /* Compute the bit operation, if at least one string is not empty. */ - if (maxlen) { - res = (unsigned char*) sdsnewlen(NULL,maxlen); - unsigned char output, byte; - long i; - - for (j = 0; j < maxlen; j++) { - output = (len[0] <= j) ? 0 : src[0][j]; - if (op == BITOP_NOT) output = ~output; - for (i = 1; i < numkeys; i++) { - byte = (len[i] <= j) ? 0 : src[i][j]; - switch(op) { - case BITOP_AND: output &= byte; break; - case BITOP_OR: output |= byte; break; - case BITOP_XOR: output ^= byte; break; - } - } - res[j] = output; - } - } - zfree(src); - zfree(len); - - /* Store the computed value into the target key */ - if (maxlen) { - o = createObject(REDIS_STRING,res); - setKey(c->db,targetkey,o); - decrRefCount(o); - } else if (dbDelete(c->db,targetkey)) { - signalModifiedKey(c->db,targetkey); - } - server.dirty++; - addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ -} - -/* BITCOUNT key [start end] */ -void bitcountCommand(redisClient *c) { - static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; - robj *o; - long start, end; - unsigned char *p; - char llbuf[32]; - size_t strlen; - - /* Lookup, check for type, and return 0 for non existing keys. */ - if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.czero)) == NULL || - checkType(c,o,REDIS_STRING)) return; - - /* Set the 'p' pointer to the string, that can be just a stack allocated - * array if our string was integer encoded. */ - if (o->encoding == REDIS_ENCODING_INT) { - p = (unsigned char*) llbuf; - strlen = ll2string(llbuf,sizeof(llbuf),(long)o->ptr); - } else { - p = (unsigned char*) o->ptr; - strlen = sdslen(o->ptr); - } - - /* Parse start/end range if any. */ - if (c->argc == 4) { - if (getLongFromObjectOrReply(c,c->argv[2],&start,NULL) != REDIS_OK) - return; - if (getLongFromObjectOrReply(c,c->argv[3],&end,NULL) != REDIS_OK) - return; - /* Convert negative indexes */ - if (start < 0) start = strlen+start; - if (end < 0) end = strlen+end; - if (start < 0) start = 0; - if (end < 0) end = 0; - if ((unsigned)end >= strlen) end = strlen-1; - } else if (c->argc == 2) { - /* The whole string. */ - start = 0; - end = strlen-1; - } else { - /* Syntax error. */ - addReply(c,shared.syntaxerr); - return; - } - - /* Precondition: end >= 0 && end < strlen, so the only condition where - * zero can be returned is: start > end. */ - if (start > end) { - addReply(c,shared.czero); - } else { - long bits = 0, bytes = end-start+1; - - /* We can finally count bits. */ - p += start; - while(bytes--) bits += bitsinbyte[*p++]; - addReplyLongLong(c,bits); - } -} From 7ffa248c7cc53d1b84dfbad2149965efcd6a72f5 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 19 May 2012 16:16:25 +0200 Subject: [PATCH 0253/1752] BITCOUNT refactoring. The low level popualtion counting function is now separated from the BITCOUNT command implementation, so that the low level function can be further optimized and eventually used in other contexts if needed. --- src/bitop.c | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/src/bitop.c b/src/bitop.c index 1eec63ab961..2b26265d7be 100644 --- a/src/bitop.c +++ b/src/bitop.c @@ -1,14 +1,9 @@ #include "redis.h" /* ----------------------------------------------------------------------------- - * Bits related string commands: GETBIT, SETBIT, BITCOUNT, BITOP. + * Helpers and low level bit functions. * -------------------------------------------------------------------------- */ -#define BITOP_AND 0 -#define BITOP_OR 1 -#define BITOP_XOR 2 -#define BITOP_NOT 3 - /* This helper function used by GETBIT / SETBIT parses the bit offset arguemnt * making sure an error is returned if it is negative or if it overflows * Redis 512 MB limit for the string value. */ @@ -30,6 +25,28 @@ static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { return REDIS_OK; } +/* Count number of bits set in the binary array pointed by 's' and long + * 'count' bytes. The implementation of this function is required to + * work with a input string length up to 512 MB. */ +long popcount(void *s, long count) { + long bits = 0; + unsigned char *p = s; + static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; + + /* We can finally count bits. */ + while(count--) bits += bitsinbyte[*p++]; + return bits; +} + +/* ----------------------------------------------------------------------------- + * Bits related string commands: GETBIT, SETBIT, BITCOUNT, BITOP. + * -------------------------------------------------------------------------- */ + +#define BITOP_AND 0 +#define BITOP_OR 1 +#define BITOP_XOR 2 +#define BITOP_NOT 3 + /* SETBIT key offset bitvalue */ void setbitCommand(redisClient *c) { robj *o; @@ -201,7 +218,6 @@ void bitopCommand(redisClient *c) { /* BITCOUNT key [start end] */ void bitcountCommand(redisClient *c) { - static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; robj *o; long start, end; unsigned char *p; @@ -249,11 +265,8 @@ void bitcountCommand(redisClient *c) { if (start > end) { addReply(c,shared.czero); } else { - long bits = 0, bytes = end-start+1; + long bytes = end-start+1; - /* We can finally count bits. */ - p += start; - while(bytes--) bits += bitsinbyte[*p++]; - addReplyLongLong(c,bits); + addReplyLongLong(c,popcount(p+start,bytes)); } } From d12a68d4158ba308b3e153158a0affa80c5fa9eb Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 20 May 2012 00:49:35 +0200 Subject: [PATCH 0254/1752] popcount() optimization for speed. We run the array by 32 bit words instead of processing it byte per byte. If the code is compiled using GCC __builtin_popcount() builtin function is used instead. --- src/bitop.c | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/bitop.c b/src/bitop.c index 2b26265d7be..053db02b5cf 100644 --- a/src/bitop.c +++ b/src/bitop.c @@ -30,10 +30,26 @@ static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { * work with a input string length up to 512 MB. */ long popcount(void *s, long count) { long bits = 0; - unsigned char *p = s; + unsigned char *p; + uint32_t *p4 = s; static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; - /* We can finally count bits. */ + /* Count bits four bytes at a time */ + while(count>=4) { + uint32_t aux = *p4++; + count -= 4; +#ifdef __GNUC__ + /* Unsigned int is always >= 4 bytes if compiler is GCC */ + bits += __builtin_popcount(aux); +#else + bits += bitsinbyte[aux&0xff] + + bitsinbyte[(aux>>8)&0xff] + + bitsinbyte[(aux>>16)&0xff] + + bitsinbyte[(aux>>24)&0xff]; +#endif + } + /* Count the remaining bytes */ + p = (unsigned char*)p4; while(count--) bits += bitsinbyte[*p++]; return bits; } From 951213e2d270c58ad776c6be49ee9e9bb8acc5d0 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 20 May 2012 11:03:54 +0200 Subject: [PATCH 0255/1752] Bit operations tests improved. Fuzzing tests of BITCOUNT / BITOP are iterated multiple times. The new BITCOUNT fuzzing test uses random strings in a wider interval of lengths including zero-len strings. --- tests/unit/bitops.tcl | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl index c072f54575a..bc4799432c7 100644 --- a/tests/unit/bitops.tcl +++ b/tests/unit/bitops.tcl @@ -44,11 +44,7 @@ start_server {tags {"bitops"}} { } 0 catch {unset num} - foreach vec [list \ - "" "\xaa" "\x00\x00\xff" "foobar" \ - [randstring 2000 3000] [randstring 2000 3000] \ - [randstring 2000 3000] \ - ] { + foreach vec [list "" "\xaa" "\x00\x00\xff" "foobar"] { incr num test "BITCOUNT against test vector #$num" { r set str $vec @@ -56,6 +52,14 @@ start_server {tags {"bitops"}} { } } + test {BITCOUNT fuzzing} { + for {set j 0} {$j < 100} {incr j} { + set str [randstring 0 3000] + r set str $str + assert {[r bitcount str] == [count_bits $str]} + } + } + test {BITCOUNT with start, end} { r set s "foobar" assert_equal [r bitcount s 0 -1] [count_bits "foobar"] @@ -114,17 +118,19 @@ start_server {tags {"bitops"}} { foreach op {and or xor} { test "BITOP $op fuzzing" { - set vec {} - set veckeys {} - set numvec [expr {[randomInt 10]+1}] - for {set j 0} {$j < $numvec} {incr j} { - set str [randstring 0 1000] - lappend vec $str - lappend veckeys vector_$j - r set vector_$j $str + for {set i 0} {$i < 10} {incr i} { + set vec {} + set veckeys {} + set numvec [expr {[randomInt 10]+1}] + for {set j 0} {$j < $numvec} {incr j} { + set str [randstring 0 1000] + lappend vec $str + lappend veckeys vector_$j + r set vector_$j $str + } + r bitop $op target {*}$veckeys + assert_equal [r get target] [simulate_bit_op $op {*}$vec] } - r bitop $op target {*}$veckeys - assert_equal [r get target] [simulate_bit_op $op {*}$vec] } } } From e53a1fdd603370b3ac0cf3644253cfce5fac6740 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 20 May 2012 11:06:29 +0200 Subject: [PATCH 0256/1752] bitop.c renamed bitops.c bitop.c contains the "Bit related string operations" so it seems more logical to call it bitops instead of bitop. This also makes it matching the name of the test (unit/bitops.tcl). --- src/Makefile | 2 +- src/{bitop.c => bitops.c} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/{bitop.c => bitops.c} (100%) diff --git a/src/Makefile b/src/Makefile index e8ff5d9838a..7c21632e901 100644 --- a/src/Makefile +++ b/src/Makefile @@ -96,7 +96,7 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN endif REDIS_SERVER_NAME= redis-server -REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitop.o +REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o REDIS_CLI_NAME= redis-cli REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o REDIS_BENCHMARK_NAME= redis-benchmark diff --git a/src/bitop.c b/src/bitops.c similarity index 100% rename from src/bitop.c rename to src/bitops.c From c19bfe9fcfdf6dac998f31463df31504c3bacb2b Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 20 May 2012 21:34:58 +0200 Subject: [PATCH 0257/1752] BITCOUNT performance improved. At Redis's default optimization level the command is now much faster, always using a constant-time bit manipualtion technique to count bits instead of GCC builtin popcount, and unrolling the loop. The current implementation performance is 1.5GB/s in a MBA 11" (1.8 Ghz i7) compiled with both GCC and clang. The algorithm used is described here: http://graphics.stanford.edu/~seander/bithacks.html --- src/bitops.c | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 053db02b5cf..0b9c7b02fb6 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -34,19 +34,28 @@ long popcount(void *s, long count) { uint32_t *p4 = s; static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; - /* Count bits four bytes at a time */ - while(count>=4) { - uint32_t aux = *p4++; - count -= 4; -#ifdef __GNUC__ - /* Unsigned int is always >= 4 bytes if compiler is GCC */ - bits += __builtin_popcount(aux); -#else - bits += bitsinbyte[aux&0xff] + - bitsinbyte[(aux>>8)&0xff] + - bitsinbyte[(aux>>16)&0xff] + - bitsinbyte[(aux>>24)&0xff]; -#endif + /* Count bits 16 bytes at a time */ + while(count>=16) { + uint32_t aux1, aux2, aux3, aux4; + + aux1 = *p4++; + aux2 = *p4++; + aux3 = *p4++; + aux4 = *p4++; + count -= 16; + + aux1 = aux1 - ((aux1 >> 1) & 0x55555555); + aux1 = (aux1 & 0x33333333) + ((aux1 >> 2) & 0x33333333); + aux2 = aux2 - ((aux2 >> 1) & 0x55555555); + aux2 = (aux2 & 0x33333333) + ((aux2 >> 2) & 0x33333333); + aux3 = aux3 - ((aux3 >> 1) & 0x55555555); + aux3 = (aux3 & 0x33333333) + ((aux3 >> 2) & 0x33333333); + aux4 = aux4 - ((aux4 >> 1) & 0x55555555); + aux4 = (aux4 & 0x33333333) + ((aux4 >> 2) & 0x33333333); + bits += ((((aux1 + (aux1 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24) + + ((((aux2 + (aux2 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24) + + ((((aux3 + (aux3 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24) + + ((((aux4 + (aux4 >> 4)) & 0x0F0F0F0F) * 0x01010101) >> 24); } /* Count the remaining bytes */ p = (unsigned char*)p4; From f0d962c0eca870e3fd2a4529de321db217c2e9fe Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 22 May 2012 17:40:20 +0200 Subject: [PATCH 0258/1752] BITOP: handle integer encoded objects correctly. A bug in the implementation caused BITOP to crash the server if at least one one of the source objects was integer encoded. The new implementation takes an additional array of Redis objects pointers and calls getDecodedObject() to get a reference to a string encoded object, and then uses decrRefCount() to release the object. Tests modified to cover the regression and improve coverage. --- src/bitops.c | 18 ++++++++++++++++-- tests/unit/bitops.tcl | 19 ++++++++++++++++++- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 0b9c7b02fb6..2f62ccc4604 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -159,6 +159,7 @@ void bitopCommand(redisClient *c) { char *opname = c->argv[1]->ptr; robj *o, *targetkey = c->argv[2]; long op, j, numkeys; + robj **objects; /* Array of soruce objects. */ unsigned char **src; /* Array of source strings pointers. */ long *len, maxlen = 0; /* Array of length of src strings, and max len. */ unsigned char *res = NULL; /* Resulting string. */ @@ -187,22 +188,30 @@ void bitopCommand(redisClient *c) { numkeys = c->argc - 3; src = zmalloc(sizeof(unsigned char*) * numkeys); len = zmalloc(sizeof(long) * numkeys); + objects = zmalloc(sizeof(robj*) * numkeys); for (j = 0; j < numkeys; j++) { o = lookupKeyRead(c->db,c->argv[j+3]); /* Handle non-existing keys as empty strings. */ if (o == NULL) { + objects[j] = NULL; src[j] = NULL; len[j] = 0; continue; } /* Return an error if one of the keys is not a string. */ if (checkType(c,o,REDIS_STRING)) { + for (j = j-1; j >= 0; j--) { + if (objects[j]) + decrRefCount(objects[j]); + } zfree(src); zfree(len); + zfree(objects); return; } - src[j] = o->ptr; - len[j] = sdslen(o->ptr); + objects[j] = getDecodedObject(o); + src[j] = objects[j]->ptr; + len[j] = sdslen(objects[j]->ptr); if (len[j] > maxlen) maxlen = len[j]; } @@ -226,8 +235,13 @@ void bitopCommand(redisClient *c) { res[j] = output; } } + for (j = 0; j < numkeys; j++) { + if (objects[j]) + decrRefCount(objects[j]); + } zfree(src); zfree(len); + zfree(objects); /* Store the computed value into the target key */ if (maxlen) { diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl index bc4799432c7..b7a76abb129 100644 --- a/tests/unit/bitops.tcl +++ b/tests/unit/bitops.tcl @@ -44,7 +44,7 @@ start_server {tags {"bitops"}} { } 0 catch {unset num} - foreach vec [list "" "\xaa" "\x00\x00\xff" "foobar"] { + foreach vec [list "" "\xaa" "\x00\x00\xff" "foobar" "123"] { incr num test "BITCOUNT against test vector #$num" { r set str $vec @@ -119,6 +119,7 @@ start_server {tags {"bitops"}} { foreach op {and or xor} { test "BITOP $op fuzzing" { for {set i 0} {$i < 10} {incr i} { + r flushall set vec {} set veckeys {} set numvec [expr {[randomInt 10]+1}] @@ -133,4 +134,20 @@ start_server {tags {"bitops"}} { } } } + + test {BITOP with integer encoded source objects} { + r set a 1 + r set b 2 + r bitop xor dest a b a + r get dest + } {2} + + test {BITOP with non string source key} { + r del c + r set a 1 + r set b 2 + r lpush c foo + catch {r bitop xor dest a b c d} e + set e + } {*ERR*} } From 8bbc0768b8a2c1e2cf5d157f75cd08ea8576bcc3 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 May 2012 22:12:50 +0200 Subject: [PATCH 0259/1752] BITOP command 10x speed improvement. This commit adds a fast-path to the BITOP that can be used for all the bytes from 0 to the minimal length of the string, and if there are at max 16 input keys. Often the intersected bitmaps are roughly the same size, so this optimization can provide a 10x speed boost to most real world usages of the command. Bytes are processed four full words at a time, in loops specialized for the specific BITOP sub-command, without the need to check for length issues with the inputs (since we run this algorithm only as far as there is data from all the keys at the same time). The remaining part of the string is intersected in the usual way using the slow but generic algorith. It is possible to do better than this with inputs that are not roughly the same size, sorting the input keys by length, by initializing the result string in a smarter way, and noticing that the final part of the output string composed of only data from the longest string does not need any proecessing since AND, OR and XOR against an empty string does not alter the output (zero in the first case, and the original string in the other two cases). More implementations will be implemented later likely, but this should be enough to release Redis 2.6-RC4 with bitops merged in. Note: this commit also adds better testing for BITOP NOT command, that is currently the faster and hard to optimize further since it just flips the bits of a single input string. --- src/bitops.c | 70 ++++++++++++++++++++++++++++++++++++++++++- tests/unit/bitops.tcl | 12 +++++++- 2 files changed, 80 insertions(+), 2 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 2f62ccc4604..d309b7afa15 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -162,6 +162,7 @@ void bitopCommand(redisClient *c) { robj **objects; /* Array of soruce objects. */ unsigned char **src; /* Array of source strings pointers. */ long *len, maxlen = 0; /* Array of length of src strings, and max len. */ + long minlen = 0; /* Min len among the input keys. */ unsigned char *res = NULL; /* Resulting string. */ /* Parse the operation name. */ @@ -213,6 +214,7 @@ void bitopCommand(redisClient *c) { src[j] = objects[j]->ptr; len[j] = sdslen(objects[j]->ptr); if (len[j] > maxlen) maxlen = len[j]; + if (j == 0 || len[j] < minlen) minlen = len[j]; } /* Compute the bit operation, if at least one string is not empty. */ @@ -221,7 +223,73 @@ void bitopCommand(redisClient *c) { unsigned char output, byte; long i; - for (j = 0; j < maxlen; j++) { + /* Fast path: as far as we have data for all the input bitmaps we + * can take a fast path that performs much better than the + * vanilla algorithm. */ + j = 0; + if (minlen && numkeys <= 16) { + unsigned long *lp[16]; + unsigned long *lres = (unsigned long*) res; + + /* Note: sds pointer is always aligned to 8 byte boundary. */ + memcpy(lp,src,sizeof(unsigned long*)*numkeys); + memcpy(res,src[0],minlen); + + /* Different branches per different operations for speed (sorry). */ + if (op == BITOP_AND) { + while(minlen >= sizeof(unsigned long)*4) { + for (i = 1; i < numkeys; i++) { + lres[0] &= lp[i][0]; + lres[1] &= lp[i][1]; + lres[2] &= lp[i][2]; + lres[3] &= lp[i][3]; + lp[i]+=4; + } + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } else if (op == BITOP_OR) { + while(minlen >= sizeof(unsigned long)*4) { + for (i = 1; i < numkeys; i++) { + lres[0] |= lp[i][0]; + lres[1] |= lp[i][1]; + lres[2] |= lp[i][2]; + lres[3] |= lp[i][3]; + lp[i]+=4; + } + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } else if (op == BITOP_XOR) { + while(minlen >= sizeof(unsigned long)*4) { + for (i = 1; i < numkeys; i++) { + lres[0] ^= lp[i][0]; + lres[1] ^= lp[i][1]; + lres[2] ^= lp[i][2]; + lres[3] ^= lp[i][3]; + lp[i]+=4; + } + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } else if (op == BITOP_NOT) { + while(minlen >= sizeof(unsigned long)*4) { + lres[0] = ~lres[0]; + lres[1] = ~lres[1]; + lres[2] = ~lres[2]; + lres[3] = ~lres[3]; + lres+=4; + j += sizeof(unsigned long)*4; + minlen -= sizeof(unsigned long)*4; + } + } + } + + /* j is set to the next byte to process by the previous loop. */ + for (; j < maxlen; j++) { output = (len[0] <= j) ? 0 : src[0][j]; if (op == BITOP_NOT) output = ~output; for (i = 1; i < numkeys; i++) { diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl index b7a76abb129..c8f58ef1e59 100644 --- a/tests/unit/bitops.tcl +++ b/tests/unit/bitops.tcl @@ -24,13 +24,13 @@ proc simulate_bit_op {op args} { set out {} for {set x 0} {$x < $maxlen} {incr x} { set bit [string range $b(0) $x $x] + if {$op eq {not}} {set bit [expr {!$bit}]} for {set j 1} {$j < $count} {incr j} { set bit2 [string range $b($j) $x $x] switch $op { and {set bit [expr {$bit & $bit2}]} or {set bit [expr {$bit | $bit2}]} xor {set bit [expr {$bit ^ $bit2}]} - not {set bit [expr {!$bit}]} } } append out $bit @@ -135,6 +135,16 @@ start_server {tags {"bitops"}} { } } + test {BITOP NOT fuzzing} { + for {set i 0} {$i < 10} {incr i} { + r flushall + set str [randstring 0 1000] + r set str $str + r bitop not target str + assert_equal [r get target] [simulate_bit_op not $str] + } + } + test {BITOP with integer encoded source objects} { r set a 1 r set b 2 From 6b4d92e21de8b7937fb7f71c0c12b72e18554fb4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 May 2012 12:11:30 +0200 Subject: [PATCH 0260/1752] Four new persistence fields in INFO. A few renamed. The 'persistence' section of INFO output now contains additional four fields related to RDB and AOF persistence: rdb_last_bgsave_time_sec Duration of latest BGSAVE in sec. rdb_current_bgsave_time_sec Duration of current BGSAVE in sec. aof_last_rewrite_time_sec Duration of latest AOF rewrite in sec. aof_current_rewrite_time_sec Duration of current AOF rewrite in sec. The 'current' fields are set to -1 if a BGSAVE / AOF rewrite is not in progress. The 'last' fileds are set to -1 if no previous BGSAVE / AOF rewrites were performed. Additionally a few fields in the persistence section were renamed for consistency: changes_since_last_save -> rdb_changes_since_last_save bgsave_in_progress -> rdb_bgsave_in_progress last_save_time -> rdb_last_save_time last_bgsave_status -> rdb_last_bgsave_status bgrewriteaof_in_progress -> aof_rewrite_in_progress bgrewriteaof_scheduled -> aof_rewrite_scheduled After the renaming, fields in the persistence section start with rdb_ or aof_ prefix depending on the persistence method they describe. The field 'loading' and related fields are not prefixed because they are unique for both the persistence methods. --- src/aof.c | 4 ++++ src/rdb.c | 3 +++ src/redis.c | 30 ++++++++++++++++++++++-------- src/redis.h | 4 ++++ 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/aof.c b/src/aof.c index 607bdce8259..09bfb049245 100644 --- a/src/aof.c +++ b/src/aof.c @@ -154,6 +154,7 @@ void stopAppendOnly(void) { aofRewriteBufferReset(); aofRemoveTempFile(server.aof_child_pid); server.aof_child_pid = -1; + server.aof_rewrite_time_start = -1; } } @@ -941,6 +942,7 @@ int rewriteAppendOnlyFileBackground(void) { redisLog(REDIS_NOTICE, "Background append only file rewriting started by pid %d",childpid); server.aof_rewrite_scheduled = 0; + server.aof_rewrite_time_start = time(NULL); server.aof_child_pid = childpid; updateDictResizePolicy(); /* We set appendseldb to -1 in order to force the next call to the @@ -1113,6 +1115,8 @@ void backgroundRewriteDoneHandler(int exitcode, int bysignal) { aofRewriteBufferReset(); aofRemoveTempFile(server.aof_child_pid); server.aof_child_pid = -1; + server.aof_rewrite_time_last = time(NULL)-server.aof_rewrite_time_start; + server.aof_rewrite_time_start = -1; /* Schedule a new rewrite if we are waiting for it to switch the AOF ON. */ if (server.aof_state == REDIS_AOF_WAIT_REWRITE) server.aof_rewrite_scheduled = 1; diff --git a/src/rdb.c b/src/rdb.c index e0491a72499..a7ce031a50e 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -708,6 +708,7 @@ int rdbSaveBackground(char *filename) { return REDIS_ERR; } redisLog(REDIS_NOTICE,"Background saving started by pid %d",childpid); + server.rdb_save_time_start = time(NULL); server.rdb_child_pid = childpid; updateDictResizePolicy(); return REDIS_OK; @@ -1152,6 +1153,8 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) { server.lastbgsave_status = REDIS_ERR; } server.rdb_child_pid = -1; + server.rdb_save_time_last = time(NULL)-server.rdb_save_time_start; + server.rdb_save_time_start = -1; /* Possibly there are slaves waiting for a BGSAVE in order to be served * (the first stage of SYNC is a bulk transfer of dump.rdb) */ updateSlavesWaitingBgsave(exitcode == 0 ? REDIS_OK : REDIS_ERR); diff --git a/src/redis.c b/src/redis.c index 9937cf56759..46915c1302f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1093,6 +1093,8 @@ void initServerConfig() { server.aof_rewrite_base_size = 0; server.aof_rewrite_scheduled = 0; server.aof_last_fsync = time(NULL); + server.aof_rewrite_time_last = -1; + server.aof_rewrite_time_start = -1; server.aof_delayed_fsync = 0; server.aof_fd = -1; server.aof_selected_db = -1; /* Make sure the first time will not match */ @@ -1284,6 +1286,8 @@ void initServer() { aofRewriteBufferReset(); server.aof_buf = sdsempty(); server.lastsave = time(NULL); + server.rdb_save_time_last = -1; + server.rdb_save_time_start = -1; server.dirty = 0; server.stat_numcommands = 0; server.stat_numconnections = 0; @@ -1856,21 +1860,31 @@ sds genRedisInfoString(char *section) { info = sdscatprintf(info, "# Persistence\r\n" "loading:%d\r\n" + "rdb_changes_since_last_save:%lld\r\n" + "rdb_bgsave_in_progress:%d\r\n" + "rdb_last_save_time:%ld\r\n" + "rdb_last_bgsave_status:%s\r\n" + "rdb_last_bgsave_time_sec:%ld\r\n" + "rdb_current_bgsave_time_sec:%ld\r\n" "aof_enabled:%d\r\n" - "changes_since_last_save:%lld\r\n" - "bgsave_in_progress:%d\r\n" - "last_save_time:%ld\r\n" - "last_bgsave_status:%s\r\n" - "bgrewriteaof_in_progress:%d\r\n" - "bgrewriteaof_scheduled:%d\r\n", + "aof_rewrite_in_progress:%d\r\n" + "aof_rewrite_scheduled:%d\r\n" + "aof_last_rewrite_time_sec:%ld\r\n" + "aof_current_rewrite_time_sec:%ld\r\n", server.loading, - server.aof_state != REDIS_AOF_OFF, server.dirty, server.rdb_child_pid != -1, server.lastsave, server.lastbgsave_status == REDIS_OK ? "ok" : "err", + server.rdb_save_time_last, + (server.rdb_child_pid == -1) ? + -1 : time(NULL)-server.rdb_save_time_start, + server.aof_state != REDIS_AOF_OFF, server.aof_child_pid != -1, - server.aof_rewrite_scheduled); + server.aof_rewrite_scheduled, + server.aof_rewrite_time_last, + (server.aof_child_pid == -1) ? + -1 : time(NULL)-server.aof_rewrite_time_start); if (server.aof_state != REDIS_AOF_OFF) { info = sdscatprintf(info, diff --git a/src/redis.h b/src/redis.h index cf971667c6c..d877165a415 100644 --- a/src/redis.h +++ b/src/redis.h @@ -511,6 +511,8 @@ struct redisServer { int aof_selected_db; /* Currently selected DB in AOF */ time_t aof_flush_postponed_start; /* UNIX time of postponed AOF flush */ time_t aof_last_fsync; /* UNIX time of last fsync() */ + time_t aof_rewrite_time_last; /* Time used by last AOF rewrite run. */ + time_t aof_rewrite_time_start; /* Current AOF rewrite start time. */ unsigned long aof_delayed_fsync; /* delayed AOF fsync() counter */ /* RDB persistence */ long long dirty; /* Changes to DB from the last save */ @@ -522,6 +524,8 @@ struct redisServer { int rdb_compression; /* Use compression in RDB? */ int rdb_checksum; /* Use RDB checksum? */ time_t lastsave; /* Unix time of last save succeeede */ + time_t rdb_save_time_last; /* Time used by last RDB save run. */ + time_t rdb_save_time_start; /* Current RDB save start time. */ int lastbgsave_status; /* REDIS_OK or REDIS_ERR */ int stop_writes_on_bgsave_err; /* Don't allow writes if can't BGSAVE */ /* Propagation of commands in AOF / replication */ From 69396249e5877ff4e7fcb6ab121a0a1de055200f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 May 2012 12:24:20 +0200 Subject: [PATCH 0261/1752] Release notes: more info about 2.4 -> 2.6 migration. Migration information moved at the end of the file as now they start to be too long to stay at the top, but a warning was added at the top of the file to remember the user to check. Added information about INFO fields that changed name between 2.4 and 2.6. --- 00-RELEASENOTES | 78 ++++++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index b232fdb200f..a5ff7f2d8ef 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,39 +1,9 @@ Redis 2.6 release notes +======================= -Migrating from 2.4 to 2.6 -========================= - -Redis 2.4 is mostly a strict subset of 2.6. However there are a few things -that you should be aware of: - -* You can't use .rdb and AOF files generated with 2.6 into a 2.4 instance. -* 2.4 slaves can be attached to 2.6 masters, but not the contrary, and only - for the time needed to perform the version upgrade. - -There are also a few API differences, that are unlikely to cause problems, -but it is better to keep them in mind: - -* SORT now will refuse to sort in numerical mode elements that can't be parsed - as numbers. -* EXPIREs now all have millisecond resolution (but this is very unlikely to - break code that was not conceived exploting the previous resolution error - in some way.) -* INFO output is a bit different now, and contains empty lines and comments - starting with '#'. All the major clients should be already fixed to work - with the new INFO format. -* Slaves are only read-only by default (but you can change this easily - setting the "slave-read-only" configuration option to "no" editing your - redis.conf or using CONFIG SET. - -Also the following redis.conf and CONFIG GET / SET parameters changed name: - - * hash-max-zipmap-entries, now replaced by hash-max-ziplist-entries - * hash-max-zipmap-value, now replaced by hash-max-ziplist-value - * glueoutputbuf option was now completely removed (was deprecated) - ---------- -CHANGELOG ---------- +** IMPORTANT ** Check the 'Migrating from 2.4 to 2.6' section at the end of + this file for information about what changed between 2.4 and + 2.6 and how this may affect your application. What's new in Redis 2.5.9 (aka 2.6 Release Candidate 3) ======================================================= @@ -115,6 +85,46 @@ An overview of new features and changes in Redis 2.6.x * Better support for big endian and *BSD systems. * Build system improved. +Migrating from 2.4 to 2.6 +========================= + +Redis 2.4 is mostly a strict subset of 2.6. However there are a few things +that you should be aware of: + +* You can't use .rdb and AOF files generated with 2.6 into a 2.4 instance. +* 2.4 slaves can be attached to 2.6 masters, but not the contrary, and only + for the time needed to perform the version upgrade. + +There are also a few API differences, that are unlikely to cause problems, +but it is better to keep them in mind: + +* SORT now will refuse to sort in numerical mode elements that can't be parsed + as numbers. +* EXPIREs now all have millisecond resolution (but this is very unlikely to + break code that was not conceived exploting the previous resolution error + in some way.) +* INFO output is a bit different now, and contains empty lines and comments + starting with '#'. All the major clients should be already fixed to work + with the new INFO format. +* Slaves are only read-only by default (but you can change this easily + setting the "slave-read-only" configuration option to "no" editing your + redis.conf or using CONFIG SET. + +The following INFO fields were renamed for consistency: + + changes_since_last_save -> rdb_changes_since_last_save + bgsave_in_progress -> rdb_bgsave_in_progress + last_save_time -> rdb_last_save_time + last_bgsave_status -> rdb_last_bgsave_status + bgrewriteaof_in_progress -> aof_rewrite_in_progress + bgrewriteaof_scheduled -> aof_rewrite_scheduled + +The following redis.conf and CONFIG GET / SET parameters changed: + + * hash-max-zipmap-entries, now replaced by hash-max-ziplist-entries + * hash-max-zipmap-value, now replaced by hash-max-ziplist-value + * glueoutputbuf option was now completely removed (was deprecated) + -------------------------------------------------------------------------------- Credits: Where not specified the implementation and design are done by From 473f3090f2574e8a2f5db5f18ab7387040829108 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 May 2012 15:20:59 +0200 Subject: [PATCH 0262/1752] Tests modified to account for INFO fields renaming. Commit 33e1db36fa3948c8b9baa3991fd40e7f6b31fb9e modified the name of a few INFO fields. This commit changes the Redis test to account for this changes. --- tests/support/util.tcl | 4 ++-- tests/unit/aofrw.tcl | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 0131c4118ad..a2a9f85133e 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -55,7 +55,7 @@ proc status {r property} { proc waitForBgsave r { while 1 { - if {[status r bgsave_in_progress] eq 1} { + if {[status r rdb_bgsave_in_progress] eq 1} { if {$::verbose} { puts -nonewline "\nWaiting for background save to finish... " flush stdout @@ -69,7 +69,7 @@ proc waitForBgsave r { proc waitForBgrewriteaof r { while 1 { - if {[status r bgrewriteaof_in_progress] eq 1} { + if {[status r aof_rewrite_in_progress] eq 1} { if {$::verbose} { puts -nonewline "\nWaiting for background AOF rewrite to finish... " flush stdout diff --git a/tests/unit/aofrw.tcl b/tests/unit/aofrw.tcl index 8b09d1995a2..e651694feac 100644 --- a/tests/unit/aofrw.tcl +++ b/tests/unit/aofrw.tcl @@ -127,8 +127,8 @@ start_server {tags {"aofrw"}} { r info persistence set res [r exec] assert_match {*scheduled*} [lindex $res 1] - assert_match {*bgrewriteaof_scheduled:1*} [lindex $res 2] - while {[string match {*bgrewriteaof_scheduled:1*} [r info persistence]]} { + assert_match {*aof_rewrite_scheduled:1*} [lindex $res 2] + while {[string match {*aof_rewrite_scheduled:1*} [r info persistence]]} { after 100 } } @@ -141,7 +141,7 @@ start_server {tags {"aofrw"}} { r exec } e assert_match {*ERR*already*} $e - while {[string match {*bgrewriteaof_scheduled:1*} [r info persistence]]} { + while {[string match {*aof_rewrite_scheduled:1*} [r info persistence]]} { after 100 } } From 9a8d51add539e0b69fa4bcd6454e6a9dd8301763 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 27 May 2012 17:17:22 +0200 Subject: [PATCH 0263/1752] 2.6 Release notes: added more new features to the list. --- 00-RELEASENOTES | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index a5ff7f2d8ef..e4b86c3067a 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -52,8 +52,12 @@ An overview of new features and changes in Redis 2.6.x * Better memory usage for "small" lists, ziplists and hashes when fields or values contain small integers. * Read only slaves. +* New bit opeations: BITCOUNT and BITOP commands. * Clients max output buffer soft and hard limits. You can specifiy different limits for different classes of clients (normal,pubsub,slave). +* More incremental (less blocking) expired keys collection algorithm, in + practical terms this means that Redis is more responsive when a very + big number of keys expire about at the same time. * AOF is now able to rewrite aggregate data types using variadic commands, often producing an AOF that is faster to save, load, and is smaller in size. * Every redis.conf directive is now accepted as a command line option for the From 8f0658cdd063b762ece37c8159a3cb1551526a7f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 May 2012 21:45:39 +0200 Subject: [PATCH 0264/1752] BITOP bug when called against non existing keys fixed. In the issue #529 an user reported a bug that can be triggered with the following code: flushdb set a "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" bitop or x a b The bug was introduced with the speed optimization in commit 8bbc076 that specializes every BITOP operation loop up to the minimum length of the input strings. However the computation of the minimum length contained an error when a non existing key was present in the input, after a key that was non zero length. This commit fixes the bug and adds a regression test for it. --- src/bitops.c | 1 + tests/unit/bitops.tcl | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/bitops.c b/src/bitops.c index d309b7afa15..00192b92b14 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -197,6 +197,7 @@ void bitopCommand(redisClient *c) { objects[j] = NULL; src[j] = NULL; len[j] = 0; + minlen = 0; continue; } /* Return an error if one of the keys is not a string. */ diff --git a/tests/unit/bitops.tcl b/tests/unit/bitops.tcl index c8f58ef1e59..0e3403bfe18 100644 --- a/tests/unit/bitops.tcl +++ b/tests/unit/bitops.tcl @@ -160,4 +160,10 @@ start_server {tags {"bitops"}} { catch {r bitop xor dest a b c d} e set e } {*ERR*} + + test {BITOP with empty string after non empty string (issue #529)} { + r flushdb + r set a "\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" + r bitop or x a b + } {32} } From ebdcd723a78d844273334685bb1f9b43290ec9c2 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 2 Jun 2012 10:21:57 +0200 Subject: [PATCH 0265/1752] RDB type loading functions clarified in comments. Improved comments to make clear that rdbLoadType() just loads a general TYPE in the context of RDB that can be an object type or an expire type, end-of-file, and so forth. While rdbLoadObjectType() enforces that the type is a valid Object Type otherwise it returns -1. --- src/rdb.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index a7ce031a50e..90c2ea0843f 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -21,6 +21,9 @@ int rdbSaveType(rio *rdb, unsigned char type) { return rdbWriteRaw(rdb,&type,1); } +/* Load a "type" in RDB format, that is a one byte unsigned integer. + * This function is not only used to load object types, but also special + * "types" like the end-of-file type, the EXPIRE type, and so forth. */ int rdbLoadType(rio *rdb) { unsigned char type; if (rioRead(rdb,&type,1) == 0) return -1; @@ -433,7 +436,8 @@ int rdbSaveObjectType(rio *rdb, robj *o) { return -1; /* avoid warning */ } -/* Load object type. Return -1 when the byte doesn't contain an object type. */ +/* Use rdbLoadType() to load a TYPE in RDB format, but returns -1 if the + * type is not specifically a valid Object Type. */ int rdbLoadObjectType(rio *rdb) { int type; if ((type = rdbLoadType(rdb)) == -1) return -1; From 591c9e6543d4474bd74921fbd15f110fa603b7b6 Mon Sep 17 00:00:00 2001 From: Alex Mitrofanov Date: Fri, 1 Jun 2012 18:48:45 -0700 Subject: [PATCH 0266/1752] Fixed RESTORE hash failure (Issue #532) (additional commit notes by antirez@gmail.com): The rdbIsObjectType() macro was not updated when the new RDB object type of ziplist encoded hashes was added. As a result RESTORE, that uses rdbLoadObjectType(), failed when a ziplist encoded hash was loaded. This does not affected normal RDB loading because in that case we use the lower-level function rdbLoadType(). The commit also adds a regression test. --- src/rdb.h | 2 +- tests/unit/dump.tcl | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/rdb.h b/src/rdb.h index 0381c5b4c92..37f4474564b 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -54,7 +54,7 @@ #define REDIS_RDB_TYPE_HASH_ZIPLIST 13 /* Test if a type is an object type. */ -#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 12)) +#define rdbIsObjectType(t) ((t >= 0 && t <= 4) || (t >= 9 && t <= 13)) /* Special RDB opcodes (saved/loaded with rdbSaveType/rdbLoadType). */ #define REDIS_RDB_OPCODE_EXPIRETIME_MS 252 diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index b73cde0c17b..be891a96f95 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -91,6 +91,26 @@ start_server {tags {"dump"}} { } } + test {MIGRATE can correctly transfer hashes} { + set first [srv 0 client] + r del key + r hmset key field1 "item 1" field2 "item 2" field3 "item 3" \ + field4 "item 4" field5 "item 5" field6 "item 6" + start_server {tags {"repl"}} { + set second [srv 0 client] + set second_host [srv 0 host] + set second_port [srv 0 port] + + assert {[$first exists key] == 1} + assert {[$second exists key] == 0} + set ret [r -1 migrate $second_host $second_port key 9 10000] + assert {$ret eq {OK}} + assert {[$first exists key] == 0} + assert {[$second exists key] == 1} + assert {[$second ttl key] == -1} + } + } + test {MIGRATE timeout actually works} { set first [srv 0 client] r set key "Some Value" From 3f12656781b883316c5e1a1375869fec1dea05a2 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 2 Jun 2012 23:29:57 +0200 Subject: [PATCH 0267/1752] EVAL replication test: less false positives. wait_for_condition is now used instead of the usual "after 1000" (that is the way to sleep in Tcl). This should avoid to find the replica in a state where it is loading the RDB in memory, returning -LOADING error. This test used to fail when running the test over valgrind, due to the added latencies. --- tests/support/util.tcl | 2 +- tests/unit/scripting.tcl | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index a2a9f85133e..96af279d841 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -83,7 +83,7 @@ proc waitForBgrewriteaof r { proc wait_for_sync r { while 1 { - if {[status r master_link_status] eq "down"} { + if {[status $r master_link_status] eq "down"} { after 10 } else { break diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index daf0c0f26e7..ec6e5c2a6be 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -319,9 +319,13 @@ start_server {tags {"scripting repl"}} { test {Connect a slave to the main instance} { r -1 slaveof [srv 0 host] [srv 0 port] - after 1000 - s -1 role - } {slave} + wait_for_condition 50 100 { + [s -1 role] eq {slave} && + [string match {*master_link_status:up*} [r -1 info replication]] + } else { + fail "Can't turn the instance into a slave" + } + } test {Now use EVALSHA against the master} { r evalsha ae3477e27be955de7e1bc9adfdca626b478d3cb2 0 From 1a3e9d951be14fecf29cb7ad3d5f068e151b66f1 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Jun 2012 11:41:38 +0200 Subject: [PATCH 0268/1752] Version bumped to 2.5.10 (2.6 RC4) + Release Notes. --- 00-RELEASENOTES | 27 +++++++++++++++++++++++++++ src/version.h | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index e4b86c3067a..031a433ea79 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -5,6 +5,33 @@ Redis 2.6 release notes this file for information about what changed between 2.4 and 2.6 and how this may affect your application. +-------------------------------------------------------------------------------- +Upgrade urgency levels: + +LOW: No need to upgrade unless there are new features you want to use. +MODERATE: Program an upgrade of the server, but it's not urgent. +HIGH: There is a critical bug that may affect some part of users. Upgrade! +CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. +-------------------------------------------------------------------------------- + +---[ Redis 2.5.10 (2.6 Release Candidate 4) ] + +UPGRADE URGENCY: HIGH. + +* [BUGFIX] Allow PREFIX to be overwritten on "make install". +* [BUGFIX] Run the test with just one client if the computer is slow. +* [BUGFIX] Event port support in our event driven libray. +* [BUGFIX] Jemalloc updated to 3.0.0. This fixes a possibly AOF rewrite issue. + See https://github.com/antirez/redis/issues/504 for info. +* [BUGFIX] Fixed issue #516: ZINTERSTORE / ZUNIONSTORE with mixed sets/zsets. +* [BUGFIX] Set fd to writable when poll(2) detects POLLERR or POLLHUP event. +* [BUGFIX] Fixed RESTORE hash failure (Issue #532). +* [IMPROVED] Allow an AOF rewrite buffer > 2GB (Related to issue #504). +* [IMPROVED] Server cron function frequency is now configurable (REDIS_HZ). +* [IMPROVED] Better, less blocking expired keys collection algorithm. +* [FEATURE] New commands: BITOP and BITCOUNT. +* [FEATURE] redis-cli --pipe for mass import. + What's new in Redis 2.5.9 (aka 2.6 Release Candidate 3) ======================================================= diff --git a/src/version.h b/src/version.h index c4e2caf3546..e2827f572f8 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.9" +#define REDIS_VERSION "2.5.10" From 0ea1a9c452141671ee3f36dab08962d811596903 Mon Sep 17 00:00:00 2001 From: dvir volk Date: Fri, 8 Jun 2012 16:03:18 +0300 Subject: [PATCH 0269/1752] fixed server install script to rewrite the default configuration file and not a template, and removed the old config template --- utils/install_server.sh | 25 +- utils/redis.conf.tpl | 495 ---------------------------------------- 2 files changed, 18 insertions(+), 502 deletions(-) delete mode 100644 utils/redis.conf.tpl diff --git a/utils/install_server.sh b/utils/install_server.sh index 70f0adfe36d..789a97803c2 100755 --- a/utils/install_server.sh +++ b/utils/install_server.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # Copyright 2011 Dvir Volk . All rights reserved. # @@ -48,6 +48,7 @@ if [ `whoami` != "root" ] ; then exit 1 fi + #Read the redis port read -p "Please select the redis port for this instance: [$_REDIS_PORT] " REDIS_PORT if [ ! `echo $REDIS_PORT | egrep "^[0-9]+\$"` ] ; then @@ -99,7 +100,7 @@ fi #render the tmplates TMP_FILE="/tmp/$REDIS_PORT.conf" -TPL_FILE="./redis.conf.tpl" +DEFAULT_CONFIG="../redis.conf" INIT_TPL_FILE="./redis_init_script.tpl" INIT_SCRIPT_DEST="/etc/init.d/redis_$REDIS_PORT" PIDFILE="/var/run/redis_$REDIS_PORT.pid" @@ -112,9 +113,19 @@ if [ ! "$CLI_EXEC" ] ; then CLI_EXEC=`dirname $REDIS_EXECUTABLE`"/redis-cli" fi -#Generate config file from template +#Generate config file from the default config file as template +#changing only the stuff we're controlling from this script echo "## Generated by install_server.sh ##" > $TMP_FILE -cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done + +SED_EXPR="s#^port [0-9]{4}\$#port ${REDIS_PORT}#;\ +s#^logfile .+\$#logfile ${REDIS_LOG_FILE}#;\ +s#^dir .+\$#dir ${REDIS_DATA_DIR}#;\ +s#^pidfile .+\$#pidfile ${PIDFILE}#;\ +s#^daemonize no\$#daemonize yes#;" +echo $SED_EXPR +sed -r "$SED_EXPR" $DEFAULT_CONFIG >> $TMP_FILE + +#cat $TPL_FILE | while read line; do eval "echo \"$line\"" >> $TMP_FILE; done cp -f $TMP_FILE $REDIS_CONFIG_FILE || exit 1 #Generate sample script from template file @@ -146,9 +157,9 @@ REDIS_CHKCONFIG_INFO=\ # Description: Redis daemon\n ### END INIT INFO\n\n" -if [[ ! `which chkconfig` ]] ; then +if [ !`which chkconfig` ] ; then #combine the header and the template (which is actually a static footer) - echo -e $REDIS_INIT_HEADER > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" + echo $REDIS_INIT_HEADER > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" else #if we're a box with chkconfig on it we want to include info for chkconfig echo -e $REDIS_INIT_HEADER $REDIS_CHKCONFIG_INFO > $TMP_FILE && cat $INIT_TPL_FILE >> $TMP_FILE || die "Could not write init script to $TMP_FILE" @@ -160,7 +171,7 @@ echo "Copied $TMP_FILE => $INIT_SCRIPT_DEST" #Install the service echo "Installing service..." -if [[ ! `which chkconfig` ]] ; then +if [ !`which chkconfig` ] ; then #if we're not a chkconfig box assume we're able to use update-rc.d update-rc.d redis_$REDIS_PORT defaults && echo "Success!" else diff --git a/utils/redis.conf.tpl b/utils/redis.conf.tpl deleted file mode 100644 index 33d99a50935..00000000000 --- a/utils/redis.conf.tpl +++ /dev/null @@ -1,495 +0,0 @@ -# Redis configuration file example - -# Note on units: when memory size is needed, it is possible to specify -# it in the usual form of 1k 5GB 4M and so forth: -# -# 1k => 1000 bytes -# 1kb => 1024 bytes -# 1m => 1000000 bytes -# 1mb => 1024*1024 bytes -# 1g => 1000000000 bytes -# 1gb => 1024*1024*1024 bytes -# -# units are case insensitive so 1GB 1Gb 1gB are all the same. - -# By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. -daemonize yes - -# When running daemonized, Redis writes a pid file in /var/run/redis.pid by -# default. You can specify a custom pid file location here. -pidfile $PIDFILE - -# Accept connections on the specified port, default is 6379. -# If port 0 is specified Redis will not listen on a TCP socket. -port $REDIS_PORT - -# If you want you can bind a single interface, if the bind option is not -# specified all the interfaces will listen for incoming connections. -# -# bind 127.0.0.1 - -# Specify the path for the unix socket that will be used to listen for -# incoming connections. There is no default, so Redis will not listen -# on a unix socket when not specified. -# -# unixsocket /tmp/redis.sock -# unixsocketperm 755 - -# Close the connection after a client is idle for N seconds (0 to disable) -timeout 0 - -# Set server verbosity to 'debug' -# it can be one of: -# debug (a lot of information, useful for development/testing) -# verbose (many rarely useful info, but not a mess like the debug level) -# notice (moderately verbose, what you want in production probably) -# warning (only very important / critical messages are logged) -loglevel notice - -# Specify the log file name. Also 'stdout' can be used to force -# Redis to log on the standard output. Note that if you use standard -# output for logging but daemonize, logs will be sent to /dev/null -logfile $REDIS_LOG_FILE - -# To enable logging to the system logger, just set 'syslog-enabled' to yes, -# and optionally update the other syslog parameters to suit your needs. -# syslog-enabled no - -# Specify the syslog identity. -# syslog-ident redis - -# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. -# syslog-facility local0 - -# Set the number of databases. The default database is DB 0, you can select -# a different one on a per-connection basis using SELECT where -# dbid is a number between 0 and 'databases'-1 -databases 16 - -################################ SNAPSHOTTING ################################# -# -# Save the DB on disk: -# -# save -# -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. -# -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed -# -# Note: you can disable saving at all commenting all the "save" lines. -# -# It is also possible to remove all the previously configured save -# points by adding a save directive with a single empty string argument -# like in the following example: -# -# save "" - -save 900 1 -save 300 10 -save 60 10000 - -# By default Redis will stop accepting writes if RDB snapshots are enabled -# (at least one save point) and the latest background save failed. -# This will make the user aware (in an hard way) that data is not persisting -# on disk properly, otherwise chances are that no one will notice and some -# distater will happen. -# -# If the background saving process will start working again Redis will -# automatically allow writes again. -# -# However if you have setup your proper monitoring of the Redis server -# and persistence, you may want to disable this feature so that Redis will -# continue to work as usually even if there are problems with disk, -# permissions, and so forth. -stop-writes-on-bgsave-error yes - -# Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. -# If you want to save some CPU in the saving child set it to 'no' but -# the dataset will likely be bigger if you have compressible values or keys. -rdbcompression yes - -# The filename where to dump the DB -dbfilename dump.rdb - -# The working directory. -# -# The DB will be written inside this directory, with the filename specified -# above using the 'dbfilename' configuration directive. -# -# Also the Append Only File will be created inside this directory. -# -# Note that you must specify a directory here, not a file name. -dir $REDIS_DATA_DIR - -################################# REPLICATION ################################# - -# Master-Slave replication. Use slaveof to make a Redis instance a copy of -# another Redis server. Note that the configuration is local to the slave -# so for example it is possible to configure the slave to save the DB with a -# different interval, or to listen to another port, and so on. -# -# slaveof - -# If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before -# starting the replication synchronization process, otherwise the master will -# refuse the slave request. -# -# masterauth - -# When a slave lost the connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: -# -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will -# still reply to client requests, possibly with out of date data, or the -# data set may just be empty if this is the first synchronization. -# -# 2) if slave-serve-stale data is set to 'no' the slave will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO and SLAVEOF. -# -slave-serve-stale-data yes - -# Slaves send PINGs to server in a predefined interval. It's possible to change -# this interval with the repl_ping_slave_period option. The default value is 10 -# seconds. -# -# repl-ping-slave-period 10 - -# The following option sets a timeout for both Bulk transfer I/O timeout and -# master data or ping response timeout. The default value is 60 seconds. -# -# It is important to make sure that this value is greater than the value -# specified for repl-ping-slave-period otherwise a timeout will be detected -# every time there is low traffic between the master and the slave. -# -# repl-timeout 60 - -################################## SECURITY ################################### - -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running redis-server. -# -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). -# -# Warning: since Redis is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. -# -# requirepass foobared - -# Command renaming. -# -# It is possible to change the name of dangerous commands in a shared -# environment. For instance the CONFIG command may be renamed into something -# of hard to guess so that it will be still available for internal-use -# tools but not available for general clients. -# -# Example: -# -# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 -# -# It is also possible to completely kill a command renaming it into -# an empty string: -# -# rename-command CONFIG "" - -################################### LIMITS #################################### - -# Set the max number of connected clients at the same time. By default -# this limit is set to 10000 clients, however if the Redis server is not -# able ot configure the process file limit to allow for the specified limit -# the max number of allowed clients is set to the current file limit -# minus 32 (as Redis reserves a few file descriptors for internal uses). -# -# Once the limit is reached Redis will close all the new connections sending -# an error 'max number of clients reached'. -# -# maxclients 10000 - -# Don't use more memory than the specified amount of bytes. -# When the memory limit is reached Redis will try to remove keys -# accordingly to the eviction policy selected (see maxmemmory-policy). -# -# If Redis can't remove keys according to the policy, or if the policy is -# set to 'noeviction', Redis will start to reply with errors to commands -# that would use more memory, like SET, LPUSH, and so on, and will continue -# to reply to read-only commands like GET. -# -# This option is usually useful when using Redis as an LRU cache, or to set -# an hard memory limit for an instance (using the 'noeviction' policy). -# -# WARNING: If you have slaves attached to an instance with maxmemory on, -# the size of the output buffers needed to feed the slaves are subtracted -# from the used memory count, so that network problems / resyncs will -# not trigger a loop where keys are evicted, and in turn the output -# buffer of slaves is full with DELs of keys evicted triggering the deletion -# of more keys, and so forth until the database is completely emptied. -# -# In short... if you have slaves attached it is suggested that you set a lower -# limit for maxmemory so that there is some free RAM on the system for slave -# output buffers (but this is not needed if the policy is 'noeviction'). -# -# maxmemory - -# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached? You can select among five behavior: -# -# volatile-lru -> remove the key with an expire set using an LRU algorithm -# allkeys-lru -> remove any key accordingly to the LRU algorithm -# volatile-random -> remove a random key with an expire set -# allkeys-random -> remove a random key, any key -# volatile-ttl -> remove the key with the nearest expire time (minor TTL) -# noeviction -> don't expire at all, just return an error on write operations -# -# Note: with all the kind of policies, Redis will return an error on write -# operations, when there are not suitable keys for eviction. -# -# At the date of writing this commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort -# -# The default is: -# -# maxmemory-policy volatile-lru - -# LRU and minimal TTL algorithms are not precise algorithms but approximated -# algorithms (in order to save memory), so you can select as well the sample -# size to check. For instance for default Redis will check three keys and -# pick the one that was used less recently, you can change the sample size -# using the following configuration directive. -# -# maxmemory-samples 3 - -############################## APPEND ONLY MODE ############################### - -# By default Redis asynchronously dumps the dataset on disk. If you can live -# with the idea that the latest records will be lost if something like a crash -# happens this is the preferred way to run Redis. If instead you care a lot -# about your data and don't want to that a single record can get lost you should -# enable the append only mode: when this mode is enabled Redis will append -# every write operation received in the file appendonly.aof. This file will -# be read on startup in order to rebuild the full dataset in memory. -# -# Note that you can have both the async dumps and the append only file if you -# like (you have to comment the "save" statements above to disable the dumps). -# Still if append only mode is enabled Redis will load the data from the -# log file at startup ignoring the dump.rdb file. -# -# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append -# log file in background when it gets too big. - -appendonly no - -# The name of the append only file (default: "appendonly.aof") -# appendfilename appendonly.aof - -# The fsync() call tells the Operating System to actually write data on disk -# instead to wait for more data in the output buffer. Some OS will really flush -# data on disk, some other OS will just try to do it ASAP. -# -# Redis supports three different modes: -# -# no: don't fsync, just let the OS flush the data when it wants. Faster. -# always: fsync after every write to the append only log . Slow, Safest. -# everysec: fsync only if one second passed since the last fsync. Compromise. -# -# The default is "everysec" that's usually the right compromise between -# speed and data safety. It's up to you to understand if you can relax this to -# "no" that will let the operating system flush the output buffer when -# it wants, for better performances (but if you can live with the idea of -# some data loss consider the default persistence mode that's snapshotting), -# or on the contrary, use "always" that's very slow but a bit safer than -# everysec. -# -# If unsure, use "everysec". - -# appendfsync always -appendfsync everysec -# appendfsync no - -# When the AOF fsync policy is set to always or everysec, and a background -# saving process (a background save or AOF log background rewriting) is -# performing a lot of I/O against the disk, in some Linux configurations -# Redis may block too long on the fsync() call. Note that there is no fix for -# this currently, as even performing fsync in a different thread will block -# our synchronous write(2) call. -# -# In order to mitigate this problem it's possible to use the following option -# that will prevent fsync() from being called in the main process while a -# BGSAVE or BGREWRITEAOF is in progress. -# -# This means that while another child is saving the durability of Redis is -# the same as "appendfsync none", that in practical terms means that it is -# possible to lost up to 30 seconds of log in the worst scenario (with the -# default Linux settings). -# -# If you have latency problems turn this to "yes". Otherwise leave it as -# "no" that is the safest pick from the point of view of durability. -no-appendfsync-on-rewrite no - -# Automatic rewrite of the append only file. -# Redis is able to automatically rewrite the log file implicitly calling -# BGREWRITEAOF when the AOF log size will growth by the specified percentage. -# -# This is how it works: Redis remembers the size of the AOF file after the -# latest rewrite (or if no rewrite happened since the restart, the size of -# the AOF at startup is used). -# -# This base size is compared to the current size. If the current size is -# bigger than the specified percentage, the rewrite is triggered. Also -# you need to specify a minimal size for the AOF file to be rewritten, this -# is useful to avoid rewriting the AOF file even if the percentage increase -# is reached but it is still pretty small. -# -# Specify a percentage of zero in order to disable the automatic AOF -# rewrite feature. - -auto-aof-rewrite-percentage 100 -auto-aof-rewrite-min-size 64mb - -################################ LUA SCRIPTING ############################### - -# Max execution time of a Lua script in milliseconds. -# -# If the maximum execution time is reached Redis will log that a script is -# still in execution after the maximum allowed time and will start to -# reply to queries with an error. -# -# When a long running script exceed the maximum execution time only the -# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be -# used to stop a script that did not yet called write commands. The second -# is the only way to shut down the server in the case a write commands was -# already issue by the script but the user don't want to wait for the natural -# termination of the script. -# -# Set it to 0 or a negative value for unlimited execution without warnings. -lua-time-limit 5000 - -################################## SLOW LOG ################################### - -# The Redis Slow Log is a system to log queries that exceeded a specified -# execution time. The execution time does not include the I/O operations -# like talking with the client, sending the reply and so forth, -# but just the time needed to actually execute the command (this is the only -# stage of command execution where the thread is blocked and can not serve -# other requests in the meantime). -# -# You can configure the slow log with two parameters: one tells Redis -# what is the execution time, in microseconds, to exceed in order for the -# command to get logged, and the other parameter is the length of the -# slow log. When a new command is logged the oldest one is removed from the -# queue of logged commands. - -# The following time is expressed in microseconds, so 1000000 is equivalent -# to one second. Note that a negative number disables the slow log, while -# a value of zero forces the logging of every command. -slowlog-log-slower-than 10000 - -# There is no limit to this length. Just be aware that it will consume memory. -# You can reclaim memory used by the slow log with SLOWLOG RESET. -slowlog-max-len 1024 - -############################### ADVANCED CONFIG ############################### - -# Hashes are encoded using a memory efficient data structure when they have a -# small number of entries, and the biggest entry does not exceed a given -# threshold. These thresholds can be configured using the following directives. -hash-max-ziplist-entries 512 -hash-max-ziplist-value 64 - -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 - -# Sets have a special encoding in just one case: when a set is composed -# of just strings that happens to be integers in radix 10 in the range -# of 64 bit signed integers. -# The following configuration setting sets the limit in the size of the -# set in order to use this special memory saving encoding. -set-max-intset-entries 512 - -# Similarly to hashes and lists, sorted sets are also specially encoded in -# order to save a lot of space. This encoding is only used when the length and -# elements of a sorted set are below the following limits: -zset-max-ziplist-entries 128 -zset-max-ziplist-value 64 - -# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in -# order to help rehashing the main Redis hash table (the one mapping top-level -# keys to values). The hash table implementation Redis uses (see dict.c) -# performs a lazy rehashing: the more operation you run into an hash table -# that is rehashing, the more rehashing "steps" are performed, so if the -# server is idle the rehashing is never complete and some more memory is used -# by the hash table. -# -# The default is to use this millisecond 10 times every second in order to -# active rehashing the main dictionaries, freeing memory when possible. -# -# If unsure: -# use "activerehashing no" if you have hard latency requirements and it is -# not a good thing in your environment that Redis can reply form time to time -# to queries with 2 milliseconds delay. -# -# use "activerehashing yes" if you don't have such hard requirements but -# want to free memory asap when possible. -activerehashing yes - -# The client output buffer limits can be used to force disconnection of clients -# that are not reading data from the server fast enough for some reason (a -# common reason is that a Pub/Sub client can't consume messages as fast as the -# publisher can produce them). -# -# The limit can be set differently for the three different classes of clients: -# -# normal -> normal clients -# slave -> slave clients and MONITOR clients -# pubsub -> clients subcribed to at least one pubsub channel or pattern -# -# The syntax of every client-output-buffer-limit directive is the following: -# -# client-output-buffer-limit -# -# A client is immediately disconnected once the hard limit is reached, or if -# the soft limit is reached and remains reached for the specified number of -# seconds (continuously). -# So for instance if the hard limit is 32 megabytes and the soft limit is -# 16 megabytes / 10 seconds, the client will get disconnected immediately -# if the size of the output buffers reach 32 megabytes, but will also get -# disconnected if the client reaches 16 megabytes and continuously overcomes -# the limit for 10 seconds. -# -# By default normal clients are not limited because they don't receive data -# without asking (in a push way), but just after a request, so only -# asynchronous clients may create a scenario where data is requested faster -# than it can read. -# -# Instead there is a default limit for pubsub and slave clients, since -# subscribers and slaves receive data in a push fashion. -# -# Both the hard or the soft limit can be disabled just setting it to zero. -client-output-buffer-limit normal 0 0 0 -client-output-buffer-limit slave 256mb 64mb 60 -client-output-buffer-limit pubsub 32mb 8mb 60 - -################################## INCLUDES ################################### - -# Include one or more other config files here. This is useful if you -# have a standard template that goes to all Redis server but also need -# to customize a few per-server settings. Include files can include -# other files, so use this wisely. -# -# include /path/to/local.conf -# include /path/to/other.conf From a85c89854f31b26cede276c730d67de954e8182f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Jun 2012 15:19:46 +0200 Subject: [PATCH 0270/1752] New test: hash ziplist -> hashtable encoding conversion. A new stress test was added to stress test the code converting a ziplist into an hash table. In this commit also randomValue helper function was modified to also return negative values. --- tests/support/util.tcl | 14 +++++++++++--- tests/unit/type/hash.tcl | 11 +++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 96af279d841..48d06b74157 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -95,6 +95,14 @@ proc randomInt {max} { expr {int(rand()*$max)} } +proc randomSignedInt {max} { + set i [randomInt $max] + if {rand() > 0.5} { + set i -$i + } + return $i +} + proc randpath args { set path [expr {int(rand()*[llength $args])}] uplevel 1 [lindex $args $path] @@ -103,13 +111,13 @@ proc randpath args { proc randomValue {} { randpath { # Small enough to likely collide - randomInt 1000 + randomSignedInt 1000 } { # 32 bit compressible signed/unsigned - randpath {randomInt 2000000000} {randomInt 4000000000} + randpath {randomSignedInt 2000000000} {randomSignedInt 4000000000} } { # 64 bit - randpath {randomInt 1000000000000} + randpath {randomSignedInt 1000000000000} } { # Random string randpath {randstring 0 256 alpha} \ diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index 950805d1bdd..dbc1c4cc31b 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -419,4 +419,15 @@ start_server {tags {"hash"}} { } } } + + test {Stress test the hash ziplist -> hashtable encoding conversion} { + r config set hash-max-ziplist-entries 32 + for {set j 0} {$j < 100} {incr j} { + r del myhash + for {set i 0} {$i < 64} {incr i} { + r hset myhash [randomValue] [randomValue] + } + assert {[r object encoding myhash] eq {hashtable}} + } + } } From 0b8441c1b2f69cba5dddfdd76a166e6ac5b2afb4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Jun 2012 23:44:34 +0200 Subject: [PATCH 0271/1752] Dump ziplist hex value on failed assertion. The ziplist -> hashtable conversion code is triggered every time an hash value must be promoted to a full hash table because the number or size of elements reached the threshold. If a problem in the ziplist causes the same field to be present multiple times, the assertion of successful addition of the element inside the hash table will fail, crashing server with a failed assertion, but providing little information about the problem. This code adds a new logging function to perform the hex dump of binary data, and makes sure that the ziplist -> hashtable conversion code uses this new logging facility to dump the content of the ziplist when the assertion fails. This change was originally made in order to investigate issue #547. --- src/debug.c | 24 ++++++++++++++++++++++++ src/redis.h | 1 + src/t_hash.c | 6 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index 4687fb6c072..e56862882a3 100644 --- a/src/debug.c +++ b/src/debug.c @@ -686,6 +686,30 @@ void sigsegvHandler(int sig, siginfo_t *info, void *secret) { } #endif /* HAVE_BACKTRACE */ +/* ==================== Logging functions for debugging ===================== */ + +void redisLogHexDump(int level, char *descr, void *value, size_t len) { + char buf[65], *b; + unsigned char *v = value; + char charset[] = "0123456789abcdef"; + + redisLog(level,"%s (hexdump):", descr); + b = buf; + while(len) { + b[0] = charset[(*v)>>4]; + b[1] = charset[(*v)&0xf]; + b[2] = '\0'; + b += 2; + len--; + v++; + if (b-buf == 64 || len == 0) { + redisLogRaw(level|REDIS_LOG_RAW,buf); + b = buf; + } + } + redisLogRaw(level|REDIS_LOG_RAW,"\n"); +} + /* =========================== Software Watchdog ============================ */ #include diff --git a/src/redis.h b/src/redis.h index d877165a415..7cde4eb71a0 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1128,4 +1128,5 @@ sds genRedisInfoString(char *section); void enableWatchdog(int period); void disableWatchdog(void); void watchdogScheduleSignal(int period); +void redisLogHexDump(int level, char *descr, void *value, size_t len); #endif diff --git a/src/t_hash.c b/src/t_hash.c index 5b7a347abbc..aa021b03814 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -403,7 +403,11 @@ void hashTypeConvertZiplist(robj *o, int enc) { value = hashTypeCurrentObject(hi, REDIS_HASH_VALUE); value = tryObjectEncoding(value); ret = dictAdd(dict, field, value); - redisAssert(ret == DICT_OK); + if (ret != DICT_OK) { + redisLogHexDump(REDIS_WARNING,"ziplist with dup elements dump", + o->ptr,ziplistBlobLen(o->ptr)); + redisAssert(ret == DICT_OK); + } } hashTypeReleaseIterator(hi); From 1364395f18e810d6ed2e07f62dfc665fa1c6634c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Jun 2012 15:20:16 +0200 Subject: [PATCH 0272/1752] Added a new hash fuzzy tester. The new fuzzy tester also removes elements from the hash instead of just adding random fields. This should increase the probability to find bugs in the implementations of the hash type internal representations. --- tests/unit/type/hash.tcl | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/unit/type/hash.tcl b/tests/unit/type/hash.tcl index dbc1c4cc31b..fa52afd167a 100644 --- a/tests/unit/type/hash.tcl +++ b/tests/unit/type/hash.tcl @@ -397,7 +397,7 @@ start_server {tags {"hash"}} { } {b} foreach size {10 512} { - test "Hash fuzzing - $size fields" { + test "Hash fuzzing #1 - $size fields" { for {set times 0} {$times < 10} {incr times} { catch {unset hash} array set hash {} @@ -418,6 +418,43 @@ start_server {tags {"hash"}} { assert_equal [array size hash] [r hlen hash] } } + + test "Hash fuzzing #2 - $size fields" { + for {set times 0} {$times < 10} {incr times} { + catch {unset hash} + array set hash {} + r del hash + + # Create + for {set j 0} {$j < $size} {incr j} { + randpath { + set field [randomValue] + set value [randomValue] + r hset hash $field $value + set hash($field) $value + } { + set field [randomSignedInt 512] + set value [randomSignedInt 512] + r hset hash $field $value + set hash($field) $value + } { + randpath { + set field [randomValue] + } { + set field [randomSignedInt 512] + } + r hdel hash $field + unset -nocomplain hash($field) + } + } + + # Verify + foreach {k v} [array get hash] { + assert_equal $v [r hget hash $k] + } + assert_equal [array size hash] [r hlen hash] + } + } } test {Stress test the hash ziplist -> hashtable encoding conversion} { From e612508d38532933c7a39299851c41ca5527d784 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Tue, 12 Jun 2012 22:35:00 -0700 Subject: [PATCH 0273/1752] Standardize punctuation in redis-cli help. Right there is a mix of help entries ending with periods or without periods. This standardizes the end of command as without periods, which seems to be the general custom in most unix tools, at least. --- src/redis-cli.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index f485587957d..97caf5be810 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -712,17 +712,17 @@ static void usage() { " -a Password to use when connecting to the server\n" " -r Execute specified command N times\n" " -i When -r is used, waits seconds per command.\n" -" It is possible to specify sub-second times like -i 0.1.\n" +" It is possible to specify sub-second times like -i 0.1\n" " -n Database number\n" " -x Read last argument from STDIN\n" " -d Multi-bulk delimiter in for raw formatting (default: \\n)\n" " -c Enable cluster mode (follow -ASK and -MOVED redirections)\n" " --raw Use raw formatting for replies (default when STDOUT is not a tty)\n" -" --latency Enter a special mode continuously sampling latency.\n" -" --slave Simulate a slave showing commands received from the master.\n" -" --pipe Transfer raw Redis protocol from stdin to server.\n" -" --bigkeys Sample Redis keys looking for big keys.\n" -" --eval Send an EVAL command using the Lua script at .\n" +" --latency Enter a special mode continuously sampling latency\n" +" --slave Simulate a slave showing commands received from the master\n" +" --pipe Transfer raw Redis protocol from stdin to server\n" +" --bigkeys Sample Redis keys looking for big keys\n" +" --eval Send an EVAL command using the Lua script at \n" " --help Output this help and exit\n" " --version Output version and exit\n" "\n" From 8361d6c406fec73106c627159f28e092cedef1ee Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Jun 2012 15:59:25 +0200 Subject: [PATCH 0274/1752] ziplistFind(): don't assume that entries are comparable by encoding. Because Redis 2.6 introduced new integer encodings it is no longer true that if two entries have a different encoding they are not equal. An old ziplist can be loaded from an RDB file generated with Redis 2.4, in this case for instance a small unsigned integers is encoded with a 16 bit encoding, while in Redis 2.6 a more specific 8 bit encoding format is used. Because of this bug hashes ended with duplicated values or fields lookup failed, causing many bad behaviors. This in turn caused a crash while converting the ziplist encoded hash into a real hash table because an assertion was raised on duplicated elements. This commit fixes issue #547. Many thanks to Pinterest's Marty Weiner and colleagues for discovering the problem and helping us in the debugging process. --- src/ziplist.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/ziplist.c b/src/ziplist.c index 31e61633e5b..50167651ba1 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -805,19 +805,24 @@ unsigned char *ziplistFind(unsigned char *p, unsigned char *vstr, unsigned int v return p; } } else { - /* Find out if the specified entry can be encoded */ + /* Find out if the searched field can be encoded. Note that + * we do it only the first time, once done vencoding is set + * to non-zero and vll is set to the integer value. */ if (vencoding == 0) { - /* UINT_MAX when the entry CANNOT be encoded */ if (!zipTryEncoding(vstr, vlen, &vll, &vencoding)) { + /* If the entry can't be encoded we set it to + * UCHAR_MAX so that we don't retry again the next + * time. */ vencoding = UCHAR_MAX; } - /* Must be non-zero by now */ assert(vencoding); } - /* Compare current entry with specified entry */ - if (encoding == vencoding) { + /* Compare current entry with specified entry, do it only + * if vencoding != UCHAR_MAX because if there is no encoding + * possible for the field it can't be a valid integer. */ + if (vencoding != UCHAR_MAX) { long long ll = zipLoadInteger(q, encoding); if (ll == vll) { return p; From 6fe9c402a2b9dcd3cb1a15aed08a86338ec143f3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 15 Jun 2012 10:03:25 +0200 Subject: [PATCH 0275/1752] Fix c->reply_bytes computation in setDeferredMultiBulkLength() In order to implement reply buffer limits introduced in 2.6 and useful to close the connection under user-selected circumastances of big output buffers (for instance slow consumers in pub/sub, a blocked slave, and so forth) Redis takes a counter with the amount of used memory in objects inside the output list stored into c->reply. The computation was broken in the function setDeferredMultiBulkLength(), in the case the object was glued with the next one. This caused the c->reply_bytes field to go out of sync, be subtracted more than needed, and wrap back near to ULONG_MAX values. This commit fixes this bug and adds an assertion that is able to trap this class of problems. This problem was discovered looking at the INFO output of an unrelated issue (issue #547). --- src/networking.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/networking.c b/src/networking.c index f922e297513..b3b7b94a7d1 100644 --- a/src/networking.c +++ b/src/networking.c @@ -364,7 +364,10 @@ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) { /* Only glue when the next node is non-NULL (an sds in this case) */ if (next->ptr != NULL) { + c->reply_bytes -= zmalloc_size_sds(len->ptr); + c->reply_bytes -= zmalloc_size_sds(next->ptr); len->ptr = sdscatlen(len->ptr,next->ptr,sdslen(next->ptr)); + c->reply_bytes += zmalloc_size_sds(len->ptr); listDelNode(c->reply,ln->next); } } @@ -1302,6 +1305,7 @@ int checkClientOutputBufferLimits(redisClient *c) { * called from contexts where the client can't be freed safely, i.e. from the * lower level functions pushing data inside the client output buffers. */ void asyncCloseClientOnOutputBufferLimitReached(redisClient *c) { + redisAssert(c->reply_bytes < ULONG_MAX-(1024*64)); if (c->reply_bytes == 0 || c->flags & REDIS_CLOSE_ASAP) return; if (checkClientOutputBufferLimits(c)) { sds client = getClientInfoString(c); From 0c9cf45270141d9f7bb1124a98fbe8c8d2e1d461 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 15 Jun 2012 13:44:17 +0200 Subject: [PATCH 0276/1752] Redis 2.5.11 (2.6 RC5). --- 00-RELEASENOTES | 13 +++++++++++++ src/version.h | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 031a433ea79..5c76973df38 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,19 @@ HIGH: There is a critical bug that may affect some part of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +---[ Redis 2.5.11 (2.6 Release Candidate 5) ] + +UPGRADE URGENCY: HIGH. + +* [BUGFIX] Fixed Hash corruption when loading an RDB file generated by + previous versions of Redis that encoded hashes using + a different ziplist encoding format for small integers. + All the fileds that are integers in the range 0-255 may not + be recognized, or duplicated un updates, causing a crash + when the ziplist is converted to a real hash. (Issue #547). +* [BUGFIX] Fixed the count of memory used by output buffers in the + setDeferredMultiBulkLength() function. + ---[ Redis 2.5.10 (2.6 Release Candidate 4) ] UPGRADE URGENCY: HIGH. diff --git a/src/version.h b/src/version.h index e2827f572f8..e98f7b40168 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.5.10" +#define REDIS_VERSION "2.5.11" From 4b3865cbdbb5ceeb7e284e622550855071c2b8d6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Jun 2012 11:50:01 +0200 Subject: [PATCH 0277/1752] Fixed a timing attack on AUTH (Issue #560). The way we compared the authentication password using strcmp() allowed an attacker to gain information about the password using a well known class of attacks called "timing attacks". The bug appears to be practically not exploitable in most modern systems running Redis since even using multiple bytes of differences in the input at a time instead of one the difference in running time in in the order of 10 nanoseconds, making it hard to exploit even on LAN. However attacks always get better so we are providing a fix ASAP. The new implementation uses two fixed length buffers and a constant time comparison function, with the goal of: 1) Completely avoid leaking information about the content of the password, since the comparison is always performed between 512 characters and without conditionals. 2) Partially avoid leaking information about the length of the password. About "2" we still have a stage in the code where the real password and the user provided password are copied in the static buffers, we also run two strlen() operations against the two inputs, so the running time of the comparison is a fixed amount plus a time proportional to LENGTH(A)+LENGTH(B). This means that the absolute time of the operation performed is still related to the length of the password in some way, but there is no way to change the input in order to get a difference in the execution time in the comparison that is not just proportional to the string provided by the user (because the password length is fixed). Thus in practical terms the user should try to discover LENGTH(PASSWORD) looking at the whole execution time of the AUTH command and trying to guess a proportionality between the whole execution time and the password length: this appears to be mostly unfeasible in the real world. Also protecting from this attack is not very useful in the case of Redis as a brute force attack is anyway feasible if the password is too short, while with a long password makes it not an issue that the attacker knows the length. --- src/config.c | 5 +++++ src/redis.c | 44 +++++++++++++++++++++++++++++++++++++++++++- src/redis.h | 1 + 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index c2ea5b76816..6b513940b2b 100644 --- a/src/config.c +++ b/src/config.c @@ -264,6 +264,10 @@ void loadServerConfigFromString(char *config) { { server.aof_rewrite_min_size = memtoll(argv[1],NULL); } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { + if (strlen(argv[1]) > REDIS_AUTHPASS_MAX_LEN) { + err = "Password is longer than REDIS_AUTHPASS_MAX_LEN"; + goto loaderr; + } server.requirepass = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"pidfile") && argc == 2) { zfree(server.pidfile); @@ -411,6 +415,7 @@ void configSetCommand(redisClient *c) { zfree(server.rdb_filename); server.rdb_filename = zstrdup(o->ptr); } else if (!strcasecmp(c->argv[2]->ptr,"requirepass")) { + if (sdslen(o->ptr) > REDIS_AUTHPASS_MAX_LEN) goto badfmt; zfree(server.requirepass); server.requirepass = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } else if (!strcasecmp(c->argv[2]->ptr,"masterauth")) { diff --git a/src/redis.c b/src/redis.c index 46915c1302f..ec313fbf8a0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1698,10 +1698,52 @@ int prepareForShutdown(int flags) { /*================================== Commands =============================== */ +/* Return 0 if strings are the same, 1 if they are not. + * The comparison is performed in a way that prevents an attacker to obtain + * information about the nature of the strings just monitoring the execution + * time of the function. + * + * Note that limiting the comparison length to strings up to 512 bytes we + * can avoid leaking any information about the password length and any + * possible branch misprediction related leak. + */ +int time_independent_strcmp(char *a, char *b) { + char bufa[REDIS_AUTHPASS_MAX_LEN], bufb[REDIS_AUTHPASS_MAX_LEN]; + /* The above two strlen perform len(a) + len(b) operations where either + * a or b are fixed (our password) length, and the difference is only + * relative to the length of the user provided string, so no information + * leak is possible in the following two lines of code. */ + int alen = strlen(a); + int blen = strlen(b); + int j; + int diff = 0; + + /* We can't compare strings longer than our static buffers. + * Note that this will never pass the first test in practical circumstances + * so there is no info leak. */ + if (alen > sizeof(bufa) || blen > sizeof(bufb)) return 1; + + memset(bufa,0,sizeof(bufa)); /* Constant time. */ + memset(bufb,0,sizeof(bufb)); /* Constant time. */ + /* Again the time of the following two copies is proportional to + * len(a) + len(b) so no info is leaked. */ + memcpy(bufa,a,alen); + memcpy(bufb,b,blen); + + /* Always compare all the chars in the two buffers without + * conditional expressions. */ + for (j = 0; j < sizeof(bufa); j++) { + diff |= (bufa[j] ^ bufb[j]); + } + /* Length must be equal as well. */ + diff |= alen ^ blen; + return diff; /* If zero strings are the same. */ +} + void authCommand(redisClient *c) { if (!server.requirepass) { addReplyError(c,"Client sent AUTH, but no password is set"); - } else if (!strcmp(c->argv[1]->ptr, server.requirepass)) { + } else if (!time_independent_strcmp(c->argv[1]->ptr, server.requirepass)) { c->authenticated = 1; addReply(c,shared.ok); } else { diff --git a/src/redis.h b/src/redis.h index 7cde4eb71a0..a194bed4468 100644 --- a/src/redis.h +++ b/src/redis.h @@ -56,6 +56,7 @@ #define REDIS_SLOWLOG_LOG_SLOWER_THAN 10000 #define REDIS_SLOWLOG_MAX_LEN 128 #define REDIS_MAX_CLIENTS 10000 +#define REDIS_AUTHPASS_MAX_LEN 512 #define REDIS_REPL_TIMEOUT 60 #define REDIS_REPL_PING_SLAVE_PERIOD 10 From b3f28b90d29f28458d4cb00382ff988a98750d05 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Jun 2012 14:25:53 +0200 Subject: [PATCH 0278/1752] Fixed comment typo into time_independent_strcmp(). --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index ec313fbf8a0..3b7af45648c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1698,7 +1698,7 @@ int prepareForShutdown(int flags) { /*================================== Commands =============================== */ -/* Return 0 if strings are the same, 1 if they are not. +/* Return zero if strings are the same, non-zero if they are not. * The comparison is performed in a way that prevents an attacker to obtain * information about the nature of the strings just monitoring the execution * time of the function. From dbd8c753c43b83fd326acc7974768261589b239e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Jun 2012 09:47:47 +0200 Subject: [PATCH 0279/1752] REPLCONF internal command introduced. The REPLCONF command is an internal command (not designed to be directly used by normal clients) that allows a slave to set some replication related state in the master before issuing SYNC to start the replication. The initial motivation for this command, and the only reason currently it is used by the implementation, is to let the slave instance communicate its listening port to the slave, so that the master can show all the slaves with their listening ports in the "replication" section of the INFO output. This allows clients to auto discover and query all the slaves attached into a master. Currently only a single option of the REPLCONF command is supported, and it is called "listening-port", so the slave now starts the replication process with something like the following chat: REPLCONF listening-prot 6380 SYNC Note that this works even if the master is an older version of Redis and does not understand REPLCONF, because the slave ignores the REPLCONF error. In the future REPLCONF can be used for partial replication and other replication related features where there is the need to exchange information between master and slave. NOTE: This commit also fixes a bug: the INFO outout already carried information about slaves, but the port was broken, and was obtained with getpeername(2), so it was actually just the ephemeral port used by the slave to connect to the master as a client. --- src/networking.c | 1 + src/redis.c | 3 +- src/redis.h | 2 + src/replication.c | 122 +++++++++++++++++++++++++++++++++++++++------- 4 files changed, 110 insertions(+), 18 deletions(-) diff --git a/src/networking.c b/src/networking.c index b3b7b94a7d1..3bc084f7d5f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -55,6 +55,7 @@ redisClient *createClient(int fd) { c->ctime = c->lastinteraction = server.unixtime; c->authenticated = 0; c->replstate = REDIS_REPL_NONE; + c->slave_listening_port = 0; c->reply = listCreate(); c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; diff --git a/src/redis.c b/src/redis.c index 3b7af45648c..63afefad922 100644 --- a/src/redis.c +++ b/src/redis.c @@ -215,6 +215,7 @@ struct redisCommand redisCommandTable[] = { {"exec",execCommand,1,"s",0,NULL,0,0,0,0,0}, {"discard",discardCommand,1,"rs",0,NULL,0,0,0,0,0}, {"sync",syncCommand,1,"ars",0,NULL,0,0,0,0,0}, + {"replconf",replconfCommand,-1,"ars",0,NULL,0,0,0,0,0}, {"flushdb",flushdbCommand,1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,1,"w",0,NULL,0,0,0,0,0}, {"sort",sortCommand,-2,"wmS",0,NULL,1,1,1,0,0}, @@ -2075,7 +2076,7 @@ sds genRedisInfoString(char *section) { } if (state == NULL) continue; info = sdscatprintf(info,"slave%d:%s,%d,%s\r\n", - slaveid,ip,port,state); + slaveid,ip,slave->slave_listening_port,state); slaveid++; } } diff --git a/src/redis.h b/src/redis.h index a194bed4468..8195665a875 100644 --- a/src/redis.h +++ b/src/redis.h @@ -346,6 +346,7 @@ typedef struct redisClient { int repldbfd; /* replication DB file descriptor */ long repldboff; /* replication DB file offset */ off_t repldbsize; /* replication DB file size */ + int slave_listening_port; /* As configured with: SLAVECONF listening-port */ multiState mstate; /* MULTI/EXEC state */ blockingState bpop; /* blocking state */ list *io_keys; /* Keys this client is waiting to be loaded from the @@ -1110,6 +1111,7 @@ void scriptCommand(redisClient *c); void timeCommand(redisClient *c); void bitopCommand(redisClient *c); void bitcountCommand(redisClient *c); +void replconfCommand(redisClient *c); #if defined(__GNUC__) void *calloc(size_t count, size_t size) __attribute__ ((deprecated)); diff --git a/src/replication.c b/src/replication.c index 8eb36f837e1..cc1ac980de3 100644 --- a/src/replication.c +++ b/src/replication.c @@ -145,6 +145,46 @@ void syncCommand(redisClient *c) { return; } +/* REPLCONF

Name

jemalloc — general purpose memory allocation functions

LIBRARY

This manual describes jemalloc 3.0.0-0-gfc9b1dbf69f59d7ecfc4ac68da9847e017e1d046. More information +JEMALLOC

Name

jemalloc — general purpose memory allocation functions

LIBRARY

This manual describes jemalloc 3.2.0-0-g87499f6748ebe4817571e817e9f680ccb5bf54a9. More information can be found at the jemalloc website.

SYNOPSIS

#include <stdlib.h>
-#include <jemalloc/jemalloc.h>

Standard API

void *malloc(size_t size);
 
void *calloc(size_t number,
 size_t size);
 
int posix_memalign(void **ptr,
 size_t alignment,
 size_t size);
 
void *aligned_alloc(size_t alignment,
 size_t size);
 
void *realloc(void *ptr,
 size_t size);
 
void free(void *ptr);
 

Non-standard API

size_t malloc_usable_size(const void *ptr);
 
void malloc_stats_print(void (*write_cb) +#include <jemalloc/jemalloc.h>

Standard API

void *malloc(size_t size);
 
void *calloc(size_t number,
 size_t size);
 
int posix_memalign(void **ptr,
 size_t alignment,
 size_t size);
 
void *aligned_alloc(size_t alignment,
 size_t size);
 
void *realloc(void *ptr,
 size_t size);
 
void free(void *ptr);
 

Non-standard API

size_t malloc_usable_size(const void *ptr);
 
void malloc_stats_print(void (*write_cb) (void *, const char *) - ,
 void *cbopaque,
 const char *opts);
 
int mallctl(const char *name,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
int mallctlnametomib(const char *name,
 size_t *mibp,
 size_t *miblenp);
 
int mallctlbymib(const size_t *mib,
 size_t miblen,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
void (*malloc_message)(void *cbopaque,
 const char *s);
 

const char *malloc_conf;

Experimental API

int allocm(void **ptr,
 size_t *rsize,
 size_t size,
 int flags);
 
int rallocm(void **ptr,
 size_t *rsize,
 size_t size,
 size_t extra,
 int flags);
 
int sallocm(const void *ptr,
 size_t *rsize,
 int flags);
 
int dallocm(void *ptr,
 int flags);
 
int nallocm(size_t *rsize,
 size_t size,
 int flags);
 

DESCRIPTION

Standard API

The malloc() function allocates + ,

 void *cbopaque,
 const char *opts);
 
int mallctl(const char *name,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
int mallctlnametomib(const char *name,
 size_t *mibp,
 size_t *miblenp);
 
int mallctlbymib(const size_t *mib,
 size_t miblen,
 void *oldp,
 size_t *oldlenp,
 void *newp,
 size_t newlen);
 
void (*malloc_message)(void *cbopaque,
 const char *s);
 

const char *malloc_conf;

Experimental API

int allocm(void **ptr,
 size_t *rsize,
 size_t size,
 int flags);
 
int rallocm(void **ptr,
 size_t *rsize,
 size_t size,
 size_t extra,
 int flags);
 
int sallocm(const void *ptr,
 size_t *rsize,
 int flags);
 
int dallocm(void *ptr,
 int flags);
 
int nallocm(size_t *rsize,
 size_t size,
 int flags);
 

DESCRIPTION

Standard API

The malloc() function allocates size bytes of uninitialized memory. The allocated space is suitably aligned (after possible pointer coercion) for storage of any type of object.

The calloc() function allocates @@ -38,7 +38,7 @@ malloc() for the specified size.

The free() function causes the allocated memory referenced by ptr to be made available for future allocations. If ptr is - NULL, no action occurs.

Non-standard API

The malloc_usable_size() function + NULL, no action occurs.

Non-standard API

The malloc_usable_size() function returns the usable size of the allocation pointed to by ptr. The return value may be larger than the size that was requested during allocation. The @@ -118,7 +118,7 @@ len = sizeof(bin_size); mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0); /* Do something with bin_size... */ -}

Experimental API

The experimental API is subject to change or removal without regard +}

Experimental API

The experimental API is subject to change or removal without regard for backward compatibility. If --disable-experimental is specified during configuration, the experimental API is omitted.

The allocm(), @@ -146,7 +146,11 @@ that are initialized to contain zero bytes. If this option is absent, newly allocated memory is uninitialized.

ALLOCM_NO_MOVE

For reallocation, fail rather than moving the object. This constraint can apply to both growth and - shrinkage.

+ shrinkage.

ALLOCM_ARENA(a) +

Use the arena specified by the index + a. This macro does not validate that + a specifies an arena in the valid + range.

The allocm() function allocates at least size bytes of memory, sets *ptr to the base address of the allocation, and @@ -404,15 +408,24 @@ (size_t) r-

Virtual memory chunk size (log base 2). The default - chunk size is 4 MiB (2^22).

+ chunk size is 4 MiB (2^22).

+ + "opt.dss" + + (const char *) + r- +

dss (sbrk(2)) allocation precedence as + related to mmap(2) allocation. The following + settings are supported: “disabled”, “primary”, + and “secondary” (default).

"opt.narenas" (size_t) r- -

Maximum number of arenas to use. The default maximum - number of arenas is four times the number of CPUs, or one if there is a - single CPU.

+

Maximum number of arenas to use for automatic + multiplexing of threads and arenas. The default is four times the + number of CPUs, or one if there is a single CPU.

"opt.lg_dirty_mult" @@ -425,7 +438,7 @@ pages via madvise(2) or a similar system call. This provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused. The - default minimum ratio is 32:1 (2^5:1); an option value of -1 will + default minimum ratio is 8:1 (2^3:1); an option value of -1 will disable dirty page purging.

"opt.stats_print" @@ -454,7 +467,8 @@ 0x5a. This is intended for debugging and will impact performance negatively. This option is disabled by default unless --enable-debug is specified during - configuration, in which case it is enabled by default.

+ configuration, in which case it is enabled by default unless running + inside Valgrind.

"opt.quarantine" @@ -470,8 +484,9 @@ option is enabled. This feature is of particular use in combination with Valgrind, which can detect attempts to access quarantined objects. This is intended for debugging and will - impact performance negatively. The default quarantine size is - 0.

+ impact performance negatively. The default quarantine size is 0 unless + running inside Valgrind, in which case the default is 16 + MiB.

"opt.redzone" @@ -489,7 +504,7 @@ which needs redzones in order to do effective buffer overflow/underflow detection. This option is intended for debugging and will impact performance negatively. This option is disabled by - default.

+ default unless running inside Valgrind.

"opt.zero" @@ -520,21 +535,9 @@ r- [--enable-valgrind]

Valgrind - support enabled/disabled. If enabled, several other options are - automatically modified during options processing to work well with - Valgrind: - "opt.junk" - - and - "opt.zero" - are set - to false, - "opt.quarantine" - is - set to 16 MiB, and - "opt.redzone" - is set to - true. This option is disabled by default.

+ support enabled/disabled. This option is vestigal because jemalloc + auto-detects whether it is running inside Valgrind. This option is + disabled by default, unless running inside Valgrind.

"opt.xmalloc" @@ -566,7 +569,7 @@ "opt.lg_tcache_max" option for related tuning information. This option is enabled by - default.

+ default unless running inside Valgrind.

"opt.lg_tcache_max" @@ -724,12 +727,8 @@ (unsigned) rw

Get or set the arena associated with the calling - thread. The arena index must be less than the maximum number of arenas - (see the - "arenas.narenas" - - mallctl). If the specified arena was not initialized beforehand (see - the + thread. If the specified arena was not initialized beforehand (see the + "arenas.initialized" mallctl), it will be automatically initialized as a side effect of @@ -804,13 +803,38 @@ a thread exits. However, garbage collection is triggered by allocation activity, so it is possible for a thread that stops allocating/deallocating to retain its cache indefinitely, in which case - the developer may find manual flushing useful.

+ the developer may find manual flushing useful.

+ + "arena.<i>.purge" + + (unsigned) + -- +

Purge unused dirty pages for arena <i>, or for + all arenas if <i> equals + "arenas.narenas" + . +

+ + "arena.<i>.dss" + + (const char *) + rw +

Set the precedence of dss allocation as related to mmap + allocation for arena <i>, or for all arenas if <i> equals + + "arenas.narenas" + . See + + "opt.dss" + for supported + settings. +

"arenas.narenas" (unsigned) r- -

Maximum number of arenas.

+

Current limit on number of arenas.

"arenas.initialized" @@ -891,7 +915,14 @@ (unsigned) -w

Purge unused dirty pages for the specified arena, or - for all arenas if none is specified.

+ for all arenas if none is specified.

+ + "arenas.extend" + + (unsigned) + r- +

Extend the array of arenas by appending a new arena, + and returning the new arena index.

"prof.active" @@ -966,7 +997,11 @@ equal to "stats.allocated" . -

+ This does not include + + "stats.arenas.<i>.pdirty" + and pages + entirely devoted to allocator metadata.

"stats.mapped" @@ -1028,6 +1063,16 @@

Cumulative number of huge deallocation requests.

+ "stats.arenas.<i>.dss" + + (const char *) + r- +

dss (sbrk(2)) allocation precedence as + related to mmap(2) allocation. See + "opt.dss" + for details. +

+ "stats.arenas.<i>.nthreads" (unsigned) @@ -1039,7 +1084,7 @@ (size_t) r- -

Number of pages in active runs.

+

Number of pages in active runs.

"stats.arenas.<i>.pdirty" @@ -1263,11 +1308,7 @@ it detects, because the performance impact for storing such information would be prohibitive. However, jemalloc does integrate with the most excellent Valgrind tool if the - --enable-valgrind configuration option is enabled and the - - "opt.valgrind" - option - is enabled.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an + --enable-valgrind configuration option is enabled.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an error or warning condition, a message will be printed to file descriptor STDERR_FILENO. Errors will result in the process dumping core. If the @@ -1283,7 +1324,7 @@ malloc_stats_print(), followed by a string pointer. Please note that doing anything which tries to allocate memory in this function is likely to result in a crash or deadlock.

All messages are prefixed by - “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and + “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and calloc() functions return a pointer to the allocated memory if successful; otherwise a NULL pointer is returned and errno is set to @@ -1311,7 +1352,7 @@ allocation failure. The realloc() function always leaves the original buffer intact when an error occurs.

The free() function returns no - value.

Non-standard API

The malloc_usable_size() function + value.

Non-standard API

The malloc_usable_size() function returns the usable size of the allocation pointed to by ptr.

The mallctl(), mallctlnametomib(), and @@ -1330,7 +1371,7 @@ occurred.

EFAULT

An interface with side effects failed in some way not directly related to mallctl*() read/write processing.

-

Experimental API

The allocm(), +

Experimental API

The allocm(), rallocm(), sallocm(), dallocm(), and diff --git a/deps/jemalloc/doc/jemalloc.xml.in b/deps/jemalloc/doc/jemalloc.xml.in index 877c500f36d..54b87474c84 100644 --- a/deps/jemalloc/doc/jemalloc.xml.in +++ b/deps/jemalloc/doc/jemalloc.xml.in @@ -368,6 +368,15 @@ for (i = 0; i < nbins; i++) { object. This constraint can apply to both growth and shrinkage. + + ALLOCM_ARENA(a) + + + Use the arena specified by the index + a. This macro does not validate that + a specifies an arena in the valid + range. + @@ -785,15 +794,29 @@ for (i = 0; i < nbins; i++) { chunk size is 4 MiB (2^22). + + + opt.dss + (const char *) + r- + + dss (sbrk + 2) allocation precedence as + related to mmap + 2 allocation. The following + settings are supported: “disabled”, “primary”, + and “secondary” (default). + + opt.narenas (size_t) r- - Maximum number of arenas to use. The default maximum - number of arenas is four times the number of CPUs, or one if there is a - single CPU. + Maximum number of arenas to use for automatic + multiplexing of threads and arenas. The default is four times the + number of CPUs, or one if there is a single CPU. @@ -810,7 +833,7 @@ for (i = 0; i < nbins; i++) { 2 or a similar system call. This provides the kernel with sufficient information to recycle dirty pages if physical memory becomes scarce and the pages remain unused. The - default minimum ratio is 32:1 (2^5:1); an option value of -1 will + default minimum ratio is 8:1 (2^3:1); an option value of -1 will disable dirty page purging. @@ -846,7 +869,9 @@ for (i = 0; i < nbins; i++) { 0x5a. This is intended for debugging and will impact performance negatively. This option is disabled by default unless is specified during - configuration, in which case it is enabled by default. + configuration, in which case it is enabled by default unless running + inside Valgrind. @@ -865,8 +890,9 @@ for (i = 0; i < nbins; i++) { enabled. This feature is of particular use in combination with Valgrind, which can detect attempts to access quarantined objects. This is intended for debugging and will - impact performance negatively. The default quarantine size is - 0. + impact performance negatively. The default quarantine size is 0 unless + running inside Valgrind, in which case the default is 16 + MiB. @@ -885,7 +911,7 @@ for (i = 0; i < nbins; i++) { which needs redzones in order to do effective buffer overflow/underflow detection. This option is intended for debugging and will impact performance negatively. This option is disabled by - default. + default unless running inside Valgrind. @@ -926,15 +952,9 @@ for (i = 0; i < nbins; i++) { [] Valgrind - support enabled/disabled. If enabled, several other options are - automatically modified during options processing to work well with - Valgrind: opt.junk - and opt.zero are set - to false, opt.quarantine is - set to 16 MiB, and opt.redzone is set to - true. This option is disabled by default. + support enabled/disabled. This option is vestigal because jemalloc + auto-detects whether it is running inside Valgrind. This option is + disabled by default, unless running inside Valgrind. @@ -972,7 +992,8 @@ malloc_conf = "xmalloc:true";]]> opt.lg_tcache_max option for related tuning information. This option is enabled by - default. + default unless running inside Valgrind. @@ -1151,11 +1172,8 @@ malloc_conf = "xmalloc:true";]]> rw Get or set the arena associated with the calling - thread. The arena index must be less than the maximum number of arenas - (see the arenas.narenas - mallctl). If the specified arena was not initialized beforehand (see - the arenas.initialized mallctl), it will be automatically initialized as a side effect of calling this interface. @@ -1247,13 +1265,40 @@ malloc_conf = "xmalloc:true";]]> the developer may find manual flushing useful. + + + arena.<i>.purge + (unsigned) + -- + + Purge unused dirty pages for arena <i>, or for + all arenas if <i> equals arenas.narenas. + + + + + + arena.<i>.dss + (const char *) + rw + + Set the precedence of dss allocation as related to mmap + allocation for arena <i>, or for all arenas if <i> equals + arenas.narenas. See + opt.dss for supported + settings. + + + arenas.narenas (unsigned) r- - Maximum number of arenas. + Current limit on number of arenas. @@ -1372,6 +1417,16 @@ malloc_conf = "xmalloc:true";]]> for all arenas if none is specified. + + + arenas.extend + (unsigned) + r- + + Extend the array of arenas by appending a new arena, + and returning the new arena index. + + prof.active @@ -1457,7 +1512,9 @@ malloc_conf = "xmalloc:true";]]> application. This is a multiple of the page size, and greater than or equal to stats.allocated. - + This does not include + stats.arenas.<i>.pdirty and pages + entirely devoted to allocator metadata. @@ -1540,6 +1597,20 @@ malloc_conf = "xmalloc:true";]]> + + + stats.arenas.<i>.dss + (const char *) + r- + + dss (sbrk + 2) allocation precedence as + related to mmap + 2 allocation. See opt.dss for details. + + + stats.arenas.<i>.nthreads @@ -1559,7 +1630,7 @@ malloc_conf = "xmalloc:true";]]> Number of pages in active runs. - + stats.arenas.<i>.pdirty (size_t) @@ -1865,9 +1936,7 @@ malloc_conf = "xmalloc:true";]]> it detects, because the performance impact for storing such information would be prohibitive. However, jemalloc does integrate with the most excellent Valgrind tool if the - configuration option is enabled and the - opt.valgrind option - is enabled. + configuration option is enabled. DIAGNOSTIC MESSAGES diff --git a/deps/jemalloc/include/jemalloc/internal/arena.h b/deps/jemalloc/include/jemalloc/internal/arena.h index 0b0f640a459..561c9b6ffe9 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena.h +++ b/deps/jemalloc/include/jemalloc/internal/arena.h @@ -38,10 +38,10 @@ * * (nactive >> opt_lg_dirty_mult) >= ndirty * - * So, supposing that opt_lg_dirty_mult is 5, there can be no less than 32 - * times as many active pages as dirty pages. + * So, supposing that opt_lg_dirty_mult is 3, there can be no less than 8 times + * as many active pages as dirty pages. */ -#define LG_DIRTY_MULT_DEFAULT 5 +#define LG_DIRTY_MULT_DEFAULT 3 typedef struct arena_chunk_map_s arena_chunk_map_t; typedef struct arena_chunk_s arena_chunk_t; @@ -69,7 +69,7 @@ struct arena_chunk_map_s { /* * Linkage for run trees. There are two disjoint uses: * - * 1) arena_t's runs_avail_{clean,dirty} trees. + * 1) arena_t's runs_avail tree. * 2) arena_run_t conceptually uses this linkage for in-use * non-full runs, rather than directly embedding linkage. */ @@ -162,20 +162,24 @@ typedef rb_tree(arena_chunk_map_t) arena_run_tree_t; /* Arena chunk header. */ struct arena_chunk_s { /* Arena that owns the chunk. */ - arena_t *arena; + arena_t *arena; - /* Linkage for the arena's chunks_dirty list. */ - ql_elm(arena_chunk_t) link_dirty; - - /* - * True if the chunk is currently in the chunks_dirty list, due to - * having at some point contained one or more dirty pages. Removal - * from chunks_dirty is lazy, so (dirtied && ndirty == 0) is possible. - */ - bool dirtied; + /* Linkage for tree of arena chunks that contain dirty runs. */ + rb_node(arena_chunk_t) dirty_link; /* Number of dirty pages. */ - size_t ndirty; + size_t ndirty; + + /* Number of available runs. */ + size_t nruns_avail; + + /* + * Number of available run adjacencies. Clean and dirty available runs + * are not coalesced, which causes virtual memory fragmentation. The + * ratio of (nruns_avail-nruns_adjac):nruns_adjac is used for tracking + * this fragmentation. + * */ + size_t nruns_adjac; /* * Map of pages within chunk that keeps track of free/large/small. The @@ -183,7 +187,7 @@ struct arena_chunk_s { * need to be tracked in the map. This omission saves a header page * for common chunk sizes (e.g. 4 MiB). */ - arena_chunk_map_t map[1]; /* Dynamically sized. */ + arena_chunk_map_t map[1]; /* Dynamically sized. */ }; typedef rb_tree(arena_chunk_t) arena_chunk_tree_t; @@ -331,8 +335,10 @@ struct arena_s { uint64_t prof_accumbytes; - /* List of dirty-page-containing chunks this arena manages. */ - ql_head(arena_chunk_t) chunks_dirty; + dss_prec_t dss_prec; + + /* Tree of dirty-page-containing chunks this arena manages. */ + arena_chunk_tree_t chunks_dirty; /* * In order to avoid rapid chunk allocation/deallocation when an arena @@ -367,18 +373,9 @@ struct arena_s { /* * Size/address-ordered trees of this arena's available runs. The trees - * are used for first-best-fit run allocation. The dirty tree contains - * runs with dirty pages (i.e. very likely to have been touched and - * therefore have associated physical pages), whereas the clean tree - * contains runs with pages that either have no associated physical - * pages, or have pages that the kernel may recycle at any time due to - * previous madvise(2) calls. The dirty tree is used in preference to - * the clean tree for allocations, because using dirty pages reduces - * the amount of dirty purging necessary to keep the active:dirty page - * ratio below the purge threshold. + * are used for first-best-fit run allocation. */ - arena_avail_tree_t runs_avail_clean; - arena_avail_tree_t runs_avail_dirty; + arena_avail_tree_t runs_avail; /* bins is used to store trees of free regions. */ arena_bin_t bins[NBINS]; @@ -422,13 +419,16 @@ void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); -void arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats); void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); -void *arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache); +void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc); +dss_prec_t arena_dss_prec_get(arena_t *arena); +void arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); +void arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, + size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, + malloc_large_stats_t *lstats); bool arena_new(arena_t *arena, unsigned ind); void arena_boot(void); void arena_prefork(arena_t *arena); diff --git a/deps/jemalloc/include/jemalloc/internal/chunk.h b/deps/jemalloc/include/jemalloc/internal/chunk.h index 8fb1fe6d15c..87d8700dac8 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk.h @@ -28,6 +28,7 @@ #ifdef JEMALLOC_H_EXTERNS extern size_t opt_lg_chunk; +extern const char *opt_dss; /* Protects stats_chunks; currently not used for any other purpose. */ extern malloc_mutex_t chunks_mtx; @@ -42,9 +43,14 @@ extern size_t chunk_npages; extern size_t map_bias; /* Number of arena chunk header pages. */ extern size_t arena_maxclass; /* Max size class for arenas. */ -void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero); +void *chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, + dss_prec_t dss_prec); +void chunk_unmap(void *chunk, size_t size); void chunk_dealloc(void *chunk, size_t size, bool unmap); bool chunk_boot(void); +void chunk_prefork(void); +void chunk_postfork_parent(void); +void chunk_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h index 6e2643b24c3..6585f071bbe 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h @@ -1,14 +1,28 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES +typedef enum { + dss_prec_disabled = 0, + dss_prec_primary = 1, + dss_prec_secondary = 2, + + dss_prec_limit = 3 +} dss_prec_t ; +#define DSS_PREC_DEFAULT dss_prec_secondary +#define DSS_DEFAULT "secondary" + #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +extern const char *dss_prec_names[]; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS +dss_prec_t chunk_dss_prec_get(void); +bool chunk_dss_prec_set(dss_prec_t dss_prec); void *chunk_alloc_dss(size_t size, size_t alignment, bool *zero); bool chunk_in_dss(void *chunk); bool chunk_dss_boot(void); diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h b/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h index b29f39e9eaf..f24abac7538 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_mmap.h @@ -9,7 +9,7 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -void pages_purge(void *addr, size_t length); +bool pages_purge(void *addr, size_t length); void *chunk_alloc_mmap(size_t size, size_t alignment, bool *zero); bool chunk_dealloc_mmap(void *chunk, size_t size); diff --git a/deps/jemalloc/include/jemalloc/internal/ctl.h b/deps/jemalloc/include/jemalloc/internal/ctl.h index adf3827f094..0ffecc5f2a2 100644 --- a/deps/jemalloc/include/jemalloc/internal/ctl.h +++ b/deps/jemalloc/include/jemalloc/internal/ctl.h @@ -33,6 +33,7 @@ struct ctl_indexed_node_s { struct ctl_arena_stats_s { bool initialized; unsigned nthreads; + const char *dss; size_t pactive; size_t pdirty; arena_stats_t astats; @@ -61,6 +62,7 @@ struct ctl_stats_s { uint64_t nmalloc; /* huge_nmalloc */ uint64_t ndalloc; /* huge_ndalloc */ } huge; + unsigned narenas; ctl_arena_stats_t *arenas; /* (narenas + 1) elements. */ }; @@ -75,6 +77,9 @@ int ctl_nametomib(const char *name, size_t *mibp, size_t *miblenp); int ctl_bymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen); bool ctl_boot(void); +void ctl_prefork(void); +void ctl_postfork_parent(void); +void ctl_postfork_child(void); #define xmallctl(name, oldp, oldlenp, newp, newlen) do { \ if (je_mallctl(name, oldp, oldlenp, newp, newlen) \ diff --git a/deps/jemalloc/include/jemalloc/internal/extent.h b/deps/jemalloc/include/jemalloc/internal/extent.h index 36af8be8995..ba95ca816bd 100644 --- a/deps/jemalloc/include/jemalloc/internal/extent.h +++ b/deps/jemalloc/include/jemalloc/internal/extent.h @@ -23,6 +23,9 @@ struct extent_node_s { /* Total region size. */ size_t size; + + /* True if zero-filled; used by chunk recycling code. */ + bool zeroed; }; typedef rb_tree(extent_node_t) extent_tree_t; diff --git a/deps/jemalloc/include/jemalloc/internal/huge.h b/deps/jemalloc/include/jemalloc/internal/huge.h index e8513c933e2..d987d370767 100644 --- a/deps/jemalloc/include/jemalloc/internal/huge.h +++ b/deps/jemalloc/include/jemalloc/internal/huge.h @@ -22,7 +22,7 @@ void *huge_palloc(size_t size, size_t alignment, bool zero); void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra); void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero); + size_t alignment, bool zero, bool try_tcache_dalloc); void huge_dalloc(void *ptr, bool unmap); size_t huge_salloc(const void *ptr); prof_ctx_t *huge_prof_ctx_get(const void *ptr); diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in index 268cd146fd4..475821acb61 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in @@ -270,6 +270,9 @@ static const bool config_ivsalloc = # ifdef __arm__ # define LG_QUANTUM 3 # endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif # ifdef __mips__ # define LG_QUANTUM 3 # endif @@ -424,6 +427,7 @@ static const bool config_ivsalloc = VALGRIND_FREELIKE_BLOCK(ptr, rzsize); \ } while (0) #else +#define RUNNING_ON_VALGRIND ((unsigned)0) #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) #define VALGRIND_FREELIKE_BLOCK(addr, rzB) @@ -510,13 +514,19 @@ extern size_t opt_narenas; /* Number of CPUs. */ extern unsigned ncpus; -extern malloc_mutex_t arenas_lock; /* Protects arenas initialization. */ +/* Protects arenas initialization (arenas, arenas_total). */ +extern malloc_mutex_t arenas_lock; /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. + * + * arenas[0..narenas_auto) are used for automatic multiplexing of threads and + * arenas. arenas[narenas_auto..narenas_total) are only used if the application + * takes some action to create them and allocate from them. */ extern arena_t **arenas; -extern unsigned narenas; +extern unsigned narenas_total; +extern unsigned narenas_auto; /* Read-only after initialization. */ arena_t *arenas_extend(unsigned ind); void arenas_cleanup(void *arg); @@ -571,6 +581,7 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), arenas, arena_t *) size_t s2u(size_t size); size_t sa2u(size_t size, size_t alignment); +unsigned narenas_total_get(void); arena_t *choose_arena(arena_t *arena); #endif @@ -675,6 +686,18 @@ sa2u(size_t size, size_t alignment) } } +JEMALLOC_INLINE unsigned +narenas_total_get(void) +{ + unsigned narenas; + + malloc_mutex_lock(&arenas_lock); + narenas = narenas_total; + malloc_mutex_unlock(&arenas_lock); + + return (narenas); +} + /* Choose an arena based on a per-thread value. */ JEMALLOC_INLINE arena_t * choose_arena(arena_t *arena) @@ -710,15 +733,24 @@ choose_arena(arena_t *arena) #include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE +void *imallocx(size_t size, bool try_tcache, arena_t *arena); void *imalloc(size_t size); +void *icallocx(size_t size, bool try_tcache, arena_t *arena); void *icalloc(size_t size); +void *ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena); void *ipalloc(size_t usize, size_t alignment, bool zero); size_t isalloc(const void *ptr, bool demote); size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); +void idallocx(void *ptr, bool try_tcache); void idalloc(void *ptr); +void iqallocx(void *ptr, bool try_tcache); void iqalloc(void *ptr); +void *irallocx(void *ptr, size_t size, size_t extra, size_t alignment, + bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena); void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, bool no_move); malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) @@ -726,29 +758,44 @@ malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) JEMALLOC_INLINE void * -imalloc(size_t size) +imallocx(size_t size, bool try_tcache, arena_t *arena) { assert(size != 0); if (size <= arena_maxclass) - return (arena_malloc(NULL, size, false, true)); + return (arena_malloc(arena, size, false, try_tcache)); else return (huge_malloc(size, false)); } JEMALLOC_INLINE void * -icalloc(size_t size) +imalloc(size_t size) +{ + + return (imallocx(size, true, NULL)); +} + +JEMALLOC_INLINE void * +icallocx(size_t size, bool try_tcache, arena_t *arena) { if (size <= arena_maxclass) - return (arena_malloc(NULL, size, true, true)); + return (arena_malloc(arena, size, true, try_tcache)); else return (huge_malloc(size, true)); } JEMALLOC_INLINE void * -ipalloc(size_t usize, size_t alignment, bool zero) +icalloc(size_t size) +{ + + return (icallocx(size, true, NULL)); +} + +JEMALLOC_INLINE void * +ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) { void *ret; @@ -756,11 +803,11 @@ ipalloc(size_t usize, size_t alignment, bool zero) assert(usize == sa2u(usize, alignment)); if (usize <= arena_maxclass && alignment <= PAGE) - ret = arena_malloc(NULL, usize, zero, true); + ret = arena_malloc(arena, usize, zero, try_tcache); else { if (usize <= arena_maxclass) { - ret = arena_palloc(choose_arena(NULL), usize, alignment, - zero); + ret = arena_palloc(choose_arena(arena), usize, + alignment, zero); } else if (alignment <= chunksize) ret = huge_malloc(usize, zero); else @@ -771,6 +818,13 @@ ipalloc(size_t usize, size_t alignment, bool zero) return (ret); } +JEMALLOC_INLINE void * +ipalloc(size_t usize, size_t alignment, bool zero) +{ + + return (ipallocx(usize, alignment, zero, true, NULL)); +} + /* * Typical usage: * void *ptr = [...] @@ -829,7 +883,7 @@ p2rz(const void *ptr) } JEMALLOC_INLINE void -idalloc(void *ptr) +idallocx(void *ptr, bool try_tcache) { arena_chunk_t *chunk; @@ -837,24 +891,38 @@ idalloc(void *ptr) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) - arena_dalloc(chunk->arena, chunk, ptr, true); + arena_dalloc(chunk->arena, chunk, ptr, try_tcache); else huge_dalloc(ptr, true); } JEMALLOC_INLINE void -iqalloc(void *ptr) +idalloc(void *ptr) +{ + + idallocx(ptr, true); +} + +JEMALLOC_INLINE void +iqallocx(void *ptr, bool try_tcache) { if (config_fill && opt_quarantine) quarantine(ptr); else - idalloc(ptr); + idallocx(ptr, try_tcache); +} + +JEMALLOC_INLINE void +iqalloc(void *ptr) +{ + + iqallocx(ptr, true); } JEMALLOC_INLINE void * -iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, - bool no_move) +irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, + bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) { void *ret; size_t oldsize; @@ -877,7 +945,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - ret = ipalloc(usize, alignment, zero); + ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); if (ret == NULL) { if (extra == 0) return (NULL); @@ -885,7 +953,8 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, usize = sa2u(size, alignment); if (usize == 0) return (NULL); - ret = ipalloc(usize, alignment, zero); + ret = ipallocx(usize, alignment, zero, try_tcache_alloc, + arena); if (ret == NULL) return (NULL); } @@ -896,7 +965,7 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, */ copysize = (size < oldsize) ? size : oldsize; memcpy(ret, ptr, copysize); - iqalloc(ptr); + iqallocx(ptr, try_tcache_dalloc); return (ret); } @@ -910,15 +979,25 @@ iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, } } else { if (size + extra <= arena_maxclass) { - return (arena_ralloc(ptr, oldsize, size, extra, - alignment, zero, true)); + return (arena_ralloc(arena, ptr, oldsize, size, extra, + alignment, zero, try_tcache_alloc, + try_tcache_dalloc)); } else { return (huge_ralloc(ptr, oldsize, size, extra, - alignment, zero)); + alignment, zero, try_tcache_dalloc)); } } } +JEMALLOC_INLINE void * +iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, + bool no_move) +{ + + return (irallocx(ptr, size, extra, alignment, zero, no_move, true, true, + NULL)); +} + malloc_tsd_externs(thread_allocated, thread_allocated_t) malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t, THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup) diff --git a/deps/jemalloc/include/jemalloc/internal/private_namespace.h b/deps/jemalloc/include/jemalloc/internal/private_namespace.h index b8166470dc3..06241cd2fa1 100644 --- a/deps/jemalloc/include/jemalloc/internal/private_namespace.h +++ b/deps/jemalloc/include/jemalloc/internal/private_namespace.h @@ -12,6 +12,8 @@ #define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) #define arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked) #define arena_dalloc_small JEMALLOC_N(arena_dalloc_small) +#define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) +#define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) #define arena_malloc JEMALLOC_N(arena_malloc) #define arena_malloc_large JEMALLOC_N(arena_malloc_large) #define arena_malloc_small JEMALLOC_N(arena_malloc_small) @@ -51,14 +53,13 @@ #define arena_stats_merge JEMALLOC_N(arena_stats_merge) #define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) #define arenas JEMALLOC_N(arenas) -#define arenas_bin_i_index JEMALLOC_N(arenas_bin_i_index) #define arenas_booted JEMALLOC_N(arenas_booted) #define arenas_cleanup JEMALLOC_N(arenas_cleanup) #define arenas_extend JEMALLOC_N(arenas_extend) #define arenas_initialized JEMALLOC_N(arenas_initialized) #define arenas_lock JEMALLOC_N(arenas_lock) -#define arenas_lrun_i_index JEMALLOC_N(arenas_lrun_i_index) #define arenas_tls JEMALLOC_N(arenas_tls) +#define arenas_tsd JEMALLOC_N(arenas_tsd) #define arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot) #define arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper) #define arenas_tsd_get JEMALLOC_N(arenas_tsd_get) @@ -101,9 +102,15 @@ #define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) #define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child) #define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent) +#define chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get) +#define chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set) #define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork) #define chunk_in_dss JEMALLOC_N(chunk_in_dss) #define chunk_npages JEMALLOC_N(chunk_npages) +#define chunk_postfork_child JEMALLOC_N(chunk_postfork_child) +#define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent) +#define chunk_prefork JEMALLOC_N(chunk_prefork) +#define chunk_unmap JEMALLOC_N(chunk_unmap) #define chunks_mtx JEMALLOC_N(chunks_mtx) #define chunks_rtree JEMALLOC_N(chunks_rtree) #define chunksize JEMALLOC_N(chunksize) @@ -129,6 +136,10 @@ #define ctl_bymib JEMALLOC_N(ctl_bymib) #define ctl_byname JEMALLOC_N(ctl_byname) #define ctl_nametomib JEMALLOC_N(ctl_nametomib) +#define ctl_postfork_child JEMALLOC_N(ctl_postfork_child) +#define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent) +#define ctl_prefork JEMALLOC_N(ctl_prefork) +#define dss_prec_names JEMALLOC_N(dss_prec_names) #define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) #define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) #define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) @@ -161,6 +172,7 @@ #define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) #define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) #define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) +#define get_errno JEMALLOC_N(get_errno) #define hash JEMALLOC_N(hash) #define huge_allocated JEMALLOC_N(huge_allocated) #define huge_boot JEMALLOC_N(huge_boot) @@ -180,11 +192,17 @@ #define huge_salloc JEMALLOC_N(huge_salloc) #define iallocm JEMALLOC_N(iallocm) #define icalloc JEMALLOC_N(icalloc) +#define icallocx JEMALLOC_N(icallocx) #define idalloc JEMALLOC_N(idalloc) +#define idallocx JEMALLOC_N(idallocx) #define imalloc JEMALLOC_N(imalloc) +#define imallocx JEMALLOC_N(imallocx) #define ipalloc JEMALLOC_N(ipalloc) +#define ipallocx JEMALLOC_N(ipallocx) #define iqalloc JEMALLOC_N(iqalloc) +#define iqallocx JEMALLOC_N(iqallocx) #define iralloc JEMALLOC_N(iralloc) +#define irallocx JEMALLOC_N(irallocx) #define isalloc JEMALLOC_N(isalloc) #define isthreaded JEMALLOC_N(isthreaded) #define ivsalloc JEMALLOC_N(ivsalloc) @@ -212,7 +230,9 @@ #define map_bias JEMALLOC_N(map_bias) #define mb_write JEMALLOC_N(mb_write) #define mutex_boot JEMALLOC_N(mutex_boot) -#define narenas JEMALLOC_N(narenas) +#define narenas_auto JEMALLOC_N(narenas_auto) +#define narenas_total JEMALLOC_N(narenas_total) +#define narenas_total_get JEMALLOC_N(narenas_total_get) #define ncpus JEMALLOC_N(ncpus) #define nhbins JEMALLOC_N(nhbins) #define opt_abort JEMALLOC_N(opt_abort) @@ -254,6 +274,9 @@ #define prof_lookup JEMALLOC_N(prof_lookup) #define prof_malloc JEMALLOC_N(prof_malloc) #define prof_mdump JEMALLOC_N(prof_mdump) +#define prof_postfork_child JEMALLOC_N(prof_postfork_child) +#define prof_postfork_parent JEMALLOC_N(prof_postfork_parent) +#define prof_prefork JEMALLOC_N(prof_prefork) #define prof_promote JEMALLOC_N(prof_promote) #define prof_realloc JEMALLOC_N(prof_realloc) #define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) @@ -264,6 +287,7 @@ #define prof_tdata_init JEMALLOC_N(prof_tdata_init) #define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized) #define prof_tdata_tls JEMALLOC_N(prof_tdata_tls) +#define prof_tdata_tsd JEMALLOC_N(prof_tdata_tsd) #define prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot) #define prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper) #define prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get) @@ -278,12 +302,13 @@ #define rtree_get JEMALLOC_N(rtree_get) #define rtree_get_locked JEMALLOC_N(rtree_get_locked) #define rtree_new JEMALLOC_N(rtree_new) +#define rtree_postfork_child JEMALLOC_N(rtree_postfork_child) +#define rtree_postfork_parent JEMALLOC_N(rtree_postfork_parent) +#define rtree_prefork JEMALLOC_N(rtree_prefork) #define rtree_set JEMALLOC_N(rtree_set) #define s2u JEMALLOC_N(s2u) #define sa2u JEMALLOC_N(sa2u) -#define stats_arenas_i_bins_j_index JEMALLOC_N(stats_arenas_i_bins_j_index) -#define stats_arenas_i_index JEMALLOC_N(stats_arenas_i_index) -#define stats_arenas_i_lruns_j_index JEMALLOC_N(stats_arenas_i_lruns_j_index) +#define set_errno JEMALLOC_N(set_errno) #define stats_cactive JEMALLOC_N(stats_cactive) #define stats_cactive_add JEMALLOC_N(stats_cactive_add) #define stats_cactive_get JEMALLOC_N(stats_cactive_get) @@ -311,6 +336,7 @@ #define tcache_enabled_initialized JEMALLOC_N(tcache_enabled_initialized) #define tcache_enabled_set JEMALLOC_N(tcache_enabled_set) #define tcache_enabled_tls JEMALLOC_N(tcache_enabled_tls) +#define tcache_enabled_tsd JEMALLOC_N(tcache_enabled_tsd) #define tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot) #define tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper) #define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get) @@ -325,6 +351,7 @@ #define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) #define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup) #define tcache_tls JEMALLOC_N(tcache_tls) +#define tcache_tsd JEMALLOC_N(tcache_tsd) #define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot) #define tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper) #define tcache_tsd_get JEMALLOC_N(tcache_tsd_get) @@ -332,6 +359,7 @@ #define thread_allocated_booted JEMALLOC_N(thread_allocated_booted) #define thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized) #define thread_allocated_tls JEMALLOC_N(thread_allocated_tls) +#define thread_allocated_tsd JEMALLOC_N(thread_allocated_tsd) #define thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot) #define thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper) #define thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get) diff --git a/deps/jemalloc/include/jemalloc/internal/prof.h b/deps/jemalloc/include/jemalloc/internal/prof.h index c3e3f9e4bcd..47f22ad2da1 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof.h +++ b/deps/jemalloc/include/jemalloc/internal/prof.h @@ -223,6 +223,9 @@ void prof_tdata_cleanup(void *arg); void prof_boot0(void); void prof_boot1(void); bool prof_boot2(void); +void prof_prefork(void); +void prof_postfork_parent(void); +void prof_postfork_child(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ @@ -506,7 +509,7 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, if ((uintptr_t)cnt > (uintptr_t)1U) { prof_ctx_set(ptr, cnt->ctx); cnt->epoch++; - } else + } else if (ptr != NULL) prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); /*********/ mb_write(); diff --git a/deps/jemalloc/include/jemalloc/internal/rtree.h b/deps/jemalloc/include/jemalloc/internal/rtree.h index 95d6355a5f4..9bd98548cfe 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree.h @@ -36,6 +36,9 @@ struct rtree_s { #ifdef JEMALLOC_H_EXTERNS rtree_t *rtree_new(unsigned bits); +void rtree_prefork(rtree_t *rtree); +void rtree_postfork_parent(rtree_t *rtree); +void rtree_postfork_child(rtree_t *rtree); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/jemalloc.h.in b/deps/jemalloc/include/jemalloc/jemalloc.h.in index ad0694858da..31b1304a20a 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc.h.in @@ -25,6 +25,8 @@ extern "C" { #endif #define ALLOCM_ZERO ((int)0x40) #define ALLOCM_NO_MOVE ((int)0x80) +/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */ +#define ALLOCM_ARENA(a) ((int)(((a)+1) << 8)) #define ALLOCM_SUCCESS 0 #define ALLOCM_ERR_OOM 1 @@ -59,7 +61,8 @@ JEMALLOC_EXPORT void * je_memalign(size_t alignment, size_t size) JEMALLOC_EXPORT void * je_valloc(size_t size) JEMALLOC_ATTR(malloc); #endif -JEMALLOC_EXPORT size_t je_malloc_usable_size(const void *ptr); +JEMALLOC_EXPORT size_t je_malloc_usable_size( + JEMALLOC_USABLE_SIZE_CONST void *ptr); JEMALLOC_EXPORT void je_malloc_stats_print(void (*write_cb)(void *, const char *), void *je_cbopaque, const char *opts); JEMALLOC_EXPORT int je_mallctl(const char *name, void *oldp, diff --git a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in index c469142a5d4..1cd60254ac3 100644 --- a/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in +++ b/deps/jemalloc/include/jemalloc/jemalloc_defs.h.in @@ -221,6 +221,15 @@ #undef JEMALLOC_OVERRIDE_MEMALIGN #undef JEMALLOC_OVERRIDE_VALLOC +/* + * At least Linux omits the "const" in: + * + * size_t malloc_usable_size(const void *ptr); + * + * Match the operating system's prototype. + */ +#undef JEMALLOC_USABLE_SIZE_CONST + /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ diff --git a/deps/jemalloc/src/arena.c b/deps/jemalloc/src/arena.c index 2a6150f3e8d..0c53b071b87 100644 --- a/deps/jemalloc/src/arena.c +++ b/deps/jemalloc/src/arena.c @@ -40,6 +40,12 @@ const uint8_t small_size2bin[] = { /******************************************************************************/ /* Function prototypes for non-inline static functions. */ +static void arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, + size_t pageind, size_t npages, bool maybe_adjac_pred, + bool maybe_adjac_succ); +static void arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, + size_t pageind, size_t npages, bool maybe_adjac_pred, + bool maybe_adjac_succ); static void arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, size_t binind, bool zero); static arena_chunk_t *arena_chunk_alloc(arena_t *arena); @@ -48,8 +54,11 @@ static arena_run_t *arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, bool zero); static arena_run_t *arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, bool zero); +static arena_chunk_t *chunks_dirty_iter_cb(arena_chunk_tree_t *tree, + arena_chunk_t *chunk, void *arg); static void arena_purge(arena_t *arena, bool all); -static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty); +static void arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, + bool cleaned); static void arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, size_t oldsize, size_t newsize); static void arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, @@ -101,9 +110,6 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) size_t a_size = a->bits & ~PAGE_MASK; size_t b_size = b->bits & ~PAGE_MASK; - assert((a->bits & CHUNK_MAP_KEY) == CHUNK_MAP_KEY || (a->bits & - CHUNK_MAP_DIRTY) == (b->bits & CHUNK_MAP_DIRTY)); - ret = (a_size > b_size) - (a_size < b_size); if (ret == 0) { uintptr_t a_mapelm, b_mapelm; @@ -129,6 +135,182 @@ arena_avail_comp(arena_chunk_map_t *a, arena_chunk_map_t *b) rb_gen(static UNUSED, arena_avail_tree_, arena_avail_tree_t, arena_chunk_map_t, u.rb_link, arena_avail_comp) +static inline int +arena_chunk_dirty_comp(arena_chunk_t *a, arena_chunk_t *b) +{ + + assert(a != NULL); + assert(b != NULL); + + /* + * Short-circuit for self comparison. The following comparison code + * would come to the same result, but at the cost of executing the slow + * path. + */ + if (a == b) + return (0); + + /* + * Order such that chunks with higher fragmentation are "less than" + * those with lower fragmentation -- purging order is from "least" to + * "greatest". Fragmentation is measured as: + * + * mean current avail run size + * -------------------------------- + * mean defragmented avail run size + * + * navail + * ----------- + * nruns_avail nruns_avail-nruns_adjac + * = ========================= = ----------------------- + * navail nruns_avail + * ----------------------- + * nruns_avail-nruns_adjac + * + * The following code multiplies away the denominator prior to + * comparison, in order to avoid division. + * + */ + { + size_t a_val = (a->nruns_avail - a->nruns_adjac) * + b->nruns_avail; + size_t b_val = (b->nruns_avail - b->nruns_adjac) * + a->nruns_avail; + + if (a_val < b_val) + return (1); + if (a_val > b_val) + return (-1); + } + /* + * Break ties by chunk address. For fragmented chunks, report lower + * addresses as "lower", so that fragmentation reduction happens first + * at lower addresses. However, use the opposite ordering for + * unfragmented chunks, in order to increase the chances of + * re-allocating dirty runs. + */ + { + uintptr_t a_chunk = (uintptr_t)a; + uintptr_t b_chunk = (uintptr_t)b; + int ret = ((a_chunk > b_chunk) - (a_chunk < b_chunk)); + if (a->nruns_adjac == 0) { + assert(b->nruns_adjac == 0); + ret = -ret; + } + return (ret); + } +} + +/* Generate red-black tree functions. */ +rb_gen(static UNUSED, arena_chunk_dirty_, arena_chunk_tree_t, arena_chunk_t, + dirty_link, arena_chunk_dirty_comp) + +static inline bool +arena_avail_adjac_pred(arena_chunk_t *chunk, size_t pageind) +{ + bool ret; + + if (pageind-1 < map_bias) + ret = false; + else { + ret = (arena_mapbits_allocated_get(chunk, pageind-1) == 0); + assert(ret == false || arena_mapbits_dirty_get(chunk, + pageind-1) != arena_mapbits_dirty_get(chunk, pageind)); + } + return (ret); +} + +static inline bool +arena_avail_adjac_succ(arena_chunk_t *chunk, size_t pageind, size_t npages) +{ + bool ret; + + if (pageind+npages == chunk_npages) + ret = false; + else { + assert(pageind+npages < chunk_npages); + ret = (arena_mapbits_allocated_get(chunk, pageind+npages) == 0); + assert(ret == false || arena_mapbits_dirty_get(chunk, pageind) + != arena_mapbits_dirty_get(chunk, pageind+npages)); + } + return (ret); +} + +static inline bool +arena_avail_adjac(arena_chunk_t *chunk, size_t pageind, size_t npages) +{ + + return (arena_avail_adjac_pred(chunk, pageind) || + arena_avail_adjac_succ(chunk, pageind, npages)); +} + +static void +arena_avail_insert(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) +{ + + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + + /* + * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be + * removed and reinserted even if the run to be inserted is clean. + */ + if (chunk->ndirty != 0) + arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); + + if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) + chunk->nruns_adjac++; + if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) + chunk->nruns_adjac++; + chunk->nruns_avail++; + assert(chunk->nruns_avail > chunk->nruns_adjac); + + if (arena_mapbits_dirty_get(chunk, pageind) != 0) { + arena->ndirty += npages; + chunk->ndirty += npages; + } + if (chunk->ndirty != 0) + arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); + + arena_avail_tree_insert(&arena->runs_avail, arena_mapp_get(chunk, + pageind)); +} + +static void +arena_avail_remove(arena_t *arena, arena_chunk_t *chunk, size_t pageind, + size_t npages, bool maybe_adjac_pred, bool maybe_adjac_succ) +{ + + assert(npages == (arena_mapbits_unallocated_size_get(chunk, pageind) >> + LG_PAGE)); + + /* + * chunks_dirty is keyed by nruns_{avail,adjac}, so the chunk must be + * removed and reinserted even if the run to be removed is clean. + */ + if (chunk->ndirty != 0) + arena_chunk_dirty_remove(&arena->chunks_dirty, chunk); + + if (maybe_adjac_pred && arena_avail_adjac_pred(chunk, pageind)) + chunk->nruns_adjac--; + if (maybe_adjac_succ && arena_avail_adjac_succ(chunk, pageind, npages)) + chunk->nruns_adjac--; + chunk->nruns_avail--; + assert(chunk->nruns_avail > chunk->nruns_adjac || (chunk->nruns_avail + == 0 && chunk->nruns_adjac == 0)); + + if (arena_mapbits_dirty_get(chunk, pageind) != 0) { + arena->ndirty -= npages; + chunk->ndirty -= npages; + } + if (chunk->ndirty != 0) + arena_chunk_dirty_insert(&arena->chunks_dirty, chunk); + + arena_avail_tree_remove(&arena->runs_avail, arena_mapp_get(chunk, + pageind)); +} + static inline void * arena_run_reg_alloc(arena_run_t *run, arena_bin_info_t *bin_info) { @@ -193,7 +375,6 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, arena_chunk_t *chunk; size_t run_ind, total_pages, need_pages, rem_pages, i; size_t flag_dirty; - arena_avail_tree_t *runs_avail; assert((large && binind == BININD_INVALID) || (large == false && binind != BININD_INVALID)); @@ -201,8 +382,6 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); - runs_avail = (flag_dirty != 0) ? &arena->runs_avail_dirty : - &arena->runs_avail_clean; total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> LG_PAGE; assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == @@ -212,7 +391,7 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; - arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, run_ind)); + arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); if (config_stats) { /* * Update stats_cactive if nactive is crossing a chunk @@ -244,14 +423,8 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, arena_mapbits_unzeroed_get(chunk, run_ind+total_pages-1)); } - arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, - run_ind+need_pages)); - } - - /* Update dirty page accounting. */ - if (flag_dirty != 0) { - chunk->ndirty -= need_pages; - arena->ndirty -= need_pages; + arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, + false, true); } /* @@ -344,8 +517,6 @@ arena_chunk_alloc(arena_t *arena) size_t i; if (arena->spare != NULL) { - arena_avail_tree_t *runs_avail; - chunk = arena->spare; arena->spare = NULL; @@ -357,14 +528,6 @@ arena_chunk_alloc(arena_t *arena) chunk_npages-1) == arena_maxclass); assert(arena_mapbits_dirty_get(chunk, map_bias) == arena_mapbits_dirty_get(chunk, chunk_npages-1)); - - /* Insert the run into the appropriate runs_avail_* tree. */ - if (arena_mapbits_dirty_get(chunk, map_bias) == 0) - runs_avail = &arena->runs_avail_clean; - else - runs_avail = &arena->runs_avail_dirty; - arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, - map_bias)); } else { bool zero; size_t unzeroed; @@ -372,7 +535,7 @@ arena_chunk_alloc(arena_t *arena) zero = false; malloc_mutex_unlock(&arena->lock); chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, - false, &zero); + false, &zero, arena->dss_prec); malloc_mutex_lock(&arena->lock); if (chunk == NULL) return (NULL); @@ -380,8 +543,6 @@ arena_chunk_alloc(arena_t *arena) arena->stats.mapped += chunksize; chunk->arena = arena; - ql_elm_new(chunk, link_dirty); - chunk->dirtied = false; /* * Claim that no pages are in use, since the header is merely @@ -389,6 +550,9 @@ arena_chunk_alloc(arena_t *arena) */ chunk->ndirty = 0; + chunk->nruns_avail = 0; + chunk->nruns_adjac = 0; + /* * Initialize the map to contain one maximal free untouched run. * Mark the pages as zeroed iff chunk_alloc() returned a zeroed @@ -412,20 +576,18 @@ arena_chunk_alloc(arena_t *arena) } arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, unzeroed); - - /* Insert the run into the runs_avail_clean tree. */ - arena_avail_tree_insert(&arena->runs_avail_clean, - arena_mapp_get(chunk, map_bias)); } + /* Insert the run into the runs_avail tree. */ + arena_avail_insert(arena, chunk, map_bias, chunk_npages-map_bias, + false, false); + return (chunk); } static void arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) { - arena_avail_tree_t *runs_avail; - assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == @@ -436,24 +598,16 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) arena_mapbits_dirty_get(chunk, chunk_npages-1)); /* - * Remove run from the appropriate runs_avail_* tree, so that the arena - * does not use it. + * Remove run from the runs_avail tree, so that the arena does not use + * it. */ - if (arena_mapbits_dirty_get(chunk, map_bias) == 0) - runs_avail = &arena->runs_avail_clean; - else - runs_avail = &arena->runs_avail_dirty; - arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, map_bias)); + arena_avail_remove(arena, chunk, map_bias, chunk_npages-map_bias, + false, false); if (arena->spare != NULL) { arena_chunk_t *spare = arena->spare; arena->spare = chunk; - if (spare->dirtied) { - ql_remove(&chunk->arena->chunks_dirty, spare, - link_dirty); - arena->ndirty -= spare->ndirty; - } malloc_mutex_unlock(&arena->lock); chunk_dealloc((void *)spare, chunksize, true); malloc_mutex_lock(&arena->lock); @@ -471,19 +625,7 @@ arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, arena_chunk_map_t *mapelm, key; key.bits = size | CHUNK_MAP_KEY; - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_dirty, &key); - if (mapelm != NULL) { - arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); - size_t pageind = (((uintptr_t)mapelm - - (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) - + map_bias; - - run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << - LG_PAGE)); - arena_run_split(arena, run, size, large, binind, zero); - return (run); - } - mapelm = arena_avail_tree_nsearch(&arena->runs_avail_clean, &key); + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); if (mapelm != NULL) { arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); size_t pageind = (((uintptr_t)mapelm - @@ -537,41 +679,40 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, static inline void arena_maybe_purge(arena_t *arena) { + size_t npurgeable, threshold; + + /* Don't purge if the option is disabled. */ + if (opt_lg_dirty_mult < 0) + return; + /* Don't purge if all dirty pages are already being purged. */ + if (arena->ndirty <= arena->npurgatory) + return; + npurgeable = arena->ndirty - arena->npurgatory; + threshold = (arena->nactive >> opt_lg_dirty_mult); + /* + * Don't purge unless the number of purgeable pages exceeds the + * threshold. + */ + if (npurgeable <= threshold) + return; - /* Enforce opt_lg_dirty_mult. */ - if (opt_lg_dirty_mult >= 0 && arena->ndirty > arena->npurgatory && - (arena->ndirty - arena->npurgatory) > chunk_npages && - (arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - - arena->npurgatory)) - arena_purge(arena, false); + arena_purge(arena, false); } -static inline void -arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) +static inline size_t +arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) { + size_t npurged; ql_head(arena_chunk_map_t) mapelms; arena_chunk_map_t *mapelm; - size_t pageind, flag_unzeroed; - size_t ndirty; + size_t pageind, npages; size_t nmadvise; ql_new(&mapelms); - flag_unzeroed = -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - /* - * madvise(..., MADV_DONTNEED) results in zero-filled pages for anonymous - * mappings, but not for file-backed mappings. - */ - 0 -#else - CHUNK_MAP_UNZEROED -#endif - ; - /* * If chunk is the spare, temporarily re-allocate it, 1) so that its - * run is reinserted into runs_avail_dirty, and 2) so that it cannot be + * run is reinserted into runs_avail, and 2) so that it cannot be * completely discarded by another thread while arena->lock is dropped * by this thread. Note that the arena_run_dalloc() call will * implicitly deallocate the chunk, so no explicit action is required @@ -591,68 +732,50 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) arena_chunk_alloc(arena); } - /* Temporarily allocate all free dirty runs within chunk. */ - for (pageind = map_bias; pageind < chunk_npages;) { + if (config_stats) + arena->stats.purged += chunk->ndirty; + + /* + * Operate on all dirty runs if there is no clean/dirty run + * fragmentation. + */ + if (chunk->nruns_adjac == 0) + all = true; + + /* + * Temporarily allocate free dirty runs within chunk. If all is false, + * only operate on dirty runs that are fragments; otherwise operate on + * all dirty runs. + */ + for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { mapelm = arena_mapp_get(chunk, pageind); if (arena_mapbits_allocated_get(chunk, pageind) == 0) { - size_t npages; + size_t run_size = + arena_mapbits_unallocated_size_get(chunk, pageind); - npages = arena_mapbits_unallocated_size_get(chunk, - pageind) >> LG_PAGE; + npages = run_size >> LG_PAGE; assert(pageind + npages <= chunk_npages); assert(arena_mapbits_dirty_get(chunk, pageind) == arena_mapbits_dirty_get(chunk, pageind+npages-1)); - if (arena_mapbits_dirty_get(chunk, pageind) != 0) { - size_t i; - - arena_avail_tree_remove( - &arena->runs_avail_dirty, mapelm); - arena_mapbits_unzeroed_set(chunk, pageind, - flag_unzeroed); - arena_mapbits_large_set(chunk, pageind, - (npages << LG_PAGE), 0); - /* - * Update internal elements in the page map, so - * that CHUNK_MAP_UNZEROED is properly set. - */ - for (i = 1; i < npages - 1; i++) { - arena_mapbits_unzeroed_set(chunk, - pageind+i, flag_unzeroed); - } - if (npages > 1) { - arena_mapbits_unzeroed_set(chunk, - pageind+npages-1, flag_unzeroed); - arena_mapbits_large_set(chunk, - pageind+npages-1, 0, 0); - } + if (arena_mapbits_dirty_get(chunk, pageind) != 0 && + (all || arena_avail_adjac(chunk, pageind, + npages))) { + arena_run_t *run = (arena_run_t *)((uintptr_t) + chunk + (uintptr_t)(pageind << LG_PAGE)); - if (config_stats) { - /* - * Update stats_cactive if nactive is - * crossing a chunk multiple. - */ - size_t cactive_diff = - CHUNK_CEILING((arena->nactive + - npages) << LG_PAGE) - - CHUNK_CEILING(arena->nactive << - LG_PAGE); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); - } - arena->nactive += npages; + arena_run_split(arena, run, run_size, true, + BININD_INVALID, false); /* Append to list for later processing. */ ql_elm_new(mapelm, u.ql_link); ql_tail_insert(&mapelms, mapelm, u.ql_link); } - - pageind += npages; } else { - /* Skip allocated run. */ - if (arena_mapbits_large_get(chunk, pageind)) - pageind += arena_mapbits_large_size_get(chunk, + /* Skip run. */ + if (arena_mapbits_large_get(chunk, pageind) != 0) { + npages = arena_mapbits_large_size_get(chunk, pageind) >> LG_PAGE; - else { + } else { size_t binind; arena_bin_info_t *bin_info; arena_run_t *run = (arena_run_t *)((uintptr_t) @@ -662,41 +785,48 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) pageind) == 0); binind = arena_bin_index(arena, run->bin); bin_info = &arena_bin_info[binind]; - pageind += bin_info->run_size >> LG_PAGE; + npages = bin_info->run_size >> LG_PAGE; } } } assert(pageind == chunk_npages); - - if (config_debug) - ndirty = chunk->ndirty; - if (config_stats) - arena->stats.purged += chunk->ndirty; - arena->ndirty -= chunk->ndirty; - chunk->ndirty = 0; - ql_remove(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = false; + assert(chunk->ndirty == 0 || all == false); + assert(chunk->nruns_adjac == 0); malloc_mutex_unlock(&arena->lock); if (config_stats) nmadvise = 0; + npurged = 0; ql_foreach(mapelm, &mapelms, u.ql_link) { - size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / + bool unzeroed; + size_t flag_unzeroed, i; + + pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t)) + map_bias; - size_t npages = arena_mapbits_large_size_get(chunk, pageind) >> + npages = arena_mapbits_large_size_get(chunk, pageind) >> LG_PAGE; - assert(pageind + npages <= chunk_npages); - assert(ndirty >= npages); - if (config_debug) - ndirty -= npages; - - pages_purge((void *)((uintptr_t)chunk + (pageind << LG_PAGE)), - (npages << LG_PAGE)); + unzeroed = pages_purge((void *)((uintptr_t)chunk + (pageind << + LG_PAGE)), (npages << LG_PAGE)); + flag_unzeroed = unzeroed ? CHUNK_MAP_UNZEROED : 0; + /* + * Set the unzeroed flag for all pages, now that pages_purge() + * has returned whether the pages were zeroed as a side effect + * of purging. This chunk map modification is safe even though + * the arena mutex isn't currently owned by this thread, + * because the run is marked as allocated, thus protecting it + * from being modified by any other thread. As long as these + * writes don't perturb the first and last elements' + * CHUNK_MAP_ALLOCATED bits, behavior is well defined. + */ + for (i = 0; i < npages; i++) { + arena_mapbits_unzeroed_set(chunk, pageind+i, + flag_unzeroed); + } + npurged += npages; if (config_stats) nmadvise++; } - assert(ndirty == 0); malloc_mutex_lock(&arena->lock); if (config_stats) arena->stats.nmadvise += nmadvise; @@ -704,14 +834,27 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk) /* Deallocate runs. */ for (mapelm = ql_first(&mapelms); mapelm != NULL; mapelm = ql_first(&mapelms)) { - size_t pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / - sizeof(arena_chunk_map_t)) + map_bias; - arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + - (uintptr_t)(pageind << LG_PAGE)); + arena_run_t *run; + pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / + sizeof(arena_chunk_map_t)) + map_bias; + run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << + LG_PAGE)); ql_remove(&mapelms, mapelm, u.ql_link); - arena_run_dalloc(arena, run, false); + arena_run_dalloc(arena, run, false, true); } + + return (npurged); +} + +static arena_chunk_t * +chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) +{ + size_t *ndirty = (size_t *)arg; + + assert(chunk->ndirty != 0); + *ndirty += chunk->ndirty; + return (NULL); } static void @@ -722,14 +865,11 @@ arena_purge(arena_t *arena, bool all) if (config_debug) { size_t ndirty = 0; - ql_foreach(chunk, &arena->chunks_dirty, link_dirty) { - assert(chunk->dirtied); - ndirty += chunk->ndirty; - } + arena_chunk_dirty_iter(&arena->chunks_dirty, NULL, + chunks_dirty_iter_cb, (void *)&ndirty); assert(ndirty == arena->ndirty); } assert(arena->ndirty > arena->npurgatory || all); - assert(arena->ndirty - arena->npurgatory > chunk_npages || all); assert((arena->nactive >> opt_lg_dirty_mult) < (arena->ndirty - arena->npurgatory) || all); @@ -741,16 +881,24 @@ arena_purge(arena_t *arena, bool all) * purge, and add the result to arena->npurgatory. This will keep * multiple threads from racing to reduce ndirty below the threshold. */ - npurgatory = arena->ndirty - arena->npurgatory; - if (all == false) { - assert(npurgatory >= arena->nactive >> opt_lg_dirty_mult); - npurgatory -= arena->nactive >> opt_lg_dirty_mult; + { + size_t npurgeable = arena->ndirty - arena->npurgatory; + + if (all == false) { + size_t threshold = (arena->nactive >> + opt_lg_dirty_mult); + + npurgatory = npurgeable - threshold; + } else + npurgatory = npurgeable; } arena->npurgatory += npurgatory; while (npurgatory > 0) { + size_t npurgeable, npurged, nunpurged; + /* Get next chunk with dirty pages. */ - chunk = ql_first(&arena->chunks_dirty); + chunk = arena_chunk_dirty_first(&arena->chunks_dirty); if (chunk == NULL) { /* * This thread was unable to purge as many pages as @@ -761,23 +909,15 @@ arena_purge(arena_t *arena, bool all) arena->npurgatory -= npurgatory; return; } - while (chunk->ndirty == 0) { - ql_remove(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = false; - chunk = ql_first(&arena->chunks_dirty); - if (chunk == NULL) { - /* Same logic as for above. */ - arena->npurgatory -= npurgatory; - return; - } - } + npurgeable = chunk->ndirty; + assert(npurgeable != 0); - if (chunk->ndirty > npurgatory) { + if (npurgeable > npurgatory && chunk->nruns_adjac == 0) { /* - * This thread will, at a minimum, purge all the dirty - * pages in chunk, so set npurgatory to reflect this - * thread's commitment to purge the pages. This tends - * to reduce the chances of the following scenario: + * This thread will purge all the dirty pages in chunk, + * so set npurgatory to reflect this thread's intent to + * purge the pages. This tends to reduce the chances + * of the following scenario: * * 1) This thread sets arena->npurgatory such that * (arena->ndirty - arena->npurgatory) is at the @@ -791,13 +931,20 @@ arena_purge(arena_t *arena, bool all) * because all of the purging work being done really * needs to happen. */ - arena->npurgatory += chunk->ndirty - npurgatory; - npurgatory = chunk->ndirty; + arena->npurgatory += npurgeable - npurgatory; + npurgatory = npurgeable; } - arena->npurgatory -= chunk->ndirty; - npurgatory -= chunk->ndirty; - arena_chunk_purge(arena, chunk); + /* + * Keep track of how many pages are purgeable, versus how many + * actually get purged, and adjust counters accordingly. + */ + arena->npurgatory -= npurgeable; + npurgatory -= npurgeable; + npurged = arena_chunk_purge(arena, chunk, all); + nunpurged = npurgeable - npurged; + arena->npurgatory += nunpurged; + npurgatory += nunpurged; } } @@ -811,11 +958,10 @@ arena_purge_all(arena_t *arena) } static void -arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) +arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) { arena_chunk_t *chunk; size_t size, run_ind, run_pages, flag_dirty; - arena_avail_tree_t *runs_avail; chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); @@ -846,15 +992,14 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) /* * The run is dirty if the caller claims to have dirtied it, as well as - * if it was already dirty before being allocated. + * if it was already dirty before being allocated and the caller + * doesn't claim to have cleaned it. */ assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - if (arena_mapbits_dirty_get(chunk, run_ind) != 0) + if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) dirty = true; flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; - runs_avail = dirty ? &arena->runs_avail_dirty : - &arena->runs_avail_clean; /* Mark pages as unallocated in the chunk map. */ if (dirty) { @@ -862,9 +1007,6 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) CHUNK_MAP_DIRTY); arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, CHUNK_MAP_DIRTY); - - chunk->ndirty += run_pages; - arena->ndirty += run_pages; } else { arena_mapbits_unallocated_set(chunk, run_ind, size, arena_mapbits_unzeroed_get(chunk, run_ind)); @@ -888,8 +1030,8 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) run_ind+run_pages+nrun_pages-1) == nrun_size); assert(arena_mapbits_dirty_get(chunk, run_ind+run_pages+nrun_pages-1) == flag_dirty); - arena_avail_tree_remove(runs_avail, - arena_mapp_get(chunk, run_ind+run_pages)); + arena_avail_remove(arena, chunk, run_ind+run_pages, nrun_pages, + false, true); size += nrun_size; run_pages += nrun_pages; @@ -915,8 +1057,8 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == prun_size); assert(arena_mapbits_dirty_get(chunk, run_ind) == flag_dirty); - arena_avail_tree_remove(runs_avail, arena_mapp_get(chunk, - run_ind)); + arena_avail_remove(arena, chunk, run_ind, prun_pages, true, + false); size += prun_size; run_pages += prun_pages; @@ -931,19 +1073,7 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty) arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); assert(arena_mapbits_dirty_get(chunk, run_ind) == arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - arena_avail_tree_insert(runs_avail, arena_mapp_get(chunk, run_ind)); - - if (dirty) { - /* - * Insert into chunks_dirty before potentially calling - * arena_chunk_dealloc(), so that chunks_dirty and - * arena->ndirty are consistent. - */ - if (chunk->dirtied == false) { - ql_tail_insert(&arena->chunks_dirty, chunk, link_dirty); - chunk->dirtied = true; - } - } + arena_avail_insert(arena, chunk, run_ind, run_pages, true, true); /* Deallocate chunk if it is now completely unused. */ if (size == arena_maxclass) { @@ -992,7 +1122,7 @@ arena_run_trim_head(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, arena_mapbits_large_set(chunk, pageind+head_npages, newsize, flag_dirty); - arena_run_dalloc(arena, run, false); + arena_run_dalloc(arena, run, false, false); } static void @@ -1025,7 +1155,7 @@ arena_run_trim_tail(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, flag_dirty); arena_run_dalloc(arena, (arena_run_t *)((uintptr_t)run + newsize), - dirty); + dirty, false); } static arena_run_t * @@ -1536,7 +1666,7 @@ arena_dalloc_bin_run(arena_t *arena, arena_chunk_t *chunk, arena_run_t *run, ((past - run_ind) << LG_PAGE), false); /* npages = past - run_ind; */ } - arena_run_dalloc(arena, run, true); + arena_run_dalloc(arena, run, true, false); malloc_mutex_unlock(&arena->lock); /****************************/ malloc_mutex_lock(&bin->lock); @@ -1629,52 +1759,6 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, mapelm = arena_mapp_get(chunk, pageind); arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); } -void -arena_stats_merge(arena_t *arena, size_t *nactive, size_t *ndirty, - arena_stats_t *astats, malloc_bin_stats_t *bstats, - malloc_large_stats_t *lstats) -{ - unsigned i; - - malloc_mutex_lock(&arena->lock); - *nactive += arena->nactive; - *ndirty += arena->ndirty; - - astats->mapped += arena->stats.mapped; - astats->npurge += arena->stats.npurge; - astats->nmadvise += arena->stats.nmadvise; - astats->purged += arena->stats.purged; - astats->allocated_large += arena->stats.allocated_large; - astats->nmalloc_large += arena->stats.nmalloc_large; - astats->ndalloc_large += arena->stats.ndalloc_large; - astats->nrequests_large += arena->stats.nrequests_large; - - for (i = 0; i < nlclasses; i++) { - lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; - lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; - lstats[i].nrequests += arena->stats.lstats[i].nrequests; - lstats[i].curruns += arena->stats.lstats[i].curruns; - } - malloc_mutex_unlock(&arena->lock); - - for (i = 0; i < NBINS; i++) { - arena_bin_t *bin = &arena->bins[i]; - - malloc_mutex_lock(&bin->lock); - bstats[i].allocated += bin->stats.allocated; - bstats[i].nmalloc += bin->stats.nmalloc; - bstats[i].ndalloc += bin->stats.ndalloc; - bstats[i].nrequests += bin->stats.nrequests; - if (config_tcache) { - bstats[i].nfills += bin->stats.nfills; - bstats[i].nflushes += bin->stats.nflushes; - } - bstats[i].nruns += bin->stats.nruns; - bstats[i].reruns += bin->stats.reruns; - bstats[i].curruns += bin->stats.curruns; - malloc_mutex_unlock(&bin->lock); - } -} void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) @@ -1694,7 +1778,7 @@ arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) } } - arena_run_dalloc(arena, (arena_run_t *)ptr, true); + arena_run_dalloc(arena, (arena_run_t *)ptr, true, false); } void @@ -1887,8 +1971,9 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, } void * -arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache) +arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, + size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, + bool try_tcache_dalloc) { void *ret; size_t copysize; @@ -1907,9 +1992,9 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - ret = ipalloc(usize, alignment, zero); + ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); } else - ret = arena_malloc(NULL, size + extra, zero, try_tcache); + ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); if (ret == NULL) { if (extra == 0) @@ -1919,9 +2004,10 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); - ret = ipalloc(usize, alignment, zero); + ret = ipallocx(usize, alignment, zero, try_tcache_alloc, + arena); } else - ret = arena_malloc(NULL, size, zero, try_tcache); + ret = arena_malloc(arena, size, zero, try_tcache_alloc); if (ret == NULL) return (NULL); @@ -1936,10 +2022,78 @@ arena_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, copysize = (size < oldsize) ? size : oldsize; VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - iqalloc(ptr); + iqallocx(ptr, try_tcache_dalloc); + return (ret); +} + +dss_prec_t +arena_dss_prec_get(arena_t *arena) +{ + dss_prec_t ret; + + malloc_mutex_lock(&arena->lock); + ret = arena->dss_prec; + malloc_mutex_unlock(&arena->lock); return (ret); } +void +arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec) +{ + + malloc_mutex_lock(&arena->lock); + arena->dss_prec = dss_prec; + malloc_mutex_unlock(&arena->lock); +} + +void +arena_stats_merge(arena_t *arena, const char **dss, size_t *nactive, + size_t *ndirty, arena_stats_t *astats, malloc_bin_stats_t *bstats, + malloc_large_stats_t *lstats) +{ + unsigned i; + + malloc_mutex_lock(&arena->lock); + *dss = dss_prec_names[arena->dss_prec]; + *nactive += arena->nactive; + *ndirty += arena->ndirty; + + astats->mapped += arena->stats.mapped; + astats->npurge += arena->stats.npurge; + astats->nmadvise += arena->stats.nmadvise; + astats->purged += arena->stats.purged; + astats->allocated_large += arena->stats.allocated_large; + astats->nmalloc_large += arena->stats.nmalloc_large; + astats->ndalloc_large += arena->stats.ndalloc_large; + astats->nrequests_large += arena->stats.nrequests_large; + + for (i = 0; i < nlclasses; i++) { + lstats[i].nmalloc += arena->stats.lstats[i].nmalloc; + lstats[i].ndalloc += arena->stats.lstats[i].ndalloc; + lstats[i].nrequests += arena->stats.lstats[i].nrequests; + lstats[i].curruns += arena->stats.lstats[i].curruns; + } + malloc_mutex_unlock(&arena->lock); + + for (i = 0; i < NBINS; i++) { + arena_bin_t *bin = &arena->bins[i]; + + malloc_mutex_lock(&bin->lock); + bstats[i].allocated += bin->stats.allocated; + bstats[i].nmalloc += bin->stats.nmalloc; + bstats[i].ndalloc += bin->stats.ndalloc; + bstats[i].nrequests += bin->stats.nrequests; + if (config_tcache) { + bstats[i].nfills += bin->stats.nfills; + bstats[i].nflushes += bin->stats.nflushes; + } + bstats[i].nruns += bin->stats.nruns; + bstats[i].reruns += bin->stats.reruns; + bstats[i].curruns += bin->stats.curruns; + malloc_mutex_unlock(&bin->lock); + } +} + bool arena_new(arena_t *arena, unsigned ind) { @@ -1968,16 +2122,17 @@ arena_new(arena_t *arena, unsigned ind) if (config_prof) arena->prof_accumbytes = 0; + arena->dss_prec = chunk_dss_prec_get(); + /* Initialize chunks. */ - ql_new(&arena->chunks_dirty); + arena_chunk_dirty_new(&arena->chunks_dirty); arena->spare = NULL; arena->nactive = 0; arena->ndirty = 0; arena->npurgatory = 0; - arena_avail_tree_new(&arena->runs_avail_clean); - arena_avail_tree_new(&arena->runs_avail_dirty); + arena_avail_tree_new(&arena->runs_avail); /* Initialize bins. */ for (i = 0; i < NBINS; i++) { diff --git a/deps/jemalloc/src/base.c b/deps/jemalloc/src/base.c index bafaa743801..b1a5945efaa 100644 --- a/deps/jemalloc/src/base.c +++ b/deps/jemalloc/src/base.c @@ -32,7 +32,8 @@ base_pages_alloc(size_t minsize) assert(minsize != 0); csize = CHUNK_CEILING(minsize); zero = false; - base_pages = chunk_alloc(csize, chunksize, true, &zero); + base_pages = chunk_alloc(csize, chunksize, true, &zero, + chunk_dss_prec_get()); if (base_pages == NULL) return (true); base_next_addr = base_pages; diff --git a/deps/jemalloc/src/chunk.c b/deps/jemalloc/src/chunk.c index 6bc24544797..1a3bb4f673f 100644 --- a/deps/jemalloc/src/chunk.c +++ b/deps/jemalloc/src/chunk.c @@ -4,7 +4,8 @@ /******************************************************************************/ /* Data. */ -size_t opt_lg_chunk = LG_CHUNK_DEFAULT; +const char *opt_dss = DSS_DEFAULT; +size_t opt_lg_chunk = LG_CHUNK_DEFAULT; malloc_mutex_t chunks_mtx; chunk_stats_t stats_chunks; @@ -15,8 +16,10 @@ chunk_stats_t stats_chunks; * address space. Depending on function, different tree orderings are needed, * which is why there are two trees with the same contents. */ -static extent_tree_t chunks_szad; -static extent_tree_t chunks_ad; +static extent_tree_t chunks_szad_mmap; +static extent_tree_t chunks_ad_mmap; +static extent_tree_t chunks_szad_dss; +static extent_tree_t chunks_ad_dss; rtree_t *chunks_rtree; @@ -30,19 +33,23 @@ size_t arena_maxclass; /* Max size class for arenas. */ /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static void *chunk_recycle(size_t size, size_t alignment, bool base, +static void *chunk_recycle(extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, size_t size, size_t alignment, bool base, bool *zero); -static void chunk_record(void *chunk, size_t size); +static void chunk_record(extent_tree_t *chunks_szad, + extent_tree_t *chunks_ad, void *chunk, size_t size); /******************************************************************************/ static void * -chunk_recycle(size_t size, size_t alignment, bool base, bool *zero) +chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, + size_t alignment, bool base, bool *zero) { void *ret; extent_node_t *node; extent_node_t key; size_t alloc_size, leadsize, trailsize; + bool zeroed; if (base) { /* @@ -61,7 +68,7 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero) key.addr = NULL; key.size = alloc_size; malloc_mutex_lock(&chunks_mtx); - node = extent_tree_szad_nsearch(&chunks_szad, &key); + node = extent_tree_szad_nsearch(chunks_szad, &key); if (node == NULL) { malloc_mutex_unlock(&chunks_mtx); return (NULL); @@ -72,13 +79,13 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero) trailsize = node->size - leadsize - size; ret = (void *)((uintptr_t)node->addr + leadsize); /* Remove node from the tree. */ - extent_tree_szad_remove(&chunks_szad, node); - extent_tree_ad_remove(&chunks_ad, node); + extent_tree_szad_remove(chunks_szad, node); + extent_tree_ad_remove(chunks_ad, node); if (leadsize != 0) { /* Insert the leading space as a smaller chunk. */ node->size = leadsize; - extent_tree_szad_insert(&chunks_szad, node); - extent_tree_ad_insert(&chunks_ad, node); + extent_tree_szad_insert(chunks_szad, node); + extent_tree_ad_insert(chunks_ad, node); node = NULL; } if (trailsize != 0) { @@ -101,23 +108,24 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero) } node->addr = (void *)((uintptr_t)(ret) + size); node->size = trailsize; - extent_tree_szad_insert(&chunks_szad, node); - extent_tree_ad_insert(&chunks_ad, node); + extent_tree_szad_insert(chunks_szad, node); + extent_tree_ad_insert(chunks_ad, node); node = NULL; } malloc_mutex_unlock(&chunks_mtx); - if (node != NULL) + zeroed = false; + if (node != NULL) { + if (node->zeroed) { + zeroed = true; + *zero = true; + } base_node_dealloc(node); -#ifdef JEMALLOC_PURGE_MADVISE_DONTNEED - /* Pages are zeroed as a side effect of pages_purge(). */ - *zero = true; -#else - if (*zero) { + } + if (zeroed == false && *zero) { VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); } -#endif return (ret); } @@ -128,7 +136,8 @@ chunk_recycle(size_t size, size_t alignment, bool base, bool *zero) * advantage of them if they are returned. */ void * -chunk_alloc(size_t size, size_t alignment, bool base, bool *zero) +chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, + dss_prec_t dss_prec) { void *ret; @@ -137,17 +146,26 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero) assert(alignment != 0); assert((alignment & chunksize_mask) == 0); - ret = chunk_recycle(size, alignment, base, zero); - if (ret != NULL) + /* "primary" dss. */ + if (config_dss && dss_prec == dss_prec_primary) { + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, + alignment, base, zero)) != NULL) + goto label_return; + if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) + goto label_return; + } + /* mmap. */ + if ((ret = chunk_recycle(&chunks_szad_mmap, &chunks_ad_mmap, size, + alignment, base, zero)) != NULL) goto label_return; - - ret = chunk_alloc_mmap(size, alignment, zero); - if (ret != NULL) + if ((ret = chunk_alloc_mmap(size, alignment, zero)) != NULL) goto label_return; - - if (config_dss) { - ret = chunk_alloc_dss(size, alignment, zero); - if (ret != NULL) + /* "secondary" dss. */ + if (config_dss && dss_prec == dss_prec_secondary) { + if ((ret = chunk_recycle(&chunks_szad_dss, &chunks_ad_dss, size, + alignment, base, zero)) != NULL) + goto label_return; + if ((ret = chunk_alloc_dss(size, alignment, zero)) != NULL) goto label_return; } @@ -189,11 +207,13 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero) } static void -chunk_record(void *chunk, size_t size) +chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, + size_t size) { + bool unzeroed; extent_node_t *xnode, *node, *prev, key; - pages_purge(chunk, size); + unzeroed = pages_purge(chunk, size); /* * Allocate a node before acquiring chunks_mtx even though it might not @@ -205,7 +225,7 @@ chunk_record(void *chunk, size_t size) malloc_mutex_lock(&chunks_mtx); key.addr = (void *)((uintptr_t)chunk + size); - node = extent_tree_ad_nsearch(&chunks_ad, &key); + node = extent_tree_ad_nsearch(chunks_ad, &key); /* Try to coalesce forward. */ if (node != NULL && node->addr == key.addr) { /* @@ -213,10 +233,11 @@ chunk_record(void *chunk, size_t size) * not change the position within chunks_ad, so only * remove/insert from/into chunks_szad. */ - extent_tree_szad_remove(&chunks_szad, node); + extent_tree_szad_remove(chunks_szad, node); node->addr = chunk; node->size += size; - extent_tree_szad_insert(&chunks_szad, node); + node->zeroed = (node->zeroed && (unzeroed == false)); + extent_tree_szad_insert(chunks_szad, node); if (xnode != NULL) base_node_dealloc(xnode); } else { @@ -234,12 +255,13 @@ chunk_record(void *chunk, size_t size) node = xnode; node->addr = chunk; node->size = size; - extent_tree_ad_insert(&chunks_ad, node); - extent_tree_szad_insert(&chunks_szad, node); + node->zeroed = (unzeroed == false); + extent_tree_ad_insert(chunks_ad, node); + extent_tree_szad_insert(chunks_szad, node); } /* Try to coalesce backward. */ - prev = extent_tree_ad_prev(&chunks_ad, node); + prev = extent_tree_ad_prev(chunks_ad, node); if (prev != NULL && (void *)((uintptr_t)prev->addr + prev->size) == chunk) { /* @@ -247,19 +269,34 @@ chunk_record(void *chunk, size_t size) * not change the position within chunks_ad, so only * remove/insert node from/into chunks_szad. */ - extent_tree_szad_remove(&chunks_szad, prev); - extent_tree_ad_remove(&chunks_ad, prev); + extent_tree_szad_remove(chunks_szad, prev); + extent_tree_ad_remove(chunks_ad, prev); - extent_tree_szad_remove(&chunks_szad, node); + extent_tree_szad_remove(chunks_szad, node); node->addr = prev->addr; node->size += prev->size; - extent_tree_szad_insert(&chunks_szad, node); + node->zeroed = (node->zeroed && prev->zeroed); + extent_tree_szad_insert(chunks_szad, node); base_node_dealloc(prev); } malloc_mutex_unlock(&chunks_mtx); } +void +chunk_unmap(void *chunk, size_t size) +{ + assert(chunk != NULL); + assert(CHUNK_ADDR2BASE(chunk) == chunk); + assert(size != 0); + assert((size & chunksize_mask) == 0); + + if (config_dss && chunk_in_dss(chunk)) + chunk_record(&chunks_szad_dss, &chunks_ad_dss, chunk, size); + else if (chunk_dealloc_mmap(chunk, size)) + chunk_record(&chunks_szad_mmap, &chunks_ad_mmap, chunk, size); +} + void chunk_dealloc(void *chunk, size_t size, bool unmap) { @@ -273,15 +310,13 @@ chunk_dealloc(void *chunk, size_t size, bool unmap) rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); if (config_stats || config_prof) { malloc_mutex_lock(&chunks_mtx); + assert(stats_chunks.curchunks >= (size / chunksize)); stats_chunks.curchunks -= (size / chunksize); malloc_mutex_unlock(&chunks_mtx); } - if (unmap) { - if ((config_dss && chunk_in_dss(chunk)) || - chunk_dealloc_mmap(chunk, size)) - chunk_record(chunk, size); - } + if (unmap) + chunk_unmap(chunk, size); } bool @@ -301,8 +336,10 @@ chunk_boot(void) } if (config_dss && chunk_dss_boot()) return (true); - extent_tree_szad_new(&chunks_szad); - extent_tree_ad_new(&chunks_ad); + extent_tree_szad_new(&chunks_szad_mmap); + extent_tree_ad_new(&chunks_ad_mmap); + extent_tree_szad_new(&chunks_szad_dss); + extent_tree_ad_new(&chunks_ad_dss); if (config_ivsalloc) { chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - opt_lg_chunk); @@ -312,3 +349,33 @@ chunk_boot(void) return (false); } + +void +chunk_prefork(void) +{ + + malloc_mutex_lock(&chunks_mtx); + if (config_ivsalloc) + rtree_prefork(chunks_rtree); + chunk_dss_prefork(); +} + +void +chunk_postfork_parent(void) +{ + + chunk_dss_postfork_parent(); + if (config_ivsalloc) + rtree_postfork_parent(chunks_rtree); + malloc_mutex_postfork_parent(&chunks_mtx); +} + +void +chunk_postfork_child(void) +{ + + chunk_dss_postfork_child(); + if (config_ivsalloc) + rtree_postfork_child(chunks_rtree); + malloc_mutex_postfork_child(&chunks_mtx); +} diff --git a/deps/jemalloc/src/chunk_dss.c b/deps/jemalloc/src/chunk_dss.c index 2d68e4804d7..24781cc52dc 100644 --- a/deps/jemalloc/src/chunk_dss.c +++ b/deps/jemalloc/src/chunk_dss.c @@ -3,6 +3,16 @@ /******************************************************************************/ /* Data. */ +const char *dss_prec_names[] = { + "disabled", + "primary", + "secondary", + "N/A" +}; + +/* Current dss precedence default, used when creating new arenas. */ +static dss_prec_t dss_prec_default = DSS_PREC_DEFAULT; + /* * Protects sbrk() calls. This avoids malloc races among threads, though it * does not protect against races with threads that call sbrk() directly. @@ -29,6 +39,31 @@ sbrk(intptr_t increment) } #endif +dss_prec_t +chunk_dss_prec_get(void) +{ + dss_prec_t ret; + + if (config_dss == false) + return (dss_prec_disabled); + malloc_mutex_lock(&dss_mtx); + ret = dss_prec_default; + malloc_mutex_unlock(&dss_mtx); + return (ret); +} + +bool +chunk_dss_prec_set(dss_prec_t dss_prec) +{ + + if (config_dss == false) + return (true); + malloc_mutex_lock(&dss_mtx); + dss_prec_default = dss_prec; + malloc_mutex_unlock(&dss_mtx); + return (false); +} + void * chunk_alloc_dss(size_t size, size_t alignment, bool *zero) { @@ -88,7 +123,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero) dss_max = dss_next; malloc_mutex_unlock(&dss_mtx); if (cpad_size != 0) - chunk_dealloc(cpad, cpad_size, true); + chunk_unmap(cpad, cpad_size); if (*zero) { VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); diff --git a/deps/jemalloc/src/chunk_mmap.c b/deps/jemalloc/src/chunk_mmap.c index c8da6556b09..8a42e75915f 100644 --- a/deps/jemalloc/src/chunk_mmap.c +++ b/deps/jemalloc/src/chunk_mmap.c @@ -113,22 +113,30 @@ pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size) #endif } -void +bool pages_purge(void *addr, size_t length) { + bool unzeroed; #ifdef _WIN32 VirtualAlloc(addr, length, MEM_RESET, PAGE_READWRITE); + unzeroed = true; #else # ifdef JEMALLOC_PURGE_MADVISE_DONTNEED # define JEMALLOC_MADV_PURGE MADV_DONTNEED +# define JEMALLOC_MADV_ZEROS true # elif defined(JEMALLOC_PURGE_MADVISE_FREE) # define JEMALLOC_MADV_PURGE MADV_FREE +# define JEMALLOC_MADV_ZEROS false # else # error "No method defined for purging unused dirty pages." # endif - madvise(addr, length, JEMALLOC_MADV_PURGE); + int err = madvise(addr, length, JEMALLOC_MADV_PURGE); + unzeroed = (JEMALLOC_MADV_ZEROS == false || err != 0); +# undef JEMALLOC_MADV_PURGE +# undef JEMALLOC_MADV_ZEROS #endif + return (unzeroed); } static void * diff --git a/deps/jemalloc/src/ctl.c b/deps/jemalloc/src/ctl.c index 55e766777fd..6e01b1e27e1 100644 --- a/deps/jemalloc/src/ctl.c +++ b/deps/jemalloc/src/ctl.c @@ -48,8 +48,8 @@ static int n##_ctl(const size_t *mib, size_t miblen, void *oldp, \ size_t *oldlenp, void *newp, size_t newlen); #define INDEX_PROTO(n) \ -const ctl_named_node_t *n##_index(const size_t *mib, size_t miblen, \ - size_t i); +static const ctl_named_node_t *n##_index(const size_t *mib, \ + size_t miblen, size_t i); static bool ctl_arena_init(ctl_arena_stats_t *astats); static void ctl_arena_clear(ctl_arena_stats_t *astats); @@ -58,6 +58,7 @@ static void ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, static void ctl_arena_stats_smerge(ctl_arena_stats_t *sstats, ctl_arena_stats_t *astats); static void ctl_arena_refresh(arena_t *arena, unsigned i); +static bool ctl_grow(void); static void ctl_refresh(void); static bool ctl_init(void); static int ctl_lookup(const char *name, ctl_node_t const **nodesp, @@ -88,6 +89,7 @@ CTL_PROTO(config_utrace) CTL_PROTO(config_valgrind) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) +CTL_PROTO(opt_dss) CTL_PROTO(opt_lg_chunk) CTL_PROTO(opt_narenas) CTL_PROTO(opt_lg_dirty_mult) @@ -110,6 +112,10 @@ CTL_PROTO(opt_prof_gdump) CTL_PROTO(opt_prof_final) CTL_PROTO(opt_prof_leak) CTL_PROTO(opt_prof_accum) +CTL_PROTO(arena_i_purge) +static void arena_purge(unsigned arena_ind); +CTL_PROTO(arena_i_dss) +INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_run_size) @@ -125,6 +131,7 @@ CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nhbins) CTL_PROTO(arenas_nlruns) CTL_PROTO(arenas_purge) +CTL_PROTO(arenas_extend) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) CTL_PROTO(prof_interval) @@ -158,6 +165,7 @@ CTL_PROTO(stats_arenas_i_lruns_j_nrequests) CTL_PROTO(stats_arenas_i_lruns_j_curruns) INDEX_PROTO(stats_arenas_i_lruns_j) CTL_PROTO(stats_arenas_i_nthreads) +CTL_PROTO(stats_arenas_i_dss) CTL_PROTO(stats_arenas_i_pactive) CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_mapped) @@ -223,6 +231,7 @@ static const ctl_named_node_t config_node[] = { static const ctl_named_node_t opt_node[] = { {NAME("abort"), CTL(opt_abort)}, + {NAME("dss"), CTL(opt_dss)}, {NAME("lg_chunk"), CTL(opt_lg_chunk)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("lg_dirty_mult"), CTL(opt_lg_dirty_mult)}, @@ -247,6 +256,18 @@ static const ctl_named_node_t opt_node[] = { {NAME("prof_accum"), CTL(opt_prof_accum)} }; +static const ctl_named_node_t arena_i_node[] = { + {NAME("purge"), CTL(arena_i_purge)}, + {NAME("dss"), CTL(arena_i_dss)} +}; +static const ctl_named_node_t super_arena_i_node[] = { + {NAME(""), CHILD(named, arena_i)} +}; + +static const ctl_indexed_node_t arena_node[] = { + {INDEX(arena_i)} +}; + static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, @@ -282,7 +303,8 @@ static const ctl_named_node_t arenas_node[] = { {NAME("bin"), CHILD(indexed, arenas_bin)}, {NAME("nlruns"), CTL(arenas_nlruns)}, {NAME("lrun"), CHILD(indexed, arenas_lrun)}, - {NAME("purge"), CTL(arenas_purge)} + {NAME("purge"), CTL(arenas_purge)}, + {NAME("extend"), CTL(arenas_extend)} }; static const ctl_named_node_t prof_node[] = { @@ -352,6 +374,7 @@ static const ctl_indexed_node_t stats_arenas_i_lruns_node[] = { static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("nthreads"), CTL(stats_arenas_i_nthreads)}, + {NAME("dss"), CTL(stats_arenas_i_dss)}, {NAME("pactive"), CTL(stats_arenas_i_pactive)}, {NAME("pdirty"), CTL(stats_arenas_i_pdirty)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, @@ -387,6 +410,7 @@ static const ctl_named_node_t root_node[] = { {NAME("thread"), CHILD(named, thread)}, {NAME("config"), CHILD(named, config)}, {NAME("opt"), CHILD(named, opt)}, + {NAME("arena"), CHILD(indexed, arena)}, {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, {NAME("stats"), CHILD(named, stats)} @@ -420,6 +444,7 @@ static void ctl_arena_clear(ctl_arena_stats_t *astats) { + astats->dss = dss_prec_names[dss_prec_limit]; astats->pactive = 0; astats->pdirty = 0; if (config_stats) { @@ -439,8 +464,8 @@ ctl_arena_stats_amerge(ctl_arena_stats_t *cstats, arena_t *arena) { unsigned i; - arena_stats_merge(arena, &cstats->pactive, &cstats->pdirty, - &cstats->astats, cstats->bstats, cstats->lstats); + arena_stats_merge(arena, &cstats->dss, &cstats->pactive, + &cstats->pdirty, &cstats->astats, cstats->bstats, cstats->lstats); for (i = 0; i < NBINS; i++) { cstats->allocated_small += cstats->bstats[i].allocated; @@ -500,7 +525,7 @@ static void ctl_arena_refresh(arena_t *arena, unsigned i) { ctl_arena_stats_t *astats = &ctl_stats.arenas[i]; - ctl_arena_stats_t *sstats = &ctl_stats.arenas[narenas]; + ctl_arena_stats_t *sstats = &ctl_stats.arenas[ctl_stats.narenas]; ctl_arena_clear(astats); @@ -518,11 +543,72 @@ ctl_arena_refresh(arena_t *arena, unsigned i) } } +static bool +ctl_grow(void) +{ + size_t astats_size; + ctl_arena_stats_t *astats; + arena_t **tarenas; + + /* Extend arena stats and arenas arrays. */ + astats_size = (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t); + if (ctl_stats.narenas == narenas_auto) { + /* ctl_stats.arenas and arenas came from base_alloc(). */ + astats = (ctl_arena_stats_t *)imalloc(astats_size); + if (astats == NULL) + return (true); + memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * + sizeof(ctl_arena_stats_t)); + + tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * + sizeof(arena_t *)); + if (tarenas == NULL) { + idalloc(astats); + return (true); + } + memcpy(tarenas, arenas, ctl_stats.narenas * sizeof(arena_t *)); + } else { + astats = (ctl_arena_stats_t *)iralloc(ctl_stats.arenas, + astats_size, 0, 0, false, false); + if (astats == NULL) + return (true); + + tarenas = (arena_t **)iralloc(arenas, (ctl_stats.narenas + 1) * + sizeof(arena_t *), 0, 0, false, false); + if (tarenas == NULL) + return (true); + } + /* Initialize the new astats and arenas elements. */ + memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); + if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) + return (true); + tarenas[ctl_stats.narenas] = NULL; + /* Swap merged stats to their new location. */ + { + ctl_arena_stats_t tstats; + memcpy(&tstats, &astats[ctl_stats.narenas], + sizeof(ctl_arena_stats_t)); + memcpy(&astats[ctl_stats.narenas], + &astats[ctl_stats.narenas + 1], sizeof(ctl_arena_stats_t)); + memcpy(&astats[ctl_stats.narenas + 1], &tstats, + sizeof(ctl_arena_stats_t)); + } + ctl_stats.arenas = astats; + ctl_stats.narenas++; + malloc_mutex_lock(&arenas_lock); + arenas = tarenas; + narenas_total++; + arenas_extend(narenas_total - 1); + malloc_mutex_unlock(&arenas_lock); + + return (false); +} + static void ctl_refresh(void) { unsigned i; - VARIABLE_ARRAY(arena_t *, tarenas, narenas); + VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); if (config_stats) { malloc_mutex_lock(&chunks_mtx); @@ -542,19 +628,19 @@ ctl_refresh(void) * Clear sum stats, since they will be merged into by * ctl_arena_refresh(). */ - ctl_stats.arenas[narenas].nthreads = 0; - ctl_arena_clear(&ctl_stats.arenas[narenas]); + ctl_stats.arenas[ctl_stats.narenas].nthreads = 0; + ctl_arena_clear(&ctl_stats.arenas[ctl_stats.narenas]); malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); - for (i = 0; i < narenas; i++) { + memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas); + for (i = 0; i < ctl_stats.narenas; i++) { if (arenas[i] != NULL) ctl_stats.arenas[i].nthreads = arenas[i]->nthreads; else ctl_stats.arenas[i].nthreads = 0; } malloc_mutex_unlock(&arenas_lock); - for (i = 0; i < narenas; i++) { + for (i = 0; i < ctl_stats.narenas; i++) { bool initialized = (tarenas[i] != NULL); ctl_stats.arenas[i].initialized = initialized; @@ -563,11 +649,13 @@ ctl_refresh(void) } if (config_stats) { - ctl_stats.allocated = ctl_stats.arenas[narenas].allocated_small - + ctl_stats.arenas[narenas].astats.allocated_large + ctl_stats.allocated = + ctl_stats.arenas[ctl_stats.narenas].allocated_small + + ctl_stats.arenas[ctl_stats.narenas].astats.allocated_large + + ctl_stats.huge.allocated; + ctl_stats.active = + (ctl_stats.arenas[ctl_stats.narenas].pactive << LG_PAGE) + ctl_stats.huge.allocated; - ctl_stats.active = (ctl_stats.arenas[narenas].pactive << - LG_PAGE) + ctl_stats.huge.allocated; ctl_stats.mapped = (ctl_stats.chunks.current << opt_lg_chunk); } @@ -585,13 +673,15 @@ ctl_init(void) * Allocate space for one extra arena stats element, which * contains summed stats across all arenas. */ + assert(narenas_auto == narenas_total_get()); + ctl_stats.narenas = narenas_auto; ctl_stats.arenas = (ctl_arena_stats_t *)base_alloc( - (narenas + 1) * sizeof(ctl_arena_stats_t)); + (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); if (ctl_stats.arenas == NULL) { ret = true; goto label_return; } - memset(ctl_stats.arenas, 0, (narenas + 1) * + memset(ctl_stats.arenas, 0, (ctl_stats.narenas + 1) * sizeof(ctl_arena_stats_t)); /* @@ -601,14 +691,14 @@ ctl_init(void) */ if (config_stats) { unsigned i; - for (i = 0; i <= narenas; i++) { + for (i = 0; i <= ctl_stats.narenas; i++) { if (ctl_arena_init(&ctl_stats.arenas[i])) { ret = true; goto label_return; } } } - ctl_stats.arenas[narenas].initialized = true; + ctl_stats.arenas[ctl_stats.narenas].initialized = true; ctl_epoch = 0; ctl_refresh(); @@ -827,6 +917,27 @@ ctl_boot(void) return (false); } +void +ctl_prefork(void) +{ + + malloc_mutex_lock(&ctl_mtx); +} + +void +ctl_postfork_parent(void) +{ + + malloc_mutex_postfork_parent(&ctl_mtx); +} + +void +ctl_postfork_child(void) +{ + + malloc_mutex_postfork_child(&ctl_mtx); +} + /******************************************************************************/ /* *_ctl() functions. */ @@ -1032,8 +1143,8 @@ thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, } READ(oldval, bool); -label_return: ret = 0; +label_return: return (ret); } @@ -1063,13 +1174,14 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; unsigned newind, oldind; + malloc_mutex_lock(&ctl_mtx); newind = oldind = choose_arena(NULL)->ind; WRITE(newind, unsigned); READ(oldind, unsigned); if (newind != oldind) { arena_t *arena; - if (newind >= narenas) { + if (newind >= ctl_stats.narenas) { /* New arena index is out of range. */ ret = EFAULT; goto label_return; @@ -1102,6 +1214,7 @@ thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, ret = 0; label_return: + malloc_mutex_unlock(&ctl_mtx); return (ret); } @@ -1135,6 +1248,7 @@ CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) /******************************************************************************/ CTL_RO_NL_GEN(opt_abort, opt_abort, bool) +CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) @@ -1158,12 +1272,123 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) +/******************************************************************************/ + +/* ctl_mutex must be held during execution of this function. */ +static void +arena_purge(unsigned arena_ind) +{ + VARIABLE_ARRAY(arena_t *, tarenas, ctl_stats.narenas); + + malloc_mutex_lock(&arenas_lock); + memcpy(tarenas, arenas, sizeof(arena_t *) * ctl_stats.narenas); + malloc_mutex_unlock(&arenas_lock); + + if (arena_ind == ctl_stats.narenas) { + unsigned i; + for (i = 0; i < ctl_stats.narenas; i++) { + if (tarenas[i] != NULL) + arena_purge_all(tarenas[i]); + } + } else { + assert(arena_ind < ctl_stats.narenas); + if (tarenas[arena_ind] != NULL) + arena_purge_all(tarenas[arena_ind]); + } +} + +static int +arena_i_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + + READONLY(); + WRITEONLY(); + malloc_mutex_lock(&ctl_mtx); + arena_purge(mib[1]); + malloc_mutex_unlock(&ctl_mtx); + + ret = 0; +label_return: + return (ret); +} + +static int +arena_i_dss_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret, i; + bool match, err; + const char *dss; + unsigned arena_ind = mib[1]; + dss_prec_t dss_prec_old = dss_prec_limit; + dss_prec_t dss_prec = dss_prec_limit; + + malloc_mutex_lock(&ctl_mtx); + WRITE(dss, const char *); + match = false; + for (i = 0; i < dss_prec_limit; i++) { + if (strcmp(dss_prec_names[i], dss) == 0) { + dss_prec = i; + match = true; + break; + } + } + if (match == false) { + ret = EINVAL; + goto label_return; + } + + if (arena_ind < ctl_stats.narenas) { + arena_t *arena = arenas[arena_ind]; + if (arena != NULL) { + dss_prec_old = arena_dss_prec_get(arena); + arena_dss_prec_set(arena, dss_prec); + err = false; + } else + err = true; + } else { + dss_prec_old = chunk_dss_prec_get(); + err = chunk_dss_prec_set(dss_prec); + } + dss = dss_prec_names[dss_prec_old]; + READ(dss, const char *); + if (err) { + ret = EFAULT; + goto label_return; + } + + ret = 0; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + +static const ctl_named_node_t * +arena_i_index(const size_t *mib, size_t miblen, size_t i) +{ + const ctl_named_node_t * ret; + + malloc_mutex_lock(&ctl_mtx); + if (i > ctl_stats.narenas) { + ret = NULL; + goto label_return; + } + + ret = super_arena_i_node; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} + + /******************************************************************************/ CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) -const ctl_named_node_t * +static const ctl_named_node_t * arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) { @@ -1173,7 +1398,7 @@ arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) } CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) -const ctl_named_node_t * +static const ctl_named_node_t * arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) { @@ -1182,7 +1407,27 @@ arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) return (super_arenas_lrun_i_node); } -CTL_RO_NL_GEN(arenas_narenas, narenas, unsigned) +static int +arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + unsigned narenas; + + malloc_mutex_lock(&ctl_mtx); + READONLY(); + if (*oldlenp != sizeof(unsigned)) { + ret = EINVAL; + goto label_return; + } + narenas = ctl_stats.narenas; + READ(narenas, unsigned); + + ret = 0; +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} static int arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, @@ -1193,13 +1438,13 @@ arenas_initialized_ctl(const size_t *mib, size_t miblen, void *oldp, malloc_mutex_lock(&ctl_mtx); READONLY(); - if (*oldlenp != narenas * sizeof(bool)) { + if (*oldlenp != ctl_stats.narenas * sizeof(bool)) { ret = EINVAL; - nread = (*oldlenp < narenas * sizeof(bool)) - ? (*oldlenp / sizeof(bool)) : narenas; + nread = (*oldlenp < ctl_stats.narenas * sizeof(bool)) + ? (*oldlenp / sizeof(bool)) : ctl_stats.narenas; } else { ret = 0; - nread = narenas; + nread = ctl_stats.narenas; } for (i = 0; i < nread; i++) @@ -1222,36 +1467,43 @@ arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - unsigned arena; + unsigned arena_ind; + malloc_mutex_lock(&ctl_mtx); WRITEONLY(); - arena = UINT_MAX; - WRITE(arena, unsigned); - if (newp != NULL && arena >= narenas) { + arena_ind = UINT_MAX; + WRITE(arena_ind, unsigned); + if (newp != NULL && arena_ind >= ctl_stats.narenas) ret = EFAULT; - goto label_return; - } else { - VARIABLE_ARRAY(arena_t *, tarenas, narenas); + else { + if (arena_ind == UINT_MAX) + arena_ind = ctl_stats.narenas; + arena_purge(arena_ind); + ret = 0; + } - malloc_mutex_lock(&arenas_lock); - memcpy(tarenas, arenas, sizeof(arena_t *) * narenas); - malloc_mutex_unlock(&arenas_lock); +label_return: + malloc_mutex_unlock(&ctl_mtx); + return (ret); +} - if (arena == UINT_MAX) { - unsigned i; - for (i = 0; i < narenas; i++) { - if (tarenas[i] != NULL) - arena_purge_all(tarenas[i]); - } - } else { - assert(arena < narenas); - if (tarenas[arena] != NULL) - arena_purge_all(tarenas[arena]); - } +static int +arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + int ret; + + malloc_mutex_lock(&ctl_mtx); + READONLY(); + if (ctl_grow()) { + ret = EAGAIN; + goto label_return; } + READ(ctl_stats.narenas - 1, unsigned); ret = 0; label_return: + malloc_mutex_unlock(&ctl_mtx); return (ret); } @@ -1356,7 +1608,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_nreruns, CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curruns, ctl_stats.arenas[mib[2]].bstats[mib[4]].curruns, size_t) -const ctl_named_node_t * +static const ctl_named_node_t * stats_arenas_i_bins_j_index(const size_t *mib, size_t miblen, size_t j) { @@ -1374,7 +1626,7 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_nrequests, CTL_RO_CGEN(config_stats, stats_arenas_i_lruns_j_curruns, ctl_stats.arenas[mib[2]].lstats[mib[4]].curruns, size_t) -const ctl_named_node_t * +static const ctl_named_node_t * stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) { @@ -1384,6 +1636,7 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) } CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) +CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, @@ -1395,13 +1648,13 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, CTL_RO_CGEN(config_stats, stats_arenas_i_purged, ctl_stats.arenas[mib[2]].astats.purged, uint64_t) -const ctl_named_node_t * +static const ctl_named_node_t * stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) { const ctl_named_node_t * ret; malloc_mutex_lock(&ctl_mtx); - if (ctl_stats.arenas[i].initialized == false) { + if (i > ctl_stats.narenas || ctl_stats.arenas[i].initialized == false) { ret = NULL; goto label_return; } diff --git a/deps/jemalloc/src/huge.c b/deps/jemalloc/src/huge.c index 8a4ec942410..aa08d43d362 100644 --- a/deps/jemalloc/src/huge.c +++ b/deps/jemalloc/src/huge.c @@ -48,7 +48,8 @@ huge_palloc(size_t size, size_t alignment, bool zero) * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - ret = chunk_alloc(csize, alignment, false, &is_zeroed); + ret = chunk_alloc(csize, alignment, false, &is_zeroed, + chunk_dss_prec_get()); if (ret == NULL) { base_node_dealloc(node); return (NULL); @@ -101,7 +102,7 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) void * huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) + size_t alignment, bool zero, bool try_tcache_dalloc) { void *ret; size_t copysize; @@ -180,7 +181,7 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, #endif { memcpy(ret, ptr, copysize); - iqalloc(ptr); + iqallocx(ptr, try_tcache_dalloc); } return (ret); } diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c index bc54cd7ca0d..8a667b62e2d 100644 --- a/deps/jemalloc/src/jemalloc.c +++ b/deps/jemalloc/src/jemalloc.c @@ -33,7 +33,8 @@ unsigned ncpus; malloc_mutex_t arenas_lock; arena_t **arenas; -unsigned narenas; +unsigned narenas_total; +unsigned narenas_auto; /* Set to true once the allocator has been initialized. */ static bool malloc_initialized = false; @@ -144,14 +145,14 @@ choose_arena_hard(void) { arena_t *ret; - if (narenas > 1) { + if (narenas_auto > 1) { unsigned i, choose, first_null; choose = 0; - first_null = narenas; + first_null = narenas_auto; malloc_mutex_lock(&arenas_lock); assert(arenas[0] != NULL); - for (i = 1; i < narenas; i++) { + for (i = 1; i < narenas_auto; i++) { if (arenas[i] != NULL) { /* * Choose the first arena that has the lowest @@ -160,7 +161,7 @@ choose_arena_hard(void) if (arenas[i]->nthreads < arenas[choose]->nthreads) choose = i; - } else if (first_null == narenas) { + } else if (first_null == narenas_auto) { /* * Record the index of the first uninitialized * arena, in case all extant arenas are in use. @@ -174,7 +175,8 @@ choose_arena_hard(void) } } - if (arenas[choose]->nthreads == 0 || first_null == narenas) { + if (arenas[choose]->nthreads == 0 + || first_null == narenas_auto) { /* * Use an unloaded arena, or the least loaded arena if * all arenas are already initialized. @@ -203,7 +205,7 @@ stats_print_atexit(void) { if (config_tcache && config_stats) { - unsigned i; + unsigned narenas, i; /* * Merge stats from extant threads. This is racy, since @@ -212,7 +214,7 @@ stats_print_atexit(void) * out of date by the time they are reported, if other threads * continue to allocate. */ - for (i = 0; i < narenas; i++) { + for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { arena_t *arena = arenas[i]; if (arena != NULL) { tcache_t *tcache; @@ -254,12 +256,13 @@ malloc_ncpus(void) result = si.dwNumberOfProcessors; #else result = sysconf(_SC_NPROCESSORS_ONLN); +#endif if (result == -1) { /* Error. */ ret = 1; - } -#endif - ret = (unsigned)result; + } else { + ret = (unsigned)result; + } return (ret); } @@ -377,6 +380,22 @@ malloc_conf_init(void) const char *opts, *k, *v; size_t klen, vlen; + /* + * Automatically configure valgrind before processing options. The + * valgrind option remains in jemalloc 3.x for compatibility reasons. + */ + if (config_valgrind) { + opt_valgrind = (RUNNING_ON_VALGRIND != 0) ? true : false; + if (config_fill && opt_valgrind) { + opt_junk = false; + assert(opt_zero == false); + opt_quarantine = JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; + opt_redzone = true; + } + if (config_tcache && opt_valgrind) + opt_tcache = false; + } + for (i = 0; i < 3; i++) { /* Get runtime configuration. */ switch (i) { @@ -537,6 +556,30 @@ malloc_conf_init(void) */ CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1) + if (strncmp("dss", k, klen) == 0) { + int i; + bool match = false; + for (i = 0; i < dss_prec_limit; i++) { + if (strncmp(dss_prec_names[i], v, vlen) + == 0) { + if (chunk_dss_prec_set(i)) { + malloc_conf_error( + "Error setting dss", + k, klen, v, vlen); + } else { + opt_dss = + dss_prec_names[i]; + match = true; + break; + } + } + } + if (match == false) { + malloc_conf_error("Invalid conf value", + k, klen, v, vlen); + } + continue; + } CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, SIZE_T_MAX) CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", @@ -553,20 +596,7 @@ malloc_conf_init(void) CONF_HANDLE_BOOL(opt_utrace, "utrace") } if (config_valgrind) { - bool hit; - CONF_HANDLE_BOOL_HIT(opt_valgrind, - "valgrind", hit) - if (config_fill && opt_valgrind && hit) { - opt_junk = false; - opt_zero = false; - if (opt_quarantine == 0) { - opt_quarantine = - JEMALLOC_VALGRIND_QUARANTINE_DEFAULT; - } - opt_redzone = true; - } - if (hit) - continue; + CONF_HANDLE_BOOL(opt_valgrind, "valgrind") } if (config_xmalloc) { CONF_HANDLE_BOOL(opt_xmalloc, "xmalloc") @@ -695,9 +725,9 @@ malloc_init_hard(void) * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ - narenas = 1; + narenas_total = narenas_auto = 1; arenas = init_arenas; - memset(arenas, 0, sizeof(arena_t *) * narenas); + memset(arenas, 0, sizeof(arena_t *) * narenas_auto); /* * Initialize one arena here. The rest are lazily created in @@ -755,20 +785,21 @@ malloc_init_hard(void) else opt_narenas = 1; } - narenas = opt_narenas; + narenas_auto = opt_narenas; /* * Make sure that the arenas array can be allocated. In practice, this * limit is enough to allow the allocator to function, but the ctl * machinery will fail to allocate memory at far lower limits. */ - if (narenas > chunksize / sizeof(arena_t *)) { - narenas = chunksize / sizeof(arena_t *); + if (narenas_auto > chunksize / sizeof(arena_t *)) { + narenas_auto = chunksize / sizeof(arena_t *); malloc_printf(": Reducing narenas to limit (%d)\n", - narenas); + narenas_auto); } + narenas_total = narenas_auto; /* Allocate and initialize arenas. */ - arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas); + arenas = (arena_t **)base_alloc(sizeof(arena_t *) * narenas_total); if (arenas == NULL) { malloc_mutex_unlock(&init_lock); return (true); @@ -777,7 +808,7 @@ malloc_init_hard(void) * Zero the array. In practice, this should always be pre-zeroed, * since it was just mmap()ed, but let's be sure. */ - memset(arenas, 0, sizeof(arena_t *) * narenas); + memset(arenas, 0, sizeof(arena_t *) * narenas_total); /* Copy the pointer to the one arena that was already initialized. */ arenas[0] = init_arenas[0]; @@ -1262,11 +1293,10 @@ je_valloc(size_t size) * passed an extra argument for the caller return address, which will be * ignored. */ -JEMALLOC_EXPORT void (* const __free_hook)(void *ptr) = je_free; -JEMALLOC_EXPORT void *(* const __malloc_hook)(size_t size) = je_malloc; -JEMALLOC_EXPORT void *(* const __realloc_hook)(void *ptr, size_t size) = - je_realloc; -JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) = +JEMALLOC_EXPORT void (* __free_hook)(void *ptr) = je_free; +JEMALLOC_EXPORT void *(* __malloc_hook)(size_t size) = je_malloc; +JEMALLOC_EXPORT void *(* __realloc_hook)(void *ptr, size_t size) = je_realloc; +JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) = je_memalign; #endif @@ -1279,7 +1309,7 @@ JEMALLOC_EXPORT void *(* const __memalign_hook)(size_t alignment, size_t size) = */ size_t -je_malloc_usable_size(const void *ptr) +je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { size_t ret; @@ -1343,18 +1373,19 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, #ifdef JEMALLOC_EXPERIMENTAL JEMALLOC_INLINE void * -iallocm(size_t usize, size_t alignment, bool zero) +iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) { assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, alignment))); if (alignment != 0) - return (ipalloc(usize, alignment, zero)); + return (ipallocx(usize, alignment, zero, try_tcache, arena)); else if (zero) - return (icalloc(usize)); + return (icallocx(usize, try_tcache, arena)); else - return (imalloc(usize)); + return (imallocx(usize, try_tcache, arena)); } int @@ -1365,6 +1396,9 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags) size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); bool zero = flags & ALLOCM_ZERO; + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + arena_t *arena; + bool try_tcache; assert(ptr != NULL); assert(size != 0); @@ -1372,6 +1406,14 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags) if (malloc_init()) goto label_oom; + if (arena_ind != UINT_MAX) { + arena = arenas[arena_ind]; + try_tcache = false; + } else { + arena = NULL; + try_tcache = true; + } + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); if (usize == 0) goto label_oom; @@ -1388,18 +1430,19 @@ je_allocm(void **ptr, size_t *rsize, size_t size, int flags) s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); assert(usize_promoted != 0); - p = iallocm(usize_promoted, alignment, zero); + p = iallocm(usize_promoted, alignment, zero, + try_tcache, arena); if (p == NULL) goto label_oom; arena_prof_promoted(p, usize); } else { - p = iallocm(usize, alignment, zero); + p = iallocm(usize, alignment, zero, try_tcache, arena); if (p == NULL) goto label_oom; } prof_malloc(p, usize, cnt); } else { - p = iallocm(usize, alignment, zero); + p = iallocm(usize, alignment, zero, try_tcache, arena); if (p == NULL) goto label_oom; } @@ -1436,6 +1479,9 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) & (SIZE_T_MAX-1)); bool zero = flags & ALLOCM_ZERO; bool no_move = flags & ALLOCM_NO_MOVE; + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + bool try_tcache_alloc, try_tcache_dalloc; + arena_t *arena; assert(ptr != NULL); assert(*ptr != NULL); @@ -1443,6 +1489,19 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) assert(SIZE_T_MAX - size >= extra); assert(malloc_initialized || IS_INITIALIZER); + if (arena_ind != UINT_MAX) { + arena_chunk_t *chunk; + try_tcache_alloc = true; + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(*ptr); + try_tcache_dalloc = (chunk == *ptr || chunk->arena != + arenas[arena_ind]); + arena = arenas[arena_ind]; + } else { + try_tcache_alloc = true; + try_tcache_dalloc = true; + arena = NULL; + } + p = *ptr; if (config_prof && opt_prof) { prof_thr_cnt_t *cnt; @@ -1469,9 +1528,10 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) <= SMALL_MAXCLASS) { - q = iralloc(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= + q = irallocx(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), - alignment, zero, no_move); + alignment, zero, no_move, try_tcache_alloc, + try_tcache_dalloc, arena); if (q == NULL) goto label_err; if (max_usize < PAGE) { @@ -1480,7 +1540,8 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) } else usize = isalloc(q, config_prof); } else { - q = iralloc(p, size, extra, alignment, zero, no_move); + q = irallocx(p, size, extra, alignment, zero, no_move, + try_tcache_alloc, try_tcache_dalloc, arena); if (q == NULL) goto label_err; usize = isalloc(q, config_prof); @@ -1497,7 +1558,8 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) old_size = isalloc(p, false); old_rzsize = u2rz(old_size); } - q = iralloc(p, size, extra, alignment, zero, no_move); + q = irallocx(p, size, extra, alignment, zero, no_move, + try_tcache_alloc, try_tcache_dalloc, arena); if (q == NULL) goto label_err; if (config_stats) @@ -1558,10 +1620,19 @@ je_dallocm(void *ptr, int flags) { size_t usize; size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + bool try_tcache; assert(ptr != NULL); assert(malloc_initialized || IS_INITIALIZER); + if (arena_ind != UINT_MAX) { + arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + try_tcache = (chunk == ptr || chunk->arena != + arenas[arena_ind]); + } else + try_tcache = true; + UTRACE(ptr, 0, 0); if (config_stats || config_valgrind) usize = isalloc(ptr, config_prof); @@ -1574,7 +1645,7 @@ je_dallocm(void *ptr, int flags) thread_allocated_tsd_get()->deallocated += usize; if (config_valgrind && opt_valgrind) rzsize = p2rz(ptr); - iqalloc(ptr); + iqallocx(ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); return (ALLOCM_SUCCESS); @@ -1611,6 +1682,27 @@ je_nallocm(size_t *rsize, size_t size, int flags) * malloc during fork(). */ +/* + * If an application creates a thread before doing any allocation in the main + * thread, then calls fork(2) in the main thread followed by memory allocation + * in the child process, a race can occur that results in deadlock within the + * child: the main thread may have forked while the created thread had + * partially initialized the allocator. Ordinarily jemalloc prevents + * fork/malloc races via the following functions it registers during + * initialization using pthread_atfork(), but of course that does no good if + * the allocator isn't fully initialized at fork time. The following library + * constructor is a partial solution to this problem. It may still possible to + * trigger the deadlock described above, but doing so would involve forking via + * a library constructor that runs before jemalloc's runs. + */ +JEMALLOC_ATTR(constructor) +static void +jemalloc_constructor(void) +{ + + malloc_init(); +} + #ifndef JEMALLOC_MUTEX_INIT_CB void jemalloc_prefork(void) @@ -1628,14 +1720,16 @@ _malloc_prefork(void) assert(malloc_initialized); /* Acquire all mutexes in a safe order. */ + ctl_prefork(); malloc_mutex_prefork(&arenas_lock); - for (i = 0; i < narenas; i++) { + for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) arena_prefork(arenas[i]); } + prof_prefork(); + chunk_prefork(); base_prefork(); huge_prefork(); - chunk_dss_prefork(); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -1655,14 +1749,16 @@ _malloc_postfork(void) assert(malloc_initialized); /* Release all mutexes, now that fork() has completed. */ - chunk_dss_postfork_parent(); huge_postfork_parent(); base_postfork_parent(); - for (i = 0; i < narenas; i++) { + chunk_postfork_parent(); + prof_postfork_parent(); + for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) arena_postfork_parent(arenas[i]); } malloc_mutex_postfork_parent(&arenas_lock); + ctl_postfork_parent(); } void @@ -1673,14 +1769,16 @@ jemalloc_postfork_child(void) assert(malloc_initialized); /* Release all mutexes, now that fork() has completed. */ - chunk_dss_postfork_child(); huge_postfork_child(); base_postfork_child(); - for (i = 0; i < narenas; i++) { + chunk_postfork_child(); + prof_postfork_child(); + for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) arena_postfork_child(arenas[i]); } malloc_mutex_postfork_child(&arenas_lock); + ctl_postfork_child(); } /******************************************************************************/ diff --git a/deps/jemalloc/src/mutex.c b/deps/jemalloc/src/mutex.c index 37a843e6e44..55e18c23713 100644 --- a/deps/jemalloc/src/mutex.c +++ b/deps/jemalloc/src/mutex.c @@ -64,7 +64,7 @@ pthread_create(pthread_t *__restrict thread, /******************************************************************************/ #ifdef JEMALLOC_MUTEX_INIT_CB -int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, +JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void *(calloc_cb)(size_t, size_t)); #endif diff --git a/deps/jemalloc/src/prof.c b/deps/jemalloc/src/prof.c index de1d392993e..04964ef7ca3 100644 --- a/deps/jemalloc/src/prof.c +++ b/deps/jemalloc/src/prof.c @@ -1270,4 +1270,46 @@ prof_boot2(void) return (false); } +void +prof_prefork(void) +{ + + if (opt_prof) { + unsigned i; + + malloc_mutex_lock(&bt2ctx_mtx); + malloc_mutex_lock(&prof_dump_seq_mtx); + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_lock(&ctx_locks[i]); + } +} + +void +prof_postfork_parent(void) +{ + + if (opt_prof) { + unsigned i; + + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_postfork_parent(&ctx_locks[i]); + malloc_mutex_postfork_parent(&prof_dump_seq_mtx); + malloc_mutex_postfork_parent(&bt2ctx_mtx); + } +} + +void +prof_postfork_child(void) +{ + + if (opt_prof) { + unsigned i; + + for (i = 0; i < PROF_NCTX_LOCKS; i++) + malloc_mutex_postfork_child(&ctx_locks[i]); + malloc_mutex_postfork_child(&prof_dump_seq_mtx); + malloc_mutex_postfork_child(&bt2ctx_mtx); + } +} + /******************************************************************************/ diff --git a/deps/jemalloc/src/rtree.c b/deps/jemalloc/src/rtree.c index eb0ff1e24af..90c6935a0ed 100644 --- a/deps/jemalloc/src/rtree.c +++ b/deps/jemalloc/src/rtree.c @@ -44,3 +44,24 @@ rtree_new(unsigned bits) return (ret); } + +void +rtree_prefork(rtree_t *rtree) +{ + + malloc_mutex_prefork(&rtree->mutex); +} + +void +rtree_postfork_parent(rtree_t *rtree) +{ + + malloc_mutex_postfork_parent(&rtree->mutex); +} + +void +rtree_postfork_child(rtree_t *rtree) +{ + + malloc_mutex_postfork_child(&rtree->mutex); +} diff --git a/deps/jemalloc/src/stats.c b/deps/jemalloc/src/stats.c index 433b80d128d..43f87af6700 100644 --- a/deps/jemalloc/src/stats.c +++ b/deps/jemalloc/src/stats.c @@ -206,6 +206,7 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, unsigned i, bool bins, bool large) { unsigned nthreads; + const char *dss; size_t page, pactive, pdirty, mapped; uint64_t npurge, nmadvise, purged; size_t small_allocated; @@ -218,6 +219,9 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_I_GET("stats.arenas.0.nthreads", &nthreads, unsigned); malloc_cprintf(write_cb, cbopaque, "assigned threads: %u\n", nthreads); + CTL_I_GET("stats.arenas.0.dss", &dss, const char *); + malloc_cprintf(write_cb, cbopaque, "dss allocation precedence: %s\n", + dss); CTL_I_GET("stats.arenas.0.pactive", &pactive, size_t); CTL_I_GET("stats.arenas.0.pdirty", &pdirty, size_t); CTL_I_GET("stats.arenas.0.npurge", &npurge, uint64_t); @@ -370,6 +374,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, "Run-time option settings:\n"); OPT_WRITE_BOOL(abort) OPT_WRITE_SIZE_T(lg_chunk) + OPT_WRITE_CHAR_P(dss) OPT_WRITE_SIZE_T(narenas) OPT_WRITE_SSIZE_T(lg_dirty_mult) OPT_WRITE_BOOL(stats_print) @@ -400,7 +405,7 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "CPUs: %u\n", ncpus); CTL_GET("arenas.narenas", &uv, unsigned); - malloc_cprintf(write_cb, cbopaque, "Max arenas: %u\n", uv); + malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv); malloc_cprintf(write_cb, cbopaque, "Pointer size: %zu\n", sizeof(void *)); @@ -472,7 +477,8 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("stats.chunks.current", &chunks_current, size_t); malloc_cprintf(write_cb, cbopaque, "chunks: nchunks " "highchunks curchunks\n"); - malloc_cprintf(write_cb, cbopaque, " %13"PRIu64"%13zu%13zu\n", + malloc_cprintf(write_cb, cbopaque, + " %13"PRIu64" %12zu %12zu\n", chunks_total, chunks_high, chunks_current); /* Print huge stats. */ diff --git a/deps/jemalloc/src/tcache.c b/deps/jemalloc/src/tcache.c index 60244c45f8b..47e14f30b1a 100644 --- a/deps/jemalloc/src/tcache.c +++ b/deps/jemalloc/src/tcache.c @@ -288,7 +288,7 @@ tcache_create(arena_t *arena) else if (size <= tcache_maxclass) tcache = (tcache_t *)arena_malloc_large(arena, size, true); else - tcache = (tcache_t *)icalloc(size); + tcache = (tcache_t *)icallocx(size, false, arena); if (tcache == NULL) return (NULL); @@ -364,7 +364,7 @@ tcache_destroy(tcache_t *tcache) arena_dalloc_large(arena, chunk, tcache); } else - idalloc(tcache); + idallocx(tcache, false); } void diff --git a/deps/jemalloc/src/util.c b/deps/jemalloc/src/util.c index 9b73c3ec045..b3a01143698 100644 --- a/deps/jemalloc/src/util.c +++ b/deps/jemalloc/src/util.c @@ -377,7 +377,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) case '\0': goto label_out; case '%': { bool alt_form = false; - bool zero_pad = false; bool left_justify = false; bool plus_space = false; bool plus_plus = false; @@ -398,10 +397,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) assert(alt_form == false); alt_form = true; break; - case '0': - assert(zero_pad == false); - zero_pad = true; - break; case '-': assert(left_justify == false); left_justify = true; diff --git a/deps/jemalloc/src/zone.c b/deps/jemalloc/src/zone.c index cde5d49a460..c62c183f65e 100644 --- a/deps/jemalloc/src/zone.c +++ b/deps/jemalloc/src/zone.c @@ -171,6 +171,16 @@ void register_zone(void) { + /* + * If something else replaced the system default zone allocator, don't + * register jemalloc's. + */ + malloc_zone_t *default_zone = malloc_default_zone(); + if (!default_zone->zone_name || + strcmp(default_zone->zone_name, "DefaultMallocZone") != 0) { + return; + } + zone.size = (void *)zone_size; zone.malloc = (void *)zone_malloc; zone.calloc = (void *)zone_calloc; @@ -241,7 +251,7 @@ register_zone(void) * then becomes the default. */ do { - malloc_zone_t *default_zone = malloc_default_zone(); + default_zone = malloc_default_zone(); malloc_zone_unregister(default_zone); malloc_zone_register(default_zone); } while (malloc_default_zone() != &zone); diff --git a/deps/jemalloc/test/ALLOCM_ARENA.c b/deps/jemalloc/test/ALLOCM_ARENA.c new file mode 100644 index 00000000000..15856908ffc --- /dev/null +++ b/deps/jemalloc/test/ALLOCM_ARENA.c @@ -0,0 +1,66 @@ +#define JEMALLOC_MANGLE +#include "jemalloc_test.h" + +#define NTHREADS 10 + +void * +je_thread_start(void *arg) +{ + unsigned thread_ind = (unsigned)(uintptr_t)arg; + unsigned arena_ind; + int r; + void *p; + size_t rsz, sz; + + sz = sizeof(arena_ind); + if (mallctl("arenas.extend", &arena_ind, &sz, NULL, 0) + != 0) { + malloc_printf("Error in arenas.extend\n"); + abort(); + } + + if (thread_ind % 4 != 3) { + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(size_t); + const char *dss_precs[] = {"disabled", "primary", "secondary"}; + const char *dss = dss_precs[thread_ind % 4]; + if (mallctlnametomib("arena.0.dss", mib, &miblen) != 0) { + malloc_printf("Error in mallctlnametomib()\n"); + abort(); + } + mib[1] = arena_ind; + if (mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss, + sizeof(const char *))) { + malloc_printf("Error in mallctlbymib()\n"); + abort(); + } + } + + r = allocm(&p, &rsz, 1, ALLOCM_ARENA(arena_ind)); + if (r != ALLOCM_SUCCESS) { + malloc_printf("Unexpected allocm() error\n"); + abort(); + } + + return (NULL); +} + +int +main(void) +{ + je_thread_t threads[NTHREADS]; + unsigned i; + + malloc_printf("Test begin\n"); + + for (i = 0; i < NTHREADS; i++) { + je_thread_create(&threads[i], je_thread_start, + (void *)(uintptr_t)i); + } + + for (i = 0; i < NTHREADS; i++) + je_thread_join(threads[i], NULL); + + malloc_printf("Test end\n"); + return (0); +} diff --git a/deps/jemalloc/test/ALLOCM_ARENA.exp b/deps/jemalloc/test/ALLOCM_ARENA.exp new file mode 100644 index 00000000000..369a88dd240 --- /dev/null +++ b/deps/jemalloc/test/ALLOCM_ARENA.exp @@ -0,0 +1,2 @@ +Test begin +Test end diff --git a/deps/jemalloc/test/thread_arena.c b/deps/jemalloc/test/thread_arena.c index 2020d99480a..2ffdb5e8004 100644 --- a/deps/jemalloc/test/thread_arena.c +++ b/deps/jemalloc/test/thread_arena.c @@ -1,7 +1,7 @@ #define JEMALLOC_MANGLE #include "jemalloc_test.h" -#define NTHREADS 10 +#define NTHREADS 10 void * je_thread_start(void *arg) @@ -66,8 +66,10 @@ main(void) goto label_return; } - for (i = 0; i < NTHREADS; i++) - je_thread_create(&threads[i], je_thread_start, (void *)&arena_ind); + for (i = 0; i < NTHREADS; i++) { + je_thread_create(&threads[i], je_thread_start, + (void *)&arena_ind); + } for (i = 0; i < NTHREADS; i++) je_thread_join(threads[i], (void *)&ret); From cd99c14e388f23f83564775badfbf5d08651bee7 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 25 Nov 2012 16:21:21 +0100 Subject: [PATCH 0407/1752] On crash memory test rewrote so that it actaully works. 1) We no longer test location by location, otherwise the CPU write cache completely makes our business useless. 2) We still need a memory test that operates in steps from the first to the last location in order to never hit the cache, but that is still able to retain the memory content. This was tested using a Linux box containing a bad memory module with a zingle bit error (always zero). So the final solution does has an error propagation step that is: 1) Invert bits at every location. 2) Swap adiacent locations. 3) Swap adiacent locations again. 4) Invert bits at every location. 5) Swap adiacent locations. 6) Swap adiacent locations again. Before and after these steps, and after step 4, a CRC64 checksum is computed. If the three CRC64 checksums don't match, a memory error was detected. --- src/debug.c | 62 ++++++++++++++++++++++++++++++++++++++++++--------- src/memtest.c | 42 ++++++++++++++++------------------ 2 files changed, 70 insertions(+), 34 deletions(-) diff --git a/src/debug.c b/src/debug.c index 2725aae6ca3..31cfac65ef9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -667,16 +667,22 @@ void logCurrentClient(void) { } #if defined(HAVE_PROC_MAPS) -int memtest_non_destructive(void *addr, size_t size); /* memtest.c */ +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); +void memtest_non_destructive_invert(void *addr, size_t size); +void memtest_non_destructive_swap(void *addr, size_t size); +#define MEMTEST_MAX_REGIONS 128 int memtest_test_linux_anonymous_maps(void) { FILE *fp = fopen("/proc/self/maps","r"); char line[1024]; size_t start_addr, end_addr, size; + size_t start_vect[MEMTEST_MAX_REGIONS]; + size_t size_vect[MEMTEST_MAX_REGIONS]; + int regions = 0, j; + uint64_t crc1 = 0, crc2 = 0, crc3 = 0; while(fgets(line,sizeof(line),fp) != NULL) { char *start, *end, *p = line; - int j; start = p; p = strchr(p,'-'); @@ -695,17 +701,51 @@ int memtest_test_linux_anonymous_maps(void) { start_addr = strtoul(start,NULL,16); end_addr = strtoul(end,NULL,16); size = end_addr-start_addr; - redisLog(REDIS_WARNING, - "Testing memory at %lx (%lu bytes)", start_addr, size); - for (j = 0; j < 3; j++) { - if (memtest_non_destructive((void*)start_addr,size) != 0) { - fclose(fp); - return 1; - } - } + + start_vect[regions] = start_addr; + size_vect[regions] = size; + printf("Testing %lx %lu\n", start_vect[regions], size_vect[regions]); + regions++; } + + /* Test all the regions as an unique sequential region. + * 1) Take the CRC64 of the memory region. */ + for (j = 0; j < regions; j++) { + crc1 = crc64(crc1,(void*)start_vect[j],size_vect[j]); + } + + /* 2) Invert bits, swap adiacent words, swap again, invert bits. + * This is the error amplification step. */ + for (j = 0; j < regions; j++) + memtest_non_destructive_invert((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_invert((void*)start_vect[j],size_vect[j]); + + /* 3) Take the CRC64 sum again. */ + for (j = 0; j < regions; j++) + crc2 = crc64(crc2,(void*)start_vect[j],size_vect[j]); + + /* 4) Swap + Swap again */ + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + for (j = 0; j < regions; j++) + memtest_non_destructive_swap((void*)start_vect[j],size_vect[j]); + + /* 5) Take the CRC64 sum again. */ + for (j = 0; j < regions; j++) + crc3 = crc64(crc3,(void*)start_vect[j],size_vect[j]); + + /* NOTE: It is very important to close the file descriptor only now + * because closing it before may result into unmapping of some memory + * region that we are testing. */ fclose(fp); - return 0; + + /* If the two CRC are not the same, we trapped a memory error. */ + return crc1 != crc2 || crc2 != crc3; } #endif diff --git a/src/memtest.c b/src/memtest.c index 82da27c8bc7..754d0202450 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -241,34 +241,30 @@ void memtest_test(size_t megabytes, int passes) { } } -/* This is a fast O(N) best effort memory test, only ZERO-ONE tests and - * checkerboard tests are performed, without pauses between setting and - * reading the value, so this can only detect a subclass of permanent errors. - * - * However the function does not destroy the content of the memory tested that - * is left unmodified. - * - * If a memory error is detected, 1 is returned. Otherwise 0 is returned. */ -int memtest_non_destructive(void *addr, size_t size) { +void memtest_non_destructive_invert(void *addr, size_t size) { volatile unsigned long *p = addr; - unsigned long val; + size_t words = size / sizeof(unsigned long); size_t j; - size /= sizeof(unsigned long); - for (j = 0; j < size; j++) { - val = p[j]; + /* Invert */ + for (j = 0; j < words; j++) + p[j] = ~p[j]; +} - p[j] = 0; if (p[j] != 0) goto err; - p[j] = (unsigned long)-1; if (p[j] != (unsigned long)-1) goto err; - p[j] = ULONG_ONEZERO; if (p[j] != ULONG_ONEZERO) goto err; - p[j] = ULONG_ZEROONE; if (p[j] != ULONG_ZEROONE) goto err; - p[j] = val; /* restore the original value. */ - } - return 0; +void memtest_non_destructive_swap(void *addr, size_t size) { + volatile unsigned long *p = addr; + size_t words = size / sizeof(unsigned long); + size_t j; -err: /* memory error detected. */ - p[j] = val; - return 1; + /* Swap */ + for (j = 0; j < words; j += 2) { + unsigned long a, b; + + a = p[j]; + b = p[j+1]; + p[j] = b; + p[j+1] = a; + } } void memtest(size_t megabytes, int passes) { From 3b71404d704ef5c3cc3c755d65a201380dea13c7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Nov 2012 14:20:08 +0100 Subject: [PATCH 0408/1752] Introduced the Build ID in INFO and --version output. The idea is to be able to identify a build in a unique way, so for instance after a bug report we can recognize that the build is the one of a popular Linux distribution and perform the debugging in the same environment. --- src/Makefile | 2 +- src/crc64.h | 8 ++++++++ src/debug.c | 2 +- src/mkreleasehdr.sh | 2 ++ src/redis-check-dump.c | 4 +--- src/redis.c | 7 +++++-- src/redis.h | 1 + src/release.c | 10 ++++++++++ src/rio.c | 3 +-- 9 files changed, 30 insertions(+), 9 deletions(-) create mode 100644 src/crc64.h diff --git a/src/Makefile b/src/Makefile index 204a2714843..9d0ec4eb85c 100644 --- a/src/Makefile +++ b/src/Makefile @@ -101,7 +101,7 @@ REDIS_SERVER_NAME= redis-server REDIS_SENTINEL_NAME= redis-sentinel REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o REDIS_CLI_NAME= redis-cli -REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o +REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o REDIS_BENCHMARK_NAME= redis-benchmark REDIS_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o REDIS_CHECK_DUMP_NAME= redis-check-dump diff --git a/src/crc64.h b/src/crc64.h new file mode 100644 index 00000000000..ab375d3f4f7 --- /dev/null +++ b/src/crc64.h @@ -0,0 +1,8 @@ +#ifndef CRC64_H +#define CRC64_H + +#include + +uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); + +#endif diff --git a/src/debug.c b/src/debug.c index 31cfac65ef9..7d6fdf97d47 100644 --- a/src/debug.c +++ b/src/debug.c @@ -29,6 +29,7 @@ #include "redis.h" #include "sha1.h" /* SHA1 is used for DEBUG DIGEST */ +#include "crc64.h" #include #include @@ -667,7 +668,6 @@ void logCurrentClient(void) { } #if defined(HAVE_PROC_MAPS) -uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); void memtest_non_destructive_invert(void *addr, size_t size); void memtest_non_destructive_swap(void *addr, size_t size); #define MEMTEST_MAX_REGIONS 128 diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index 30984160e7e..d07cf6ae058 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -1,9 +1,11 @@ #!/bin/sh GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` GIT_DIRTY=`git diff 2> /dev/null | wc -l` +BUILD_ID=`uname -n`"-"`date +%s` test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ (cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already uptodate echo "#define REDIS_GIT_SHA1 \"$GIT_SHA1\"" > release.h echo "#define REDIS_GIT_DIRTY \"$GIT_DIRTY\"" >> release.h +echo "#define REDIS_BUILD_ID \"$BUILD_ID\"" >> release.h touch release.c # Force recompile of release.c diff --git a/src/redis-check-dump.c b/src/redis-check-dump.c index 7efecb1a399..950655a024e 100644 --- a/src/redis-check-dump.c +++ b/src/redis-check-dump.c @@ -40,6 +40,7 @@ #include #include #include "lzf.h" +#include "crc64.h" /* Object types */ #define REDIS_STRING 0 @@ -140,9 +141,6 @@ static double R_Zero, R_PosInf, R_NegInf, R_Nan; /* store string types for output */ static char types[256][16]; -/* Prototypes */ -uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); - /* Return true if 't' is a valid object type. */ int checkType(unsigned char t) { /* In case a new object type is added, update the following diff --git a/src/redis.c b/src/redis.c index ff2d2dc14ee..5169f3f6915 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1868,6 +1868,7 @@ sds genRedisInfoString(char *section) { "redis_version:%s\r\n" "redis_git_sha1:%s\r\n" "redis_git_dirty:%d\r\n" + "redis_build_id:%llx\r\n" "redis_mode:%s\r\n" "os:%s %s %s\r\n" "arch_bits:%d\r\n" @@ -1882,6 +1883,7 @@ sds genRedisInfoString(char *section) { REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, + redisBuildId(), mode, name.sysname, name.release, name.machine, server.arch_bits, @@ -2417,12 +2419,13 @@ void daemonize(void) { } void version() { - printf("Redis server v=%s sha=%s:%d malloc=%s bits=%d\n", + printf("Redis server v=%s sha=%s:%d malloc=%s bits=%d build=%llx\n", REDIS_VERSION, redisGitSHA1(), atoi(redisGitDirty()) > 0, ZMALLOC_LIB, - sizeof(long) == 4 ? 32 : 64); + sizeof(long) == 4 ? 32 : 64, + redisBuildId()); exit(0); } diff --git a/src/redis.h b/src/redis.h index 8fd6f447d38..42edb46fe1b 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1046,6 +1046,7 @@ void scriptingInit(void); /* Git SHA1 */ char *redisGitSHA1(void); char *redisGitDirty(void); +uint64_t redisBuildId(void); /* Commands prototypes */ void authCommand(redisClient *c); diff --git a/src/release.c b/src/release.c index 46761448cec..34c3d813cc5 100644 --- a/src/release.c +++ b/src/release.c @@ -31,7 +31,11 @@ * small file is recompiled, as we access this information in all the other * files using this functions. */ +#include + #include "release.h" +#include "version.h" +#include "crc64.h" char *redisGitSHA1(void) { return REDIS_GIT_SHA1; @@ -40,3 +44,9 @@ char *redisGitSHA1(void) { char *redisGitDirty(void) { return REDIS_GIT_DIRTY; } + +uint64_t redisBuildId(void) { + char *buildid = REDIS_VERSION REDIS_BUILD_ID REDIS_GIT_DIRTY REDIS_GIT_SHA1; + + return crc64(0,(unsigned char*)buildid,strlen(buildid)); +} diff --git a/src/rio.c b/src/rio.c index 45bb8989698..d87d62fd791 100644 --- a/src/rio.c +++ b/src/rio.c @@ -50,8 +50,7 @@ #include #include "rio.h" #include "util.h" - -uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); +#include "crc64.h" /* Returns 1 or 0 for success/failure. */ static size_t rioBufferWrite(rio *r, const void *buf, size_t len) { From e34c14df1afa6b6903b6c5f536d0b45ae1296ec6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Nov 2012 16:12:14 +0100 Subject: [PATCH 0409/1752] Make an EXEC test more latency proof. --- tests/unit/multi.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 6f44767b502..f7a1f6468ec 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -63,13 +63,13 @@ start_server {tags {"multi"}} { r multi r set foo1 bar1 $rd config set maxmemory 1 + assert {[$rd read] eq {OK}} catch {r lpush mylist myvalue} $rd config set maxmemory 0 + assert {[$rd read] eq {OK}} r set foo2 bar2 catch {r exec} e assert_match {EXECABORT*} $e - assert {[$rd read] eq {OK}} - assert {[$rd read] eq {OK}} $rd close list [r exists foo1] [r exists foo2] } {0 0} From 2572bb13c62072f0c184a83c7530373b83d546ee Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Nov 2012 15:41:09 +0100 Subject: [PATCH 0410/1752] redis-benchmark: seed the PRNG with time() at startup. --- src/redis-benchmark.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 8d72573d5d5..7ab700d3fca 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -510,6 +511,7 @@ int main(int argc, const char **argv) { client c; + srandom(time(NULL)); signal(SIGHUP, SIG_IGN); signal(SIGPIPE, SIG_IGN); From ccc974d90be59daef2d2eab27920cb08df1f8132 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Nov 2012 15:41:26 +0100 Subject: [PATCH 0411/1752] SDIFF is now able to select between two algorithms for speed. SDIFF used an algorithm that was O(N) where N is the total number of elements of all the sets involved in the operation. The algorithm worked like that: ALGORITHM 1: 1) For the first set, add all the members to an auxiliary set. 2) For all the other sets, remove all the members of the set from the auxiliary set. So it is an O(N) algorithm where N is the total number of elements in all the sets involved in the diff operation. Cristobal Viedma suggested to modify the algorithm to the following: ALGORITHM 2: 1) Iterate all the elements of the first set. 2) For every element, check if the element also exists in all the other remaining sets. 3) Add the element to the auxiliary set only if it does not exist in any of the other sets. The complexity of this algorithm on the worst case is O(N*M) where N is the size of the first set and M the total number of sets involved in the operation. However when there are elements in common, with this algorithm we stop the computation for a given element as long as we find a duplicated element into another set. I (antirez) added an additional step to algorithm 2 to make it faster, that is to sort the set to subtract from the biggest to the smallest, so that it is more likely to find a duplicate in a larger sets that are checked before the smaller ones. WHAT IS BETTER? None of course, for instance if the first set is much larger than the other sets the second algorithm does a lot more work compared to the first algorithm. Similarly if the first set is much smaller than the other sets, the original algorithm will less work. So this commit makes Redis able to guess the number of operations required by each algorithm, and select the best at runtime according to the input received. However, since the second algorithm has better constant times and can do less work if there are duplicated elements, an advantage is given to the second algorithm. --- src/t_set.c | 113 ++++++++++++++++++++++++++++++++++------ tests/unit/type/set.tcl | 7 +-- 2 files changed, 101 insertions(+), 19 deletions(-) diff --git a/src/t_set.c b/src/t_set.c index 46a0c6ee502..01ac92aa468 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -565,6 +565,14 @@ int qsortCompareSetsByCardinality(const void *s1, const void *s2) { return setTypeSize(*(robj**)s1)-setTypeSize(*(robj**)s2); } +/* This is used by SDIFF and in this case we can receive NULL that should + * be handled as empty sets. */ +int qsortCompareSetsByRevCardinality(const void *s1, const void *s2) { + robj *o1 = *(robj**)s1, *o2 = *(robj**)s2; + + return (o2 ? setTypeSize(o2) : 0) - (o1 ? setTypeSize(o1) : 0); +} + void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, robj *dstkey) { robj **sets = zmalloc(sizeof(robj*)*setnum); setTypeIterator *si; @@ -712,6 +720,7 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * setTypeIterator *si; robj *ele, *dstset = NULL; int j, cardinality = 0; + int diff_algo = 1; for (j = 0; j < setnum; j++) { robj *setobj = dstkey ? @@ -728,34 +737,106 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * sets[j] = setobj; } + /* Select what DIFF algorithm to use. + * + * Algorithm 1 is O(N*M) where N is the size of the element first set + * and M the total number of sets. + * + * Algorithm 2 is O(N) where N is the total number of elements in all + * the sets. + * + * We compute what is the best bet with the current input here. */ + if (op == REDIS_OP_DIFF && sets[0]) { + long long algo_one_work = 0, algo_two_work = 0; + + for (j = 0; j < setnum; j++) { + if (sets[j] == NULL) continue; + + algo_one_work += setTypeSize(sets[0]); + algo_two_work += setTypeSize(sets[j]); + } + + /* Algorithm 1 has better constant times and performs less operations + * if there are elements in common. Give it some advantage. */ + algo_one_work /= 2; + diff_algo = (algo_one_work <= algo_two_work) ? 1 : 2; + + if (diff_algo == 1 && setnum > 1) { + /* With algorithm 1 it is better to order the sets to subtract + * by decreasing size, so that we are more likely to find + * duplicated elements ASAP. */ + qsort(sets+1,setnum-1,sizeof(robj*), + qsortCompareSetsByRevCardinality); + } + } + /* We need a temp set object to store our union. If the dstkey * is not NULL (that is, we are inside an SUNIONSTORE operation) then * this set object will be the resulting object to set into the target key*/ dstset = createIntsetObject(); - /* Iterate all the elements of all the sets, add every element a single - * time to the result set */ - for (j = 0; j < setnum; j++) { - if (op == REDIS_OP_DIFF && j == 0 && !sets[j]) break; /* result set is empty */ - if (!sets[j]) continue; /* non existing keys are like empty sets */ + if (op == REDIS_OP_UNION) { + /* Union is trivial, just add every element of every set to the + * temporary set. */ + for (j = 0; j < setnum; j++) { + if (!sets[j]) continue; /* non existing keys are like empty sets */ - si = setTypeInitIterator(sets[j]); + si = setTypeInitIterator(sets[j]); + while((ele = setTypeNextObject(si)) != NULL) { + if (setTypeAdd(dstset,ele)) cardinality++; + decrRefCount(ele); + } + setTypeReleaseIterator(si); + } + } else if (op == REDIS_OP_DIFF && sets[0] && diff_algo == 1) { + /* DIFF Algorithm 1: + * + * We perform the diff by iterating all the elements of the first set, + * and only adding it to the target set if the element does not exist + * into all the other sets. + * + * This way we perform at max N*M operations, where N is the size of + * the first set, and M the number of sets. */ + si = setTypeInitIterator(sets[0]); while((ele = setTypeNextObject(si)) != NULL) { - if (op == REDIS_OP_UNION || j == 0) { - if (setTypeAdd(dstset,ele)) { - cardinality++; - } - } else if (op == REDIS_OP_DIFF) { - if (setTypeRemove(dstset,ele)) { - cardinality--; - } + for (j = 1; j < setnum; j++) { + if (!sets[j]) continue; /* no key is an empty set. */ + if (setTypeIsMember(sets[j],ele)) break; + } + if (j == setnum) { + /* There is no other set with this element. Add it. */ + setTypeAdd(dstset,ele); + cardinality++; } decrRefCount(ele); } setTypeReleaseIterator(si); + } else if (op == REDIS_OP_DIFF && sets[0] && diff_algo == 2) { + /* DIFF Algorithm 2: + * + * Add all the elements of the first set to the auxiliary set. + * Then remove all the elements of all the next sets from it. + * + * This is O(N) where N is the sum of all the elements in every + * set. */ + for (j = 0; j < setnum; j++) { + if (!sets[j]) continue; /* non existing keys are like empty sets */ + + si = setTypeInitIterator(sets[j]); + while((ele = setTypeNextObject(si)) != NULL) { + if (j == 0) { + if (setTypeAdd(dstset,ele)) cardinality++; + } else { + if (setTypeRemove(dstset,ele)) cardinality--; + } + decrRefCount(ele); + } + setTypeReleaseIterator(si); - /* Exit when result set is empty. */ - if (op == REDIS_OP_DIFF && cardinality == 0) break; + /* Exit if result set is empty as any additional removal + * of elements will have no effect. */ + if (cardinality == 0) break; + } } /* Output the content of the resulting set, if not in STORE mode */ diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index b5d44ea9b7a..2234ee4fcc6 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -199,9 +199,10 @@ start_server { test "SDIFFSTORE with three sets - $type" { r sdiffstore setres set1 set4 set5 - # The type is determined by type of the first key to diff against. - # See the implementation for more information. - assert_encoding $type setres + # When we start with intsets, we should always end with intsets. + if {$type eq {intset}} { + assert_encoding intset setres + } assert_equal {1 2 3 4} [lsort [r smembers setres]] } } From c1eda786d65e5492f63eadc271d20af6ac55b76c Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Nov 2012 01:35:34 +0100 Subject: [PATCH 0412/1752] SDIFF fuzz test added. --- tests/unit/type/set.tcl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index 2234ee4fcc6..a77759e5d7e 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -214,6 +214,32 @@ start_server { r sdiff set1 set2 set3 } {} + test "SDIFF fuzzing" { + for {set j 0} {$j < 100} {incr j} { + unset -nocomplain s + array set s {} + set args {} + set num_sets [expr {[randomInt 10]+1}] + for {set i 0} {$i < $num_sets} {incr i} { + set num_elements [randomInt 100] + r del set_$i + lappend args set_$i + while {$num_elements} { + set ele [randomValue] + r sadd set_$i $ele + if {$i == 0} { + set s($ele) x + } else { + unset -nocomplain s($ele) + } + incr num_elements -1 + } + } + set result [lsort [r sdiff {*}$args]] + assert_equal $result [lsort [array names s]] + } + } + test "SINTER against non-set should throw error" { r set key1 x assert_error "WRONGTYPE*" {r sinter key1 noset} From 64e69c0d5f53087312261e08c3eea1774338cf15 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 1 Dec 2012 12:26:07 +0100 Subject: [PATCH 0413/1752] Client should not block multiple times on the same key. Sending a command like: BLPOP foo foo foo foo 0 Resulted into a crash before this commit since the client ended being inserted in the waiting list for this key multiple times. This resulted into the function handleClientsBlockedOnLists() to fail because we have code like that: if (de) { list *clients = dictGetVal(de); int numclients = listLength(clients); while(numclients--) { listNode *clientnode = listFirst(clients); /* server clients here... */ } } The code to serve clients used to remove the served client from the waiting list, so if a client is blocking multiple times, eventually the call to listFirst() will return NULL or worse will access random memory since the list may no longer exist as it is removed by the function unblockClientWaitingData() if there are no more clients waiting for this list. To avoid making the rest of the implementation more complex, this commit modifies blockForKeys() so that a client will be put just a single time into the waiting list for a given key. Since it is Saturday, I hope this fixes issue #801. --- src/t_list.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/t_list.c b/src/t_list.c index 7d7e83cbb70..1bd81c2e1c8 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -753,22 +753,33 @@ void rpoplpushCommand(redisClient *c) { /* Set a client in blocking mode for the specified key, with the specified * timeout */ void blockForKeys(redisClient *c, robj **keys, int numkeys, time_t timeout, robj *target) { + dict *added; dictEntry *de; list *l; - int j; + int j, i; c->bpop.keys = zmalloc(sizeof(robj*)*numkeys); - c->bpop.count = numkeys; c->bpop.timeout = timeout; c->bpop.target = target; - if (target != NULL) { - incrRefCount(target); - } + if (target != NULL) incrRefCount(target); + + /* Create a dictionary that we use to avoid adding duplicated keys + * in case the user calls something like: "BLPOP foo foo foo 0". + * The rest of the implementation is simpler if we know there are no + * duplications in the key waiting list. */ + added = dictCreate(&setDictType,NULL); + i = 0; /* The index for c->bpop.keys[...], we can't use the j loop + variable as the list of keys may have duplicated elements. */ for (j = 0; j < numkeys; j++) { + /* Add the key in the "added" dictionary to make sure there are + * no duplicated keys. */ + if (dictAdd(added,keys[j],NULL) != DICT_OK) continue; + incrRefCount(keys[j]); + /* Add the key in the client structure, to map clients -> keys */ - c->bpop.keys[j] = keys[j]; + c->bpop.keys[i++] = keys[j]; incrRefCount(keys[j]); /* And in the other "side", to map keys -> clients */ @@ -786,9 +797,12 @@ void blockForKeys(redisClient *c, robj **keys, int numkeys, time_t timeout, robj } listAddNodeTail(l,c); } + c->bpop.count = i; + /* Mark the client as a blocked client */ c->flags |= REDIS_BLOCKED; server.bpop_blocked_clients++; + dictRelease(added); } /* Unblock a client that's waiting in a blocking operation such as BLPOP */ From fbf3e33d18890990aa2a7e5347cf4d7543c49ee9 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 1 Dec 2012 23:07:09 +0100 Subject: [PATCH 0414/1752] Test: regression for issue #801. --- tests/unit/type/list.tcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/type/list.tcl b/tests/unit/type/list.tcl index 642762cff56..e665afc0a14 100644 --- a/tests/unit/type/list.tcl +++ b/tests/unit/type/list.tcl @@ -190,6 +190,27 @@ start_server { $rd read } {list b} + test "BLPOP with same key multiple times should work (issue #801)" { + set rd [redis_deferring_client] + r del list1 list2 + + # Data arriving after the BLPOP. + $rd blpop list1 list2 list2 list1 0 + r lpush list1 a + assert_equal [$rd read] {list1 a} + $rd blpop list1 list2 list2 list1 0 + r lpush list2 b + assert_equal [$rd read] {list2 b} + + # Data already there. + r lpush list1 a + r lpush list2 b + $rd blpop list1 list2 list2 list1 0 + assert_equal [$rd read] {list1 a} + $rd blpop list1 list2 list2 list1 0 + assert_equal [$rd read] {list2 b} + } + test "MULTI/EXEC is isolated from the point of view of BLPOP" { set rd [redis_deferring_client] r del list From 07a9f85405e90f6f4f3f1ddab2c024745ee0993d Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 2 Dec 2012 20:36:18 +0100 Subject: [PATCH 0415/1752] Blocking POP: use a dictionary to store keys clinet side. To store the keys we block for during a blocking pop operation, in the case the client is blocked for more data to arrive, we used a simple linear array of redis objects, in the blockingState structure: robj **keys; int count; However in order to fix issue #801 we also use a dictionary in order to avoid to end in the blocked clients queue for the same key multiple times with the same client. The dictionary was only temporary, just to avoid duplicates, but since we create / destroy it there is no point in doing this duplicated work, so this commit simply use a dictionary as the main structure to store the keys we are blocked for. So instead of the previous fields we now just have: dict *keys; This simplifies the code and reduces the work done by the server during a blocking POP operation. --- src/networking.c | 3 +-- src/redis.h | 3 +-- src/t_list.c | 50 +++++++++++++++++------------------------------- 3 files changed, 20 insertions(+), 36 deletions(-) diff --git a/src/networking.c b/src/networking.c index c0dd4d0d80b..a352ba7621c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -90,8 +90,7 @@ redisClient *createClient(int fd) { c->obuf_soft_limit_reached_time = 0; listSetFreeMethod(c->reply,decrRefCount); listSetDupMethod(c->reply,dupClientReplyValue); - c->bpop.keys = NULL; - c->bpop.count = 0; + c->bpop.keys = dictCreate(&setDictType,NULL); c->bpop.timeout = 0; c->bpop.target = NULL; c->io_keys = listCreate(); diff --git a/src/redis.h b/src/redis.h index 42edb46fe1b..92cc4480bcb 100644 --- a/src/redis.h +++ b/src/redis.h @@ -350,9 +350,8 @@ typedef struct multiState { } multiState; typedef struct blockingState { - robj **keys; /* The key we are waiting to terminate a blocking + dict *keys; /* The keys we are waiting to terminate a blocking * operation such as BLPOP. Otherwise NULL. */ - int count; /* Number of blocking keys */ time_t timeout; /* Blocking operation timeout. If UNIX current time * is >= timeout then the operation timed out. */ robj *target; /* The key that should receive the element, diff --git a/src/t_list.c b/src/t_list.c index 1bd81c2e1c8..50db6c536b7 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -753,33 +753,18 @@ void rpoplpushCommand(redisClient *c) { /* Set a client in blocking mode for the specified key, with the specified * timeout */ void blockForKeys(redisClient *c, robj **keys, int numkeys, time_t timeout, robj *target) { - dict *added; dictEntry *de; list *l; - int j, i; + int j; - c->bpop.keys = zmalloc(sizeof(robj*)*numkeys); c->bpop.timeout = timeout; c->bpop.target = target; if (target != NULL) incrRefCount(target); - /* Create a dictionary that we use to avoid adding duplicated keys - * in case the user calls something like: "BLPOP foo foo foo 0". - * The rest of the implementation is simpler if we know there are no - * duplications in the key waiting list. */ - added = dictCreate(&setDictType,NULL); - - i = 0; /* The index for c->bpop.keys[...], we can't use the j loop - variable as the list of keys may have duplicated elements. */ for (j = 0; j < numkeys; j++) { - /* Add the key in the "added" dictionary to make sure there are - * no duplicated keys. */ - if (dictAdd(added,keys[j],NULL) != DICT_OK) continue; - incrRefCount(keys[j]); - - /* Add the key in the client structure, to map clients -> keys */ - c->bpop.keys[i++] = keys[j]; + /* If the key already exists in the dict ignore it. */ + if (dictAdd(c->bpop.keys,keys[j],NULL) != DICT_OK) continue; incrRefCount(keys[j]); /* And in the other "side", to map keys -> clients */ @@ -797,39 +782,40 @@ void blockForKeys(redisClient *c, robj **keys, int numkeys, time_t timeout, robj } listAddNodeTail(l,c); } - c->bpop.count = i; /* Mark the client as a blocked client */ c->flags |= REDIS_BLOCKED; server.bpop_blocked_clients++; - dictRelease(added); } /* Unblock a client that's waiting in a blocking operation such as BLPOP */ void unblockClientWaitingData(redisClient *c) { dictEntry *de; + dictIterator *di; list *l; - int j; - redisAssertWithInfo(c,NULL,c->bpop.keys != NULL); + redisAssertWithInfo(c,NULL,dictSize(c->bpop.keys) != 0); + di = dictGetIterator(c->bpop.keys); /* The client may wait for multiple keys, so unblock it for every key. */ - for (j = 0; j < c->bpop.count; j++) { + while((de = dictNext(di)) != NULL) { + robj *key = dictGetKey(de); + /* Remove this client from the list of clients waiting for this key. */ - de = dictFind(c->db->blocking_keys,c->bpop.keys[j]); - redisAssertWithInfo(c,c->bpop.keys[j],de != NULL); - l = dictGetVal(de); + l = dictFetchValue(c->db->blocking_keys,key); + redisAssertWithInfo(c,key,l != NULL); listDelNode(l,listSearchKey(l,c)); /* If the list is empty we need to remove it to avoid wasting memory */ if (listLength(l) == 0) - dictDelete(c->db->blocking_keys,c->bpop.keys[j]); - decrRefCount(c->bpop.keys[j]); + dictDelete(c->db->blocking_keys,key); } + dictReleaseIterator(di); /* Cleanup the client structure */ - zfree(c->bpop.keys); - c->bpop.keys = NULL; - if (c->bpop.target) decrRefCount(c->bpop.target); - c->bpop.target = NULL; + dictEmpty(c->bpop.keys); + if (c->bpop.target) { + decrRefCount(c->bpop.target); + c->bpop.target = NULL; + } c->flags &= ~REDIS_BLOCKED; c->flags |= REDIS_UNBLOCKED; server.bpop_blocked_clients--; From e39735214d5554c1eb48aec3a449d6e21df7529b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 3 Dec 2012 12:06:38 +0100 Subject: [PATCH 0416/1752] Test: fixed osx "leaks" support in test. Due to changes in recent releases of osx leaks utility, the osx leak detection no longer worked. Now it is fixed in a way that should be backward compatible. --- tests/support/server.tcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/support/server.tcl b/tests/support/server.tcl index 2c2665b6449..acbfe73cff4 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -38,7 +38,9 @@ proc kill_server config { if {[string match {*Darwin*} [exec uname -a]]} { tags {"leaks"} { test "Check for memory leaks (pid $pid)" { - exec leaks $pid + set output {0 leaks} + catch {exec leaks $pid} output + set output } {*0 leaks*} } } From 124cb6dda1676f248f293cea2a824fe97ec68ef2 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 3 Dec 2012 12:12:53 +0100 Subject: [PATCH 0417/1752] Memory leak fixed: release client's bpop->keys dictionary. Refactoring performed after issue #801 resolution (see commit 2f87cf8b0162bd9d78c3a89860c0971cd71d39db) introduced a memory leak that is fixed by this commit. I simply forgot to free the new allocated dictionary in the client structure trusting the output of "make test" on OSX. However due to changes in the "leaks" utility the test was no longer testing memory leaks. This problem was also fixed. Fortunately the CI test running at ci.redis.io spotted the bug in the valgrind run. The leak never ended into a stable release. --- src/networking.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/networking.c b/src/networking.c index a352ba7621c..4365bc8ef0f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -605,6 +605,7 @@ void freeClient(redisClient *c) { c->querybuf = NULL; if (c->flags & REDIS_BLOCKED) unblockClientWaitingData(c); + dictRelease(c->bpop.keys); /* UNWATCH all the keys */ unwatchAllKeys(c); From 17e31eb03ac36c4a7f63b08f1be031c33438281e Mon Sep 17 00:00:00 2001 From: "Brian J. McManus" Date: Sun, 2 Dec 2012 21:46:37 -0700 Subject: [PATCH 0418/1752] Issue 804 Add Default-Start and Default-Stop LSB tags for RedHat startup and update-rc.d compatability. --- utils/install_server.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/install_server.sh b/utils/install_server.sh index 789a97803c2..c5ca944e593 100755 --- a/utils/install_server.sh +++ b/utils/install_server.sh @@ -151,6 +151,8 @@ REDIS_CHKCONFIG_INFO=\ # Provides: redis_6379\n # Required-Start: $network $local_fs $remote_fs\n # Required-Stop: $network $local_fs $remote_fs\n +# Default-Start: 2 3 4 5\n +# Default-Stop: 0 1 6\n # Should-Start: $syslog $named\n # Should-Stop: $syslog $named\n # Short-Description: start and stop redis_6379\n From bbee226ecd03dff7da57a2366d72be9065b45cfb Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 11 Dec 2012 17:01:00 +0100 Subject: [PATCH 0419/1752] Fix config.h endianess detection to work on Linux / PPC64. Config.h performs endianess detection including OS-specific headers to define the endianess macros, or when this is not possible, checking the processor type via ifdefs. Sometimes when the OS-specific macro is included, only __BYTE_ORDER is defined, while BYTE_ORDER remains undefined. There is code at the end of config.h endianess detection in order to define the macros without the underscore, but it was not working correctly. This commit fixes endianess detection fixing Redis on Linux / PPC64 and possibly other systems. --- src/config.h | 17 +++++++++++++++-- src/endianconv.h | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/config.h b/src/config.h index 97c599745bd..472f10a3a4b 100644 --- a/src/config.h +++ b/src/config.h @@ -139,13 +139,27 @@ #endif /* BSD */ #endif /* BYTE_ORDER */ -#if defined(__BYTE_ORDER) && !defined(BYTE_ORDER) +/* Sometimes after including an OS-specific header that defines the + * endianess we end with __BYTE_ORDER but not with BYTE_ORDER that is what + * the Redis code uses. In this case let's define everything without the + * underscores. */ +#ifndef BYTE_ORDER +#ifdef __BYTE_ORDER +#if defined(__LITTLE_ENDIAN) && defined(__BIG_ENDIAN) +#ifndef LITTLE_ENDIAN +#define LITTLE_ENDIAN __LITTLE_ENDIAN +#endif +#ifndef BIG_ENDIAN +#define BIG_ENDIAN __BIG_ENDIAN +#endif #if (__BYTE_ORDER == __LITTLE_ENDIAN) #define BYTE_ORDER LITTLE_ENDIAN #else #define BYTE_ORDER BIG_ENDIAN #endif #endif +#endif +#endif #if !defined(BYTE_ORDER) || \ (BYTE_ORDER != BIG_ENDIAN && BYTE_ORDER != LITTLE_ENDIAN) @@ -164,5 +178,4 @@ #endif #endif - #endif diff --git a/src/endianconv.h b/src/endianconv.h index f76e0e6b3eb..7afe61c62d8 100644 --- a/src/endianconv.h +++ b/src/endianconv.h @@ -33,6 +33,7 @@ #ifndef __ENDIANCONV_H #define __ENDIANCONV_H +#include "config.h" #include void memrev16(void *p); From 5207c03c498d324ceaad799d9a9fd4675cdbd91e Mon Sep 17 00:00:00 2001 From: Patrick TJ McPhee Date: Wed, 12 Dec 2012 10:49:12 -0500 Subject: [PATCH 0420/1752] Define _XOPEN_SOURCE appropriately on NetBSD. --- deps/hiredis/fmacros.h | 2 +- src/fmacros.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/hiredis/fmacros.h b/deps/hiredis/fmacros.h index 21cd9cfee3d..96324eb4980 100644 --- a/deps/hiredis/fmacros.h +++ b/deps/hiredis/fmacros.h @@ -7,7 +7,7 @@ #if defined(__sun__) #define _POSIX_C_SOURCE 200112L -#elif defined(__linux__) +#elif defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) #define _XOPEN_SOURCE 600 #else #define _XOPEN_SOURCE diff --git a/src/fmacros.h b/src/fmacros.h index 6f3c4b4e8b2..a6cf3578c13 100644 --- a/src/fmacros.h +++ b/src/fmacros.h @@ -36,7 +36,7 @@ #define _GNU_SOURCE #endif -#if defined(__linux__) || defined(__OpenBSD__) +#if defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) #define _XOPEN_SOURCE 700 #else #define _XOPEN_SOURCE From a6d117b6c05740a8ebfe8a4cc5024c0f98889844 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 14 Dec 2012 17:10:40 +0100 Subject: [PATCH 0421/1752] serverCron() frequency is now a runtime parameter (was REDIS_HZ). REDIS_HZ is the frequency our serverCron() function is called with. A more frequent call to this function results into less latency when the server is trying to handle very expansive background operations like mass expires of a lot of keys at the same time. Redis 2.4 used to have an HZ of 10. This was good enough with almost every setup, but the incremental key expiration algorithm was working a bit better under *extreme* pressure when HZ was set to 100 for Redis 2.6. However for most users a latency spike of 30 milliseconds when million of keys are expiring at the same time is acceptable, on the other hand a default HZ of 100 in Redis 2.6 was causing idle instances to use some CPU time compared to Redis 2.4. The CPU usage was in the order of 0.3% for an idle instance, however this is a shame as more energy is consumed by the server, if not important resources. This commit introduces HZ as a runtime parameter, that can be queried by INFO or CONFIG GET, and can be modified with CONFIG SET. At the same time the default frequency is set back to 10. In this way we default to a sane value of 10, but allows users to easily switch to values up to 500 for near real-time applications if needed and if they are willing to pay this small CPU usage penalty. --- redis.conf | 17 +++++++++++++++++ src/config.c | 11 +++++++++++ src/debug.c | 2 +- src/redis.c | 19 +++++++++++-------- src/redis.h | 9 ++++++--- src/replication.c | 2 +- 6 files changed, 47 insertions(+), 13 deletions(-) diff --git a/redis.conf b/redis.conf index 97aea334b4e..b8f0c486696 100644 --- a/redis.conf +++ b/redis.conf @@ -529,6 +529,23 @@ client-output-buffer-limit normal 0 0 0 client-output-buffer-limit slave 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 +# Redis calls an internal function to perform many background tasks, like +# closing connections of clients in timeot, purging expired keys that are +# never requested, and so forth. +# +# Not all tasks are perforemd with the same frequency, but Redis checks for +# tasks to perform accordingly to the specified "hz" value. +# +# By default "hz" is set to 10. Raising the value will use more CPU when +# Redis is idle, but at the same time will make Redis more responsive when +# there are many keys expiring at the same time, and timeouts may be +# handled with more precision. +# +# The range is between 1 and 500, however a value over 100 is usually not +# a good idea. Most users should use the default of 10 and raise this up to +# 100 only in environments where very low latency is required. +hz 10 + ################################## INCLUDES ################################### # Include one or more other config files here. This is useful if you diff --git a/src/config.c b/src/config.c index 7e8f8a2ba94..1765b7d908e 100644 --- a/src/config.c +++ b/src/config.c @@ -256,6 +256,10 @@ void loadServerConfigFromString(char *config) { if ((server.daemonize = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"hz") && argc == 2) { + server.hz = atoi(argv[1]); + if (server.hz < REDIS_MIN_HZ) server.hz = REDIS_MIN_HZ; + if (server.hz > REDIS_MAX_HZ) server.hz = REDIS_MAX_HZ; } else if (!strcasecmp(argv[0],"appendonly") && argc == 2) { int yes; @@ -475,6 +479,12 @@ void configSetCommand(redisClient *c) { } freeMemoryIfNeeded(); } + } else if (!strcasecmp(c->argv[2]->ptr,"hz")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.hz = (int) ll; + if (server.hz < REDIS_MIN_HZ) server.hz = REDIS_MIN_HZ; + if (server.hz > REDIS_MAX_HZ) server.hz = REDIS_MAX_HZ; } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory-policy")) { if (!strcasecmp(o->ptr,"volatile-lru")) { server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU; @@ -791,6 +801,7 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); config_get_numerical_field("slave-priority",server.slave_priority); + config_get_numerical_field("hz",server.hz); /* Bool (yes/no) values */ config_get_bool_field("no-appendfsync-on-rewrite", diff --git a/src/debug.c b/src/debug.c index 7d6fdf97d47..39ddb432746 100644 --- a/src/debug.c +++ b/src/debug.c @@ -891,7 +891,7 @@ void enableWatchdog(int period) { /* If the configured period is smaller than twice the timer period, it is * too short for the software watchdog to work reliably. Fix it now * if needed. */ - min_period = (1000/REDIS_HZ)*2; + min_period = (1000/server.hz)*2; if (period < min_period) period = min_period; watchdogScheduleSignal(period); /* Adjust the current timer. */ server.watchdog_period = period; diff --git a/src/redis.c b/src/redis.c index 5169f3f6915..329de459f7b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -627,9 +627,9 @@ void activeExpireCycle(void) { /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time * per iteration. Since this function gets called with a frequency of - * REDIS_HZ times per second, the following is the max amount of + * server.hz times per second, the following is the max amount of * microseconds we can spend in this function. */ - timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/REDIS_HZ/100; + timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/server.hz/100; if (timelimit <= 0) timelimit = 1; for (j = 0; j < server.dbnum; j++) { @@ -762,13 +762,13 @@ int clientsCronResizeQueryBuffer(redisClient *c) { } void clientsCron(void) { - /* Make sure to process at least 1/(REDIS_HZ*10) of clients per call. - * Since this function is called REDIS_HZ times per second we are sure that + /* Make sure to process at least 1/(server.hz*10) of clients per call. + * Since this function is called server.hz times per second we are sure that * in the worst case we process all the clients in 10 seconds. * In normal conditions (a reasonable number of clients) we process * all the clients in a shorter time. */ int numclients = listLength(server.clients); - int iterations = numclients/(REDIS_HZ*10); + int iterations = numclients/(server.hz*10); if (iterations < 50) iterations = (numclients < 50) ? numclients : 50; @@ -790,7 +790,7 @@ void clientsCron(void) { } } -/* This is our timer interrupt, called REDIS_HZ times per second. +/* This is our timer interrupt, called server.hz times per second. * Here is where we do a number of things that need to be done asynchronously. * For instance: * @@ -804,7 +804,7 @@ void clientsCron(void) { * - Replication reconnection. * - Many more... * - * Everything directly called here will be called REDIS_HZ times per second, + * Everything directly called here will be called server.hz times per second, * so in order to throttle execution of things we want to do less frequently * a macro is used: run_with_period(milliseconds) { .... } */ @@ -976,7 +976,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } server.cronloops++; - return 1000/REDIS_HZ; + return 1000/server.hz; } /* This function gets called every time Redis is entering the @@ -1082,6 +1082,7 @@ void createSharedObjects(void) { void initServerConfig() { getRandomHexChars(server.runid,REDIS_RUN_ID_SIZE); + server.hz = REDIS_DEFAULT_HZ; server.runid[REDIS_RUN_ID_SIZE] = '\0'; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = REDIS_SERVERPORT; @@ -1879,6 +1880,7 @@ sds genRedisInfoString(char *section) { "tcp_port:%d\r\n" "uptime_in_seconds:%ld\r\n" "uptime_in_days:%ld\r\n" + "hz:%d\r\n" "lru_clock:%ld\r\n", REDIS_VERSION, redisGitSHA1(), @@ -1898,6 +1900,7 @@ sds genRedisInfoString(char *section) { server.port, uptime, uptime/(3600*24), + server.hz, (unsigned long) server.lruclock); } diff --git a/src/redis.h b/src/redis.h index 92cc4480bcb..a9c3eb66a04 100644 --- a/src/redis.h +++ b/src/redis.h @@ -67,7 +67,9 @@ #define REDIS_ERR -1 /* Static server configuration */ -#define REDIS_HZ 100 /* Time interrupt calls/sec. */ +#define REDIS_DEFAULT_HZ 10 /* Time interrupt calls/sec. */ +#define REDIS_MIN_HZ 1 +#define REDIS_MAX_HZ 500 #define REDIS_SERVERPORT 6379 /* TCP port */ #define REDIS_MAXIDLETIME 0 /* default client timeout: infinite */ #define REDIS_DEFAULT_DBNUM 16 @@ -291,8 +293,8 @@ /* Using the following macro you can run code inside serverCron() with the * specified period, specified in milliseconds. - * The actual resolution depends on REDIS_HZ. */ -#define run_with_period(_ms_) if (!(server.cronloops%((_ms_)/(1000/REDIS_HZ)))) + * The actual resolution depends on server.hz. */ +#define run_with_period(_ms_) if ((_ms_ <= 1000/server.hz) || !(server.cronloops%((_ms_)/(1000/server.hz)))) /* We can print the stacktrace, so our assert is defined this way: */ #define redisAssertWithInfo(_c,_o,_e) ((_e)?(void)0 : (_redisAssertWithInfo(_c,_o,#_e,__FILE__,__LINE__),_exit(1))) @@ -493,6 +495,7 @@ typedef struct redisOpArray { struct redisServer { /* General */ + int hz; /* serverCron() calls frequency in hertz */ redisDb *db; dict *commands; /* Command table hash table */ aeEventLoop *el; diff --git a/src/replication.c b/src/replication.c index 720cd4c1959..36afd83c59a 100644 --- a/src/replication.c +++ b/src/replication.c @@ -778,7 +778,7 @@ void replicationCron(void) { * So slaves can implement an explicit timeout to masters, and will * be able to detect a link disconnection even if the TCP connection * will not actually go down. */ - if (!(server.cronloops % (server.repl_ping_slave_period * REDIS_HZ))) { + if (!(server.cronloops % (server.repl_ping_slave_period * server.hz))) { listIter li; listNode *ln; From 6fa2610ac93c75f8327547b91c8ccb2e3f55fb85 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 17 Dec 2012 11:11:09 +0100 Subject: [PATCH 0422/1752] Added missing license and copyright in deps/hiredis. --- deps/hiredis/adapters/ae.h | 30 ++++++++++++++++++++++++++++++ deps/hiredis/adapters/libev.h | 30 ++++++++++++++++++++++++++++++ deps/hiredis/adapters/libevent.h | 30 ++++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+) diff --git a/deps/hiredis/adapters/ae.h b/deps/hiredis/adapters/ae.h index 65235f802aa..5c551c2ed74 100644 --- a/deps/hiredis/adapters/ae.h +++ b/deps/hiredis/adapters/ae.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __HIREDIS_AE_H__ #define __HIREDIS_AE_H__ #include diff --git a/deps/hiredis/adapters/libev.h b/deps/hiredis/adapters/libev.h index 534d7436017..2bf8d521fc2 100644 --- a/deps/hiredis/adapters/libev.h +++ b/deps/hiredis/adapters/libev.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __HIREDIS_LIBEV_H__ #define __HIREDIS_LIBEV_H__ #include diff --git a/deps/hiredis/adapters/libevent.h b/deps/hiredis/adapters/libevent.h index 4055ec0f112..1c2b271bbc1 100644 --- a/deps/hiredis/adapters/libevent.h +++ b/deps/hiredis/adapters/libevent.h @@ -1,3 +1,33 @@ +/* + * Copyright (c) 2010-2011, Pieter Noordhuis + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + #ifndef __HIREDIS_LIBEVENT_H__ #define __HIREDIS_LIBEVENT_H__ #include From 61fb0aacf53b45a3fa22868c3c40ab94c4927d48 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 17 Dec 2012 11:13:54 +0100 Subject: [PATCH 0423/1752] CONTRIBUTING updated with request to add BSD license. --- CONTRIBUTING | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING b/CONTRIBUTING index 17078749d15..06e1e9660c0 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -2,7 +2,8 @@ Note: by contributing code to the Redis project in any form, including sending a pull request via Github, a code fragment or patch via private email or public discussion groups, you agree to release your code under the terms of the BSD license that you can find in the COPYING file included in the Redis -source distribution. +source distribution. You will include BSD license in the COPYING file within +each source file that you contribute. # IMPORTANT: HOW TO USE REDIS GITHUB ISSUES From 858b5576ac70702e5b3a0ca530753d864d901e53 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Dec 2012 15:20:55 +0100 Subject: [PATCH 0424/1752] Fix overflow in mstime() in redis-cli and benchmark. The problem does not exist in the Redis server implementation of mstime() but is only limited to redis-cli and redis-benchmark. Thix fixes issue #839. --- src/redis-benchmark.c | 2 +- src/redis-cli.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 7ab700d3fca..69c740242dd 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -107,7 +107,7 @@ static long long mstime(void) { long long mst; gettimeofday(&tv, NULL); - mst = ((long)tv.tv_sec)*1000; + mst = ((long long)tv.tv_sec)*1000; mst += tv.tv_usec/1000; return mst; } diff --git a/src/redis-cli.c b/src/redis-cli.c index e8c6be5e0be..3969fbab58c 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -95,7 +95,7 @@ static long long mstime(void) { long long mst; gettimeofday(&tv, NULL); - mst = ((long)tv.tv_sec)*1000; + mst = ((long long)tv.tv_sec)*1000; mst += tv.tv_usec/1000; return mst; } From 6570d2f89e00aafc9b3ec2379699c4c44866ea49 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Jan 2013 14:18:03 +0100 Subject: [PATCH 0425/1752] ae.c: set errno when error is not a failing syscall. In this way the caller is able to perform better error checking or to use strerror() without the risk of meaningless error messages being displayed. --- src/ae.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/ae.c b/src/ae.c index d2faed32630..90be4e28f41 100644 --- a/src/ae.c +++ b/src/ae.c @@ -38,6 +38,7 @@ #include #include #include +#include #include "ae.h" #include "zmalloc.h" @@ -104,7 +105,10 @@ void aeStop(aeEventLoop *eventLoop) { int aeCreateFileEvent(aeEventLoop *eventLoop, int fd, int mask, aeFileProc *proc, void *clientData) { - if (fd >= eventLoop->setsize) return AE_ERR; + if (fd >= eventLoop->setsize) { + errno = ERANGE; + return AE_ERR; + } aeFileEvent *fe = &eventLoop->events[fd]; if (aeApiAddEvent(eventLoop, fd, mask) == -1) From cefff3a1b38684016fb865575cb1cd89c1974683 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 3 Jan 2013 14:22:55 +0100 Subject: [PATCH 0426/1752] Better error reporting when fd event creation fails. --- src/networking.c | 4 +++- src/replication.c | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 4365bc8ef0f..c23939c5c1b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -517,7 +517,9 @@ void copyClientOutputBuffer(redisClient *dst, redisClient *src) { static void acceptCommonHandler(int fd, int flags) { redisClient *c; if ((c = createClient(fd)) == NULL) { - redisLog(REDIS_WARNING,"Error allocating resources for the client"); + redisLog(REDIS_WARNING, + "Error registering fd event for the new client: %s (fd=%d)", + strerror(errno),fd); close(fd); /* May be already closed, just ignore errors */ return; } diff --git a/src/replication.c b/src/replication.c index 36afd83c59a..a51e3f56f74 100644 --- a/src/replication.c +++ b/src/replication.c @@ -636,7 +636,9 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { if (aeCreateFileEvent(server.el,fd, AE_READABLE,readSyncBulkPayload,NULL) == AE_ERR) { - redisLog(REDIS_WARNING,"Can't create readable event for SYNC"); + redisLog(REDIS_WARNING, + "Can't create readable event for SYNC: %s (fd=%d)", + strerror(errno),fd); goto error; } From 414e06653ca9c9f3bca3392a1d5b7cc07c2a2cc7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Jan 2013 10:46:05 +0100 Subject: [PATCH 0427/1752] Multiple fixes for EVAL (issue #872). 1) The event handler was no restored after a timeout condition if the command was eventually executed with success. 2) The command was not converted to EVAL in case of errors in the middle of the execution. 3) Terrible duplication of code without any apparent reason. --- src/scripting.c | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index d614f42a13f..46301eb1130 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -787,7 +787,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { lua_State *lua = server.lua; char funcname[43]; long long numkeys; - int delhook = 0; + int delhook = 0, err; /* We want the same PRNG sequence at every call so that our PRNG is * not affected by external state. */ @@ -869,30 +869,31 @@ void evalGenericCommand(redisClient *c, int evalsha) { /* At this point whatever this script was never seen before or if it was * already defined, we can call it. We have zero arguments and expect * a single return value. */ - if (lua_pcall(lua,0,1,0)) { - if (delhook) lua_sethook(lua,luaMaskCountHook,0,0); /* Disable hook */ - if (server.lua_timedout) { - server.lua_timedout = 0; - /* Restore the readable handler that was unregistered when the - * script timeout was detected. */ - aeCreateFileEvent(server.el,c->fd,AE_READABLE, - readQueryFromClient,c); - } - server.lua_caller = NULL; - selectDb(c,server.lua_client->db->id); /* set DB ID from Lua client */ - addReplyErrorFormat(c,"Error running script (call to %s): %s\n", - funcname, lua_tostring(lua,-1)); - lua_pop(lua,1); - lua_gc(lua,LUA_GCCOLLECT,0); - return; - } + err = lua_pcall(lua,0,1,0); + + /* Perform some cleanup that we need to do both on error and success. */ if (delhook) lua_sethook(lua,luaMaskCountHook,0,0); /* Disable hook */ - server.lua_timedout = 0; + if (server.lua_timedout) { + server.lua_timedout = 0; + /* Restore the readable handler that was unregistered when the + * script timeout was detected. */ + aeCreateFileEvent(server.el,c->fd,AE_READABLE, + readQueryFromClient,c); + } server.lua_caller = NULL; selectDb(c,server.lua_client->db->id); /* set DB ID from Lua client */ - luaReplyToRedisReply(c,lua); lua_gc(lua,LUA_GCSTEP,1); + if (err) { + addReplyErrorFormat(c,"Error running script (call to %s): %s\n", + funcname, lua_tostring(lua,-1)); + lua_pop(lua,1); /* Consume the Lua reply. */ + } else { + /* On success convert the Lua return value into Redis protocol, and + * send it to * the client. */ + luaReplyToRedisReply(c,lua); + } + /* If we have slaves attached we want to replicate this command as * EVAL instead of EVALSHA. We do this also in the AOF as currently there * is no easy way to propagate a command in a different way in the AOF From e2e36cf19b67b564988126fe737d6c9c8a5592b3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Jan 2013 11:10:31 +0100 Subject: [PATCH 0428/1752] Test: added regression for issue #872. --- tests/unit/scripting.tcl | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index f1df11f3c57..3dbfc902bc2 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -297,6 +297,12 @@ start_server {tags {"scripting"}} { assert_equal [r ping] "PONG" } + test {Timedout script link is still usable after Lua returns} { + r config set lua-time-limit 10 + r eval {for i=1,100000 do redis.call('ping') end return 'ok'} 0 + r ping + } {PONG} + test {Timedout scripts that modified data can't be killed by SCRIPT KILL} { set rd [redis_deferring_client] r config set lua-time-limit 10 @@ -310,6 +316,8 @@ start_server {tags {"scripting"}} { assert_match {BUSY*} $e } + # Note: keep this test at the end of this server stanza because it + # kills the server. test {SHUTDOWN NOSAVE can kill a timedout script anyway} { # The server sould be still unresponding to normal commands. catch {r ping} e @@ -323,9 +331,16 @@ start_server {tags {"scripting"}} { start_server {tags {"scripting repl"}} { start_server {} { - test {Before the slave connects we issue an EVAL command} { + test {Before the slave connects we issue two EVAL commands} { + # One with an error, but still executing a command. + # SHA is: 6e8bd6bdccbe78899e3cc06b31b6dbf4324c2e56 + catch { + r eval {redis.call('incr','x'); redis.call('nonexisting')} 0 + } + # One command is correct: + # SHA is: ae3477e27be955de7e1bc9adfdca626b478d3cb2 r eval {return redis.call('incr','x')} 0 - } {1} + } {2} test {Connect a slave to the main instance} { r -1 slaveof [srv 0 host] [srv 0 port] @@ -337,15 +352,20 @@ start_server {tags {"scripting repl"}} { } } - test {Now use EVALSHA against the master} { + test {Now use EVALSHA against the master, with both SHAs} { + # The server should replicate successful and unsuccessful + # commands as EVAL instead of EVALSHA. + catch { + r evalsha 6e8bd6bdccbe78899e3cc06b31b6dbf4324c2e56 0 + } r evalsha ae3477e27be955de7e1bc9adfdca626b478d3cb2 0 - } {2} + } {4} - test {If EVALSHA was replicated as EVAL the slave should be ok} { + test {If EVALSHA was replicated as EVAL, 'x' should be '4'} { wait_for_condition 50 100 { - [r -1 get x] eq {2} + [r -1 get x] eq {4} } else { - fail "Expected 2 in x, but value is '[r -1 get x]'" + fail "Expected 4 in x, but value is '[r -1 get x]'" } } From 8a70007f9e3892425fe26aa76e0bea734203d4e7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 10 Jan 2013 11:19:40 +0100 Subject: [PATCH 0429/1752] Comment in the call() function clarified a bit. --- src/redis.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 329de459f7b..2ae567ddf5c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1529,8 +1529,9 @@ void call(redisClient *c, int flags) { if (flags != REDIS_PROPAGATE_NONE) propagate(c->cmd,c->db->id,c->argv,c->argc,flags); } - /* Commands such as LPUSH or BRPOPLPUSH may propagate an additional - * PUSH command. */ + + /* Handle the alsoPropagate() API to handle commands that want to propagate + * multiple separated commands. */ if (server.also_propagate.numops) { int j; redisOp *rop; From 64ee9c861fd7b887122ff65221d0789a6c1493c6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 Jan 2013 23:51:12 +0100 Subject: [PATCH 0430/1752] Makefile.dep updated. --- src/Makefile.dep | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/src/Makefile.dep b/src/Makefile.dep index 5331189f1bc..4e07a563d78 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -1,6 +1,7 @@ adlist.o: adlist.c adlist.h zmalloc.h ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c ae_epoll.o: ae_epoll.c +ae_evport.o: ae_evport.c ae_kqueue.o: ae_kqueue.c ae_select.o: ae_select.c anet.o: anet.c fmacros.h anet.h @@ -10,6 +11,9 @@ aof.o: aof.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ bio.o: bio.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h bio.h +bitops.o: bitops.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h config.o: config.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h @@ -19,13 +23,13 @@ db.o: db.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ziplist.h intset.h version.h util.h rdb.h rio.h debug.o: debug.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ - ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h + ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h crc64.h bio.h dict.o: dict.c fmacros.h dict.h zmalloc.h endianconv.o: endianconv.c -intset.o: intset.c intset.h zmalloc.h endianconv.h +intset.o: intset.c intset.h zmalloc.h endianconv.h config.h lzf_c.o: lzf_c.c lzfP.h lzf_d.o: lzf_d.c lzfP.h -memtest.o: memtest.c +memtest.o: memtest.c config.h migrate.o: migrate.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h endianconv.h @@ -51,25 +55,30 @@ rdb.o: rdb.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ redis-benchmark.o: redis-benchmark.c fmacros.h ae.h \ ../deps/hiredis/hiredis.h sds.h adlist.h zmalloc.h redis-check-aof.o: redis-check-aof.c fmacros.h config.h -redis-check-dump.o: redis-check-dump.c lzf.h +redis-check-dump.o: redis-check-dump.c lzf.h crc64.h redis-cli.o: redis-cli.c fmacros.h version.h ../deps/hiredis/hiredis.h \ - sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h + sds.h zmalloc.h ../deps/linenoise/linenoise.h help.h anet.h ae.h redis.o: redis.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h slowlog.h bio.h \ asciilogo.h -release.o: release.c release.h +release.o: release.c release.h version.h crc64.h replication.o: replication.c redis.h fmacros.h config.h \ ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ rio.h -rio.o: rio.c fmacros.h rio.h sds.h util.h +rio.o: rio.c fmacros.h rio.h sds.h util.h crc64.h scripting.o: scripting.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h rand.h \ ../deps/lua/src/lauxlib.h ../deps/lua/src/lua.h \ ../deps/lua/src/lualib.h sds.o: sds.c sds.h zmalloc.h +sentinel.o: sentinel.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h \ + ../deps/hiredis/hiredis.h ../deps/hiredis/async.h \ + ../deps/hiredis/hiredis.h sha1.o: sha1.c sha1.h config.h slowlog.o: slowlog.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ @@ -96,6 +105,6 @@ t_zset.o: t_zset.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h util.o: util.c fmacros.h util.h -ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h -zipmap.o: zipmap.c zmalloc.h endianconv.h +ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h config.h +zipmap.o: zipmap.c zmalloc.h endianconv.h config.h zmalloc.o: zmalloc.c config.h zmalloc.h From aa9497fdb3f0f38eb0f0e51c183bc4641a89b07a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 14 Jan 2013 11:39:54 +0100 Subject: [PATCH 0431/1752] Undo slave-master handshake when SLAVEOF sets a new slave. Issue #828 shows how Redis was not correctly undoing a non-blocking connection attempt with the previous master when the master was set to a new address using the SLAVEOF command. This was also a result of lack of refactoring, so now there is a function to cancel the non blocking handshake with the master. The new function is now used when SLAVEOF NO ONE is called or when SLAVEOF is used to set the master to a different address. --- src/replication.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/replication.c b/src/replication.c index a51e3f56f74..15fc71e7dd0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -695,6 +695,27 @@ void undoConnectWithMaster(void) { server.repl_state = REDIS_REPL_CONNECT; } +/* This function aborts a non blocking replication attempt if there is one + * in progress, by canceling the non-blocking connect attempt or + * the initial bulk transfer. + * + * If there was a replication handshake in progress 1 is returned and + * the replication state (server.repl_state) set to REDIS_REPL_CONNECT. + * + * Otherwise zero is returned and no operation is perforemd at all. */ +int cancelReplicationHandshake(void) { + if (server.repl_state == REDIS_REPL_TRANSFER) { + replicationAbortSyncTransfer(); + } else if (server.repl_state == REDIS_REPL_CONNECTING || + server.repl_state == REDIS_REPL_RECEIVE_PONG) + { + undoConnectWithMaster(); + } else { + return 0; + } + return 1; +} + void slaveofCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"no") && !strcasecmp(c->argv[2]->ptr,"one")) { @@ -702,11 +723,7 @@ void slaveofCommand(redisClient *c) { sdsfree(server.masterhost); server.masterhost = NULL; if (server.master) freeClient(server.master); - if (server.repl_state == REDIS_REPL_TRANSFER) - replicationAbortSyncTransfer(); - else if (server.repl_state == REDIS_REPL_CONNECTING || - server.repl_state == REDIS_REPL_RECEIVE_PONG) - undoConnectWithMaster(); + cancelReplicationHandshake(); server.repl_state = REDIS_REPL_NONE; redisLog(REDIS_NOTICE,"MASTER MODE enabled (user request)"); } @@ -730,8 +747,7 @@ void slaveofCommand(redisClient *c) { server.masterport = port; if (server.master) freeClient(server.master); disconnectSlaves(); /* Force our slaves to resync with us as well. */ - if (server.repl_state == REDIS_REPL_TRANSFER) - replicationAbortSyncTransfer(); + cancelReplicationHandshake(); server.repl_state = REDIS_REPL_CONNECT; redisLog(REDIS_NOTICE,"SLAVE OF %s:%d enabled (user request)", server.masterhost, server.masterport); From 786bd3938eaf72b0276f11aca9ddf44e86518dba Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 Jan 2013 18:43:28 +0100 Subject: [PATCH 0432/1752] CLIENT GETNAME and CLIENT SETNAME introduced. Sometimes it is much simpler to debug complex Redis installations if it is possible to assign clients a name that is displayed in the CLIENT LIST output. This is the case, for example, for "leaked" connections. The ability to provide a name to the client makes it quite trivial to understand what is the part of the code implementing the client not releasing the resources appropriately. Behavior: CLIENT SETNAME: set a name for the client, or remove the current name if an empty name is set. CLIENT GETNAME: get the current name, or a nil. CLIENT LIST: now displays the client name if any. Thanks to Mark Gravell for pushing this idea forward. --- src/aof.c | 1 + src/networking.c | 41 +++++++++++++++++++++++++++++++++++++++-- src/redis.h | 1 + 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/src/aof.c b/src/aof.c index 6289e4dd5bd..fe5c64972f1 100644 --- a/src/aof.c +++ b/src/aof.c @@ -442,6 +442,7 @@ struct redisClient *createFakeClient(void) { selectDb(c,0); c->fd = -1; + c->name = NULL; c->querybuf = sdsempty(); c->querybuf_peak = 0; c->argc = 0; diff --git a/src/networking.c b/src/networking.c index c23939c5c1b..d935eaa82b0 100644 --- a/src/networking.c +++ b/src/networking.c @@ -70,6 +70,7 @@ redisClient *createClient(int fd) { selectDb(c,0); c->fd = fd; + c->name = NULL; c->bufpos = 0; c->querybuf = sdsempty(); c->querybuf_peak = 0; @@ -668,6 +669,7 @@ void freeClient(redisClient *c) { } /* Release memory */ + if (c->name) decrRefCount(c->name); zfree(c->argv); freeClientMultiState(c); zfree(c); @@ -1123,9 +1125,11 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatprintf(sdsempty(), - "addr=%s:%d fd=%d age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", + "addr=%s:%d fd=%d name=%s age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", (client->flags & REDIS_UNIX_SOCKET) ? server.unixsocket : ip, - port,client->fd, + port, + client->fd, + client->name ? (char*)client->name->ptr : "", (long)(server.unixtime - client->ctime), (long)(server.unixtime - client->lastinteraction), flags, @@ -1190,6 +1194,39 @@ void clientCommand(redisClient *c) { } } addReplyError(c,"No such client"); + } else if (!strcasecmp(c->argv[1]->ptr,"setname") && c->argc == 3) { + int j, len = sdslen(c->argv[2]->ptr); + char *p = c->argv[2]->ptr; + + /* Setting the client name to an empty string actually removes + * the current name. */ + if (len == 0) { + if (c->name) decrRefCount(c->name); + c->name = NULL; + addReply(c,shared.ok); + return; + } + + /* Otherwise check if the charset is ok. We need to do this otherwise + * CLIENT LIST format will break. You should always be able to + * split by space to get the different fields. */ + for (j = 0; j < len; j++) { + if (p[j] < '!' || p[j] > '~') { /* ASCI is assumed. */ + addReplyError(c, + "Client names cannot contain spaces, " + "newlines or special characters."); + return; + } + } + if (c->name) decrRefCount(c->name); + c->name = c->argv[2]; + incrRefCount(c->name); + addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"getname") && c->argc == 2) { + if (c->name) + addReplyBulk(c,c->name); + else + addReply(c,shared.nullbulk); } else { addReplyError(c, "Syntax error, try CLIENT (LIST | KILL ip:port)"); } diff --git a/src/redis.h b/src/redis.h index a9c3eb66a04..524bc960117 100644 --- a/src/redis.h +++ b/src/redis.h @@ -382,6 +382,7 @@ typedef struct redisClient { int fd; redisDb *db; int dictid; + robj *name; /* As set by CLIENT SETNAME */ sds querybuf; size_t querybuf_peak; /* Recent (100ms or more) peak of querybuf size */ int argc; From f31d10b9a6995dd9ebfed5f8ededb8c99fa363ea Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 11 Jan 2013 18:50:40 +0100 Subject: [PATCH 0433/1752] Typo fixed, ASCI -> ASCII. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index d935eaa82b0..3448ea8de4c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1211,7 +1211,7 @@ void clientCommand(redisClient *c) { * CLIENT LIST format will break. You should always be able to * split by space to get the different fields. */ for (j = 0; j < len; j++) { - if (p[j] < '!' || p[j] > '~') { /* ASCI is assumed. */ + if (p[j] < '!' || p[j] > '~') { /* ASCII is assumed. */ addReplyError(c, "Client names cannot contain spaces, " "newlines or special characters."); From 7897e0a0d6aaacd0057d3db378e0200dd6a10989 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 14 Jan 2013 10:19:20 +0100 Subject: [PATCH 0434/1752] Tests for CLIENT GETNAME/SETNAME. --- tests/unit/introspection.tcl | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index 9db0395a2bc..a6c2130f53e 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -19,4 +19,37 @@ start_server {tags {"introspection"}} { assert_match {*eval*} [$rd read] assert_match {*lua*"set"*"foo"*"bar"*} [$rd read] } + + test {CLIENT GETNAME should return NIL if name is not assigned} { + r client getname + } {} + + test {CLIENT LIST shows empty fields for unassigned names} { + r client list + } {*name= *} + + test {CLIENT SETNAME does not accept spaces} { + catch {r client setname "foo bar"} e + set e + } {ERR*} + + test {CLIENT SETNAME can assign a name to this connection} { + assert_equal [r client setname myname] {OK} + r client list + } {*name=myname*} + + test {CLIENT SETNAME can change the name of an existing connection} { + assert_equal [r client setname someothername] {OK} + r client list + } {*name=someothername*} + + test {After CLIENT SETNAME, connection can still be closed} { + set rd [redis_deferring_client] + $rd client setname foobar + assert_equal [$rd read] "OK" + assert_match {*foobar*} [r client list] + $rd close + # Now the client should no longer be listed + string match {*foobar*} [r client list] + } {0} } From c48584b85dad5f2e18c607e235e7702dd02d97ad Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 16 Jan 2013 19:42:40 +0100 Subject: [PATCH 0435/1752] redis-cli: save an RDB dump from remote server to local file. --- src/redis-cli.c | 89 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 81 insertions(+), 8 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 3969fbab58c..c3938636b43 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -41,6 +41,7 @@ #include #include #include +#include #include "hiredis.h" #include "sds.h" @@ -73,6 +74,8 @@ static struct config { int cluster_reissue_command; int slave_mode; int pipe_mode; + int getrdb_mode; + char *rdb_filename; int bigkeys; int stdinarg; /* get last arg from stdin. (-x option) */ char *auth; @@ -660,6 +663,9 @@ static int parseOptions(int argc, char **argv) { config.latency_mode = 1; } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; + } else if (!strcmp(argv[i],"--rdb") && !lastarg) { + config.getrdb_mode = 1; + config.rdb_filename = argv[++i]; } else if (!strcmp(argv[i],"--pipe")) { config.pipe_mode = 1; } else if (!strcmp(argv[i],"--bigkeys")) { @@ -720,6 +726,7 @@ static void usage() { " --raw Use raw formatting for replies (default when STDOUT is not a tty)\n" " --latency Enter a special mode continuously sampling latency\n" " --slave Simulate a slave showing commands received from the master\n" +" --rdb Transfer an RDB dump from remote server to local file.\n" " --pipe Transfer raw Redis protocol from stdin to server\n" " --bigkeys Sample Redis keys looking for big keys\n" " --eval Send an EVAL command using the Lua script at \n" @@ -927,15 +934,15 @@ static void latencyMode(void) { } } -static void slaveMode(void) { +/* Sends SYNC and reads the number of bytes in the payload. Used both by + * slaveMode() and getRDB(). */ +unsigned long long sendSync(int fd) { /* To start we need to send the SYNC command and return the payload. * The hiredis client lib does not understand this part of the protocol * and we don't want to mess with its buffers, so everything is performed * using direct low-level I/O. */ - int fd = context->fd; - char buf[1024], *p; + char buf[4096], *p; ssize_t nread; - unsigned long long payload; /* Send the SYNC command. */ if (write(fd,"SYNC\r\n",6) != 6) { @@ -955,12 +962,25 @@ static void slaveMode(void) { p++; } *p = '\0'; - payload = strtoull(buf+1,NULL,10); - fprintf(stderr,"SYNC with master, discarding %lld bytes of bulk tranfer...\n", - payload); + if (buf[0] == '-') { + printf("SYNC with master failed: %s\n", buf); + exit(1); + } + return strtoull(buf+1,NULL,10); +} + +static void slaveMode(void) { + int fd = context->fd; + unsigned long long payload = sendSync(fd); + char buf[1024]; + + fprintf(stderr,"SYNC with master, discarding %llu " + "bytes of bulk tranfer...\n", payload); /* Discard the payload. */ while(payload) { + ssize_t nread; + nread = read(fd,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload); if (nread <= 0) { fprintf(stderr,"Error reading RDB payload while SYNCing\n"); @@ -970,11 +990,56 @@ static void slaveMode(void) { } fprintf(stderr,"SYNC done. Logging commands from master.\n"); - /* Now we can use the hiredis to read the incoming protocol. */ + /* Now we can use hiredis to read the incoming protocol. */ config.output = OUTPUT_CSV; while (cliReadReply(0) == REDIS_OK); } +/* This function implements --rdb, so it uses the replication protocol in order + * to fetch the RDB file from a remote server. */ +static void getRDB(void) { + int s = context->fd; + int fd; + unsigned long long payload = sendSync(s); + char buf[4096]; + + fprintf(stderr,"SYNC sent to master, writing %llu bytes to '%s'\n", + payload, config.rdb_filename); + + /* Write to file. */ + if (!strcmp(config.rdb_filename,"-")) { + fd = STDOUT_FILENO; + } else { + fd = open(config.rdb_filename, O_CREAT|O_WRONLY, 0644); + if (fd == -1) { + fprintf(stderr, "Error opening '%s': %s\n", config.rdb_filename, + strerror(errno)); + exit(1); + } + } + + while(payload) { + ssize_t nread, nwritten; + + nread = read(s,buf,(payload > sizeof(buf)) ? sizeof(buf) : payload); + if (nread <= 0) { + fprintf(stderr,"I/O Error reading RDB payload from socket\n"); + exit(1); + } + nwritten = write(fd, buf, nread); + if (nwritten != nread) { + fprintf(stderr,"Error writing data to file: %s\n", + strerror(errno)); + exit(1); + } + payload -= nread; + } + close(s); /* Close the file descriptor ASAP as fsync() may take time. */ + fsync(fd); + fprintf(stderr,"Transfer finished with success.\n"); + exit(0); +} + static void pipeMode(void) { int fd = context->fd; long long errors = 0, replies = 0, obuf_len = 0, obuf_pos = 0; @@ -1203,6 +1268,8 @@ int main(int argc, char **argv) { config.latency_mode = 0; config.cluster_mode = 0; config.slave_mode = 0; + config.getrdb_mode = 0; + config.rdb_filename = NULL; config.pipe_mode = 0; config.bigkeys = 0; config.stdinarg = 0; @@ -1231,6 +1298,12 @@ int main(int argc, char **argv) { slaveMode(); } + /* Get RDB mode. */ + if (config.getrdb_mode) { + cliConnect(0); + getRDB(); + } + /* Pipe mode */ if (config.pipe_mode) { if (cliConnect(0) == REDIS_ERR) exit(1); From f2bc198d40355cdfb628f626408b4f54a4939b8d Mon Sep 17 00:00:00 2001 From: Nathan Parry Date: Thu, 17 Jan 2013 23:55:43 -0500 Subject: [PATCH 0436/1752] redis-cli --rdb fails if server sends a ping Redis pings slaves in "pre-synchronization stage" with newlines. (See https://github.com/antirez/redis/blob/2.6.9/src/replication.c#L814) However, redis-cli does not expect this - it sees the newline as the end of the bulk length line, and ends up returning 0 as bulk the length. This manifests as the following when running redis-cli: $ ./src/redis-cli --rdb some_file SYNC sent to master, writing 0 bytes to 'some_file' Transfer finished with success. With this commit, we just ignore leading newlines while reading the bulk length line. To reproduce the problem, load enough data into Redis so that the preparation of the RDB snapshot takes long enough for a ping to occur while redis-cli is waiting for the data. --- src/redis-cli.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index c3938636b43..0bc61bb25a0 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -958,8 +958,8 @@ unsigned long long sendSync(int fd) { fprintf(stderr,"Error reading bulk length while SYNCing\n"); exit(1); } - if (*p == '\n') break; - p++; + if (*p == '\n' && p != buf) break; + if (*p != '\n') p++; } *p = '\0'; if (buf[0] == '-') { From aadcda9977dfe770f3c719a51bfbc73455da4aa3 Mon Sep 17 00:00:00 2001 From: bitterb Date: Sat, 19 Jan 2013 14:11:33 +0900 Subject: [PATCH 0437/1752] Fix an error reply for CLIENT command --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 3448ea8de4c..d1a8d3b3f33 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1228,7 +1228,7 @@ void clientCommand(redisClient *c) { else addReply(c,shared.nullbulk); } else { - addReplyError(c, "Syntax error, try CLIENT (LIST | KILL ip:port)"); + addReplyError(c, "Syntax error, try CLIENT (LIST | KILL ip:port | GETNAME | SETNAME connection-name)"); } } From a8f9cec16a79d17f72c30a95773ba572bc3eabf5 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Fri, 18 Jan 2013 10:13:10 +0100 Subject: [PATCH 0438/1752] Always exit if connection fails. This avoids unnecessary core dumps. Fixes antirez/redis#894 --- src/redis-cli.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 0bc61bb25a0..a789ab13d20 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1288,19 +1288,19 @@ int main(int argc, char **argv) { /* Latency mode */ if (config.latency_mode) { - cliConnect(0); + if (cliConnect(0) == REDIS_ERR) exit(1); latencyMode(); } /* Slave mode */ if (config.slave_mode) { - cliConnect(0); + if (cliConnect(0) == REDIS_ERR) exit(1); slaveMode(); } /* Get RDB mode. */ if (config.getrdb_mode) { - cliConnect(0); + if (cliConnect(0) == REDIS_ERR) exit(1); getRDB(); } @@ -1312,7 +1312,7 @@ int main(int argc, char **argv) { /* Find big keys */ if (config.bigkeys) { - cliConnect(0); + if (cliConnect(0) == REDIS_ERR) exit(1); findBigKeys(); } From ff1e4d22a67c9de04d3b26c5840e9b97095bfa64 Mon Sep 17 00:00:00 2001 From: charsyam Date: Wed, 16 Jan 2013 17:20:54 -0800 Subject: [PATCH 0439/1752] redis-cli prompt bug fix --- src/redis-cli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index a789ab13d20..1fb076fa54f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -801,6 +801,7 @@ static void repl() { sdsfree(config.hostip); config.hostip = sdsnew(argv[1]); config.hostport = atoi(argv[2]); + cliRefreshPrompt(); cliConnect(1); } else if (argc == 1 && !strcasecmp(argv[0],"clear")) { linenoiseClearScreen(); From 1caf09399efbe2fcfa3a4e04b7fca6ea7a23a793 Mon Sep 17 00:00:00 2001 From: guiquanz Date: Thu, 17 Jan 2013 01:00:20 +0800 Subject: [PATCH 0440/1752] Fixed many typos. Conflicts fixed, mainly because 2.8 has no cluster support / files: 00-RELEASENOTES src/cluster.c src/crc16.c src/redis-trib.rb src/redis.h --- 00-RELEASENOTES | 4 +--- redis.conf | 2 +- sentinel.conf | 2 +- src/adlist.c | 6 +++--- src/ae.c | 6 +++--- src/ae_evport.c | 12 ++++++------ src/anet.c | 4 ++-- src/aof.c | 2 +- src/bio.c | 2 +- src/bitops.c | 4 ++-- src/config.c | 6 +++--- src/db.c | 6 +++--- src/debug.c | 4 ++-- src/dict.c | 2 +- src/lzfP.h | 8 ++++---- src/mkreleasehdr.sh | 2 +- src/multi.c | 4 ++-- src/networking.c | 8 ++++---- src/object.c | 2 +- src/pubsub.c | 2 +- src/rdb.c | 10 +++++----- src/rdb.h | 2 +- src/redis-check-dump.c | 4 ++-- src/redis-cli.c | 6 +++--- src/redis.c | 28 ++++++++++++++-------------- src/redis.h | 22 +++++++++++----------- src/release.c | 4 ++-- src/replication.c | 10 +++++----- src/scripting.c | 12 ++++++------ src/sds.c | 4 ++-- src/sentinel.c | 20 ++++++++++---------- src/sha1.c | 36 ++++++++++++++++++------------------ src/sort.c | 6 +++--- src/t_list.c | 10 +++++----- src/t_set.c | 4 ++-- src/t_string.c | 4 ++-- src/t_zset.c | 4 ++-- src/zmalloc.c | 2 +- tests/unit/scripting.tcl | 2 +- 39 files changed, 138 insertions(+), 140 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 864c8c145aa..8910ad91854 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -3,6 +3,4 @@ Redis 2.7 release notes This is the unstable version of Redis 2.8, currently work in progress. -When it will be ready it will be releaed as Redis 2.8.0 - -Unstable versions of Redis 2.8 will be called 2.7.0, 2.7.1, and so forth. +When it will be ready for production, it will be releaed as Redis 2.8.0. diff --git a/redis.conf b/redis.conf index b8f0c486696..797347c69ea 100644 --- a/redis.conf +++ b/redis.conf @@ -246,7 +246,7 @@ slave-priority 100 # Set the max number of connected clients at the same time. By default # this limit is set to 10000 clients, however if the Redis server is not -# able ot configure the process file limit to allow for the specified limit +# able to configure the process file limit to allow for the specified limit # the max number of allowed clients is set to the current file limit # minus 32 (as Redis reserves a few file descriptors for internal uses). # diff --git a/sentinel.conf b/sentinel.conf index 94169ee8f99..ac687b5358c 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -71,7 +71,7 @@ sentinel parallel-syncs mymaster 1 # Default is 15 minutes. sentinel failover-timeout mymaster 900000 -# SCRIPTS EXECTION +# SCRIPTS EXECUTION # # sentinel notification-script and sentinel reconfig-script are used in order # to configure scripts that are called to notify the system administrator diff --git a/src/adlist.c b/src/adlist.c index e48957e3af9..f075e1bda71 100644 --- a/src/adlist.c +++ b/src/adlist.c @@ -97,7 +97,7 @@ list *listAddNodeHead(list *list, void *value) return list; } -/* Add a new node to the list, to tail, contaning the specified 'value' +/* Add a new node to the list, to tail, containing the specified 'value' * pointer as value. * * On error, NULL is returned and no operation is performed (i.e. the @@ -308,7 +308,7 @@ listNode *listSearchKey(list *list, void *key) /* Return the element at the specified zero-based index * where 0 is the head, 1 is the element next to head * and so on. Negative integers are used in order to count - * from the tail, -1 is the last element, -2 the penultimante + * from the tail, -1 is the last element, -2 the penultimate * and so on. If the index is out of range NULL is returned. */ listNode *listIndex(list *list, long index) { listNode *n; @@ -330,7 +330,7 @@ void listRotate(list *list) { if (listLength(list) <= 1) return; - /* Detatch current tail */ + /* Detach current tail */ list->tail = tail->prev; list->tail->next = NULL; /* Move it as head */ diff --git a/src/ae.c b/src/ae.c index 90be4e28f41..6ca9a51538d 100644 --- a/src/ae.c +++ b/src/ae.c @@ -309,7 +309,7 @@ static int processTimeEvents(aeEventLoop *eventLoop) { /* Process every pending time event, then every pending file event * (that may be registered by time event callbacks just processed). * Without special flags the function sleeps until some file event - * fires, or when the next time event occurrs (if any). + * fires, or when the next time event occurs (if any). * * If flags is 0, the function does nothing and returns. * if flags has AE_ALL_EVENTS set, all the kind of events are processed. @@ -356,7 +356,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) if (tvp->tv_usec < 0) tvp->tv_usec = 0; } else { /* If we have to check for events but need to return - * ASAP because of AE_DONT_WAIT we need to se the timeout + * ASAP because of AE_DONT_WAIT we need to set the timeout * to zero */ if (flags & AE_DONT_WAIT) { tv.tv_sec = tv.tv_usec = 0; @@ -395,7 +395,7 @@ int aeProcessEvents(aeEventLoop *eventLoop, int flags) return processed; /* return the number of processed file/time events */ } -/* Wait for millseconds until the given file descriptor becomes +/* Wait for milliseconds until the given file descriptor becomes * writable/readable/exception */ int aeWait(int fd, int mask, long long milliseconds) { struct pollfd pfd; diff --git a/src/ae_evport.c b/src/ae_evport.c index 0196dccf42d..94413c132d0 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -50,15 +50,15 @@ static int evport_debug = 0; * aeApiPoll, the corresponding file descriptors become dissociated from the * port. This is necessary because poll events are level-triggered, so if the * fd didn't become dissociated, it would immediately fire another event since - * the underlying state hasn't changed yet. We must reassociate the file + * the underlying state hasn't changed yet. We must re-associate the file * descriptor, but only after we know that our caller has actually read from it. * The ae API does not tell us exactly when that happens, but we do know that * it must happen by the time aeApiPoll is called again. Our solution is to - * keep track of the last fds returned by aeApiPoll and reassociate them next + * keep track of the last fds returned by aeApiPoll and re-associate them next * time aeApiPoll is invoked. * * To summarize, in this module, each fd association is EITHER (a) represented - * only via the in-kernel assocation OR (b) represented by pending_fds and + * only via the in-kernel association OR (b) represented by pending_fds and * pending_masks. (b) is only true for the last fds we returned from aeApiPoll, * and only until we enter aeApiPoll again (at which point we restore the * in-kernel association). @@ -164,7 +164,7 @@ static int aeApiAddEvent(aeEventLoop *eventLoop, int fd, int mask) { * This fd was recently returned from aeApiPoll. It should be safe to * assume that the consumer has processed that poll event, but we play * it safer by simply updating pending_mask. The fd will be - * reassociated as usual when aeApiPoll is called again. + * re-associated as usual when aeApiPoll is called again. */ if (evport_debug) fprintf(stderr, "aeApiAddEvent: adding to pending fd %d\n", fd); @@ -228,7 +228,7 @@ static void aeApiDelEvent(aeEventLoop *eventLoop, int fd, int mask) { * ENOMEM is a potentially transient condition, but the kernel won't * generally return it unless things are really bad. EAGAIN indicates * we've reached an resource limit, for which it doesn't make sense to - * retry (counterintuitively). All other errors indicate a bug. In any + * retry (counter-intuitively). All other errors indicate a bug. In any * of these cases, the best we can do is to abort. */ abort(); /* will not return */ @@ -243,7 +243,7 @@ static int aeApiPoll(aeEventLoop *eventLoop, struct timeval *tvp) { port_event_t event[MAX_EVENT_BATCHSZ]; /* - * If we've returned fd events before, we must reassociate them with the + * If we've returned fd events before, we must re-associate them with the * port now, before calling port_get(). See the block comment at the top of * this file for an explanation of why. */ diff --git a/src/anet.c b/src/anet.c index ae8e9a6585a..4da3e28db7e 100644 --- a/src/anet.c +++ b/src/anet.c @@ -61,7 +61,7 @@ int anetNonBlock(char *err, int fd) { int flags; - /* Set the socket nonblocking. + /* Set the socket non-blocking. * Note that fcntl(2) for F_GETFL and F_SETFL can't be * interrupted by a signal. */ if ((flags = fcntl(fd, F_GETFL)) == -1) { @@ -132,7 +132,7 @@ static int anetCreateSocket(char *err, int domain) { return ANET_ERR; } - /* Make sure connection-intensive things like the redis benckmark + /* Make sure connection-intensive things like the redis benchmark * will be able to close/open sockets a zillion of times */ if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno)); diff --git a/src/aof.c b/src/aof.c index fe5c64972f1..7e1512ebdbd 100644 --- a/src/aof.c +++ b/src/aof.c @@ -385,7 +385,7 @@ void feedAppendOnlyFile(struct redisCommand *cmd, int dictid, robj **argv, int a sds buf = sdsempty(); robj *tmpargv[3]; - /* The DB this command was targetting is not the same as the last command + /* The DB this command was targeting is not the same as the last command * we appendend. To issue a SELECT command is needed. */ if (dictid != server.aof_selected_db) { char seldb[64]; diff --git a/src/bio.c b/src/bio.c index 0fec24d0bd0..4bd5a17c613 100644 --- a/src/bio.c +++ b/src/bio.c @@ -74,7 +74,7 @@ static list *bio_jobs[REDIS_BIO_NUM_OPS]; static unsigned long long bio_pending[REDIS_BIO_NUM_OPS]; /* This structure represents a background Job. It is only used locally to this - * file as the API deos not expose the internals at all. */ + * file as the API does not expose the internals at all. */ struct bio_job { time_t time; /* Time at which the job was created. */ /* Job specific arguments pointers. If we need to pass more than three diff --git a/src/bitops.c b/src/bitops.c index 3ef0a8f3d6c..75f3317a93e 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -34,7 +34,7 @@ * Helpers and low level bit functions. * -------------------------------------------------------------------------- */ -/* This helper function used by GETBIT / SETBIT parses the bit offset arguemnt +/* This helper function used by GETBIT / SETBIT parses the bit offset argument * making sure an error is returned if it is negative or if it overflows * Redis 512 MB limit for the string value. */ static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { @@ -189,7 +189,7 @@ void bitopCommand(redisClient *c) { char *opname = c->argv[1]->ptr; robj *o, *targetkey = c->argv[2]; long op, j, numkeys; - robj **objects; /* Array of soruce objects. */ + robj **objects; /* Array of source objects. */ unsigned char **src; /* Array of source strings pointers. */ long *len, maxlen = 0; /* Array of length of src strings, and max len. */ long minlen = 0; /* Min len among the input keys. */ diff --git a/src/config.c b/src/config.c index 1765b7d908e..1fa498d3c8c 100644 --- a/src/config.c +++ b/src/config.c @@ -333,7 +333,7 @@ void loadServerConfigFromString(char *config) { goto loaderr; } - /* If the target command name is the emtpy string we just + /* If the target command name is the empty string we just * remove it from the command table. */ retval = dictDelete(server.commands, argv[1]); redisAssert(retval == DICT_OK); @@ -371,7 +371,7 @@ void loadServerConfigFromString(char *config) { soft = memtoll(argv[3],NULL); soft_seconds = atoi(argv[4]); if (soft_seconds < 0) { - err = "Negative number of seconds in soft limt is invalid"; + err = "Negative number of seconds in soft limit is invalid"; goto loaderr; } server.client_obuf_limits[class].hard_limit_bytes = hard; @@ -416,7 +416,7 @@ void loadServerConfigFromString(char *config) { * in the 'options' string to the config file before loading. * * Both filename and options can be NULL, in such a case are considered - * emtpy. This way loadServerConfig can be used to just load a file or + * empty. This way loadServerConfig can be used to just load a file or * just load a string. */ void loadServerConfig(char *filename, char *options) { sds config = sdsempty(); diff --git a/src/db.c b/src/db.c index f5e8861872f..1cb89e3c2bc 100644 --- a/src/db.c +++ b/src/db.c @@ -44,7 +44,7 @@ robj *lookupKey(redisDb *db, robj *key) { if (de) { robj *val = dictGetVal(de); - /* Update the access time for the aging algorithm. + /* Update the access time for the ageing algorithm. * Don't do it if we have a saving child, as this will trigger * a copy on write madness. */ if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) @@ -85,7 +85,7 @@ robj *lookupKeyWriteOrReply(redisClient *c, robj *key, robj *reply) { } /* Add the key to the DB. It's up to the caller to increment the reference - * counte of the value if needed. + * counter of the value if needed. * * The program is aborted if the key already exists. */ void dbAdd(redisDb *db, robj *key, robj *val) { @@ -538,7 +538,7 @@ int expireIfNeeded(redisDb *db, robj *key) { * for *AT variants of the command, or the current time for relative expires). * * unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for - * the argv[2] parameter. The basetime is always specified in milliesconds. */ + * the argv[2] parameter. The basetime is always specified in milliseconds. */ void expireGenericCommand(redisClient *c, long long basetime, int unit) { dictEntry *de; robj *key = c->argv[1], *param = c->argv[2]; diff --git a/src/debug.c b/src/debug.c index 39ddb432746..897a71acea2 100644 --- a/src/debug.c +++ b/src/debug.c @@ -44,7 +44,7 @@ /* ================================= Debugging ============================== */ /* Compute the sha1 of string at 's' with 'len' bytes long. - * The SHA1 is then xored againt the string pointed by digest. + * The SHA1 is then xored against the string pointed by digest. * Since xor is commutative, this operation is used in order to * "add" digests relative to unordered elements. * @@ -69,7 +69,7 @@ void xorObjectDigest(unsigned char *digest, robj *o) { } /* This function instead of just computing the SHA1 and xoring it - * against diget, also perform the digest of "digest" itself and + * against digest, also perform the digest of "digest" itself and * replace the old value with the new one. * * So the final digest will be: diff --git a/src/dict.c b/src/dict.c index d7376eb78a3..c6a1b88cfbd 100644 --- a/src/dict.c +++ b/src/dict.c @@ -610,7 +610,7 @@ static int _dictExpandIfNeeded(dict *d) /* Incremental rehashing already in progress. Return. */ if (dictIsRehashing(d)) return DICT_OK; - /* If the hash table is empty expand it to the intial size. */ + /* If the hash table is empty expand it to the initial size. */ if (d->ht[0].size == 0) return dictExpand(d, DICT_HT_INITIAL_SIZE); /* If we reached the 1:1 ratio, and we are allowed to resize the hash diff --git a/src/lzfP.h b/src/lzfP.h index d533f18292b..10d804e04b5 100644 --- a/src/lzfP.h +++ b/src/lzfP.h @@ -93,7 +93,7 @@ /* * Avoid assigning values to errno variable? for some embedding purposes - * (linux kernel for example), this is neccessary. NOTE: this breaks + * (linux kernel for example), this is necessary. NOTE: this breaks * the documentation in lzf.h. */ #ifndef AVOID_ERRNO @@ -101,7 +101,7 @@ #endif /* - * Wether to pass the LZF_STATE variable as argument, or allocate it + * Whether to pass the LZF_STATE variable as argument, or allocate it * on the stack. For small-stack environments, define this to 1. * NOTE: this breaks the prototype in lzf.h. */ @@ -110,11 +110,11 @@ #endif /* - * Wether to add extra checks for input validity in lzf_decompress + * Whether to add extra checks for input validity in lzf_decompress * and return EINVAL if the input stream has been corrupted. This * only shields against overflowing the input buffer and will not * detect most corrupted streams. - * This check is not normally noticable on modern hardware + * This check is not normally noticeable on modern hardware * (<1% slowdown), but might slow down older cpus considerably. */ #ifndef CHECK_INPUT diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index d07cf6ae058..f3bbac0156f 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -4,7 +4,7 @@ GIT_DIRTY=`git diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ -(cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already uptodate +(cat release.h | grep DIRTY | grep $GIT_DIRTY) && exit 0 # Already up-to-date echo "#define REDIS_GIT_SHA1 \"$GIT_SHA1\"" > release.h echo "#define REDIS_GIT_DIRTY \"$GIT_DIRTY\"" >> release.h echo "#define REDIS_BUILD_ID \"$BUILD_ID\"" >> release.h diff --git a/src/multi.c b/src/multi.c index 064a4094412..dfac15c344f 100644 --- a/src/multi.c +++ b/src/multi.c @@ -102,7 +102,7 @@ void discardCommand(redisClient *c) { } /* Send a MULTI command to all the slaves and AOF file. Check the execCommand - * implememntation for more information. */ + * implementation for more information. */ void execCommandReplicateMulti(redisClient *c) { robj *multistring = createStringObject("MULTI",5); @@ -223,7 +223,7 @@ void watchForKey(redisClient *c, robj *key) { incrRefCount(key); } listAddNodeTail(clients,c); - /* Add the new key to the lits of keys watched by this client */ + /* Add the new key to the list of keys watched by this client */ wk = zmalloc(sizeof(*wk)); wk->key = key; wk->db = c->db; diff --git a/src/networking.c b/src/networking.c index d1a8d3b3f33..f48d27c0723 100644 --- a/src/networking.c +++ b/src/networking.c @@ -378,7 +378,7 @@ void *addDeferredMultiBulkLength(redisClient *c) { return listLast(c->reply); } -/* Populate the length object and try glueing it to the next chunk. */ +/* Populate the length object and try gluing it to the next chunk. */ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) { listNode *ln = (listNode*)node; robj *len, *next; @@ -404,7 +404,7 @@ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) { asyncCloseClientOnOutputBufferLimitReached(c); } -/* Add a duble as a bulk reply */ +/* Add a double as a bulk reply */ void addReplyDouble(redisClient *c, double d) { char dbuf[128], sbuf[128]; int dlen, slen; @@ -526,7 +526,7 @@ static void acceptCommonHandler(int fd, int flags) { } /* If maxclient directive is set and this is one client more... close the * connection. Note that we create the client instead to check before - * for this condition, since now the socket is already set in nonblocking + * for this condition, since now the socket is already set in non-blocking * mode and we can send an error for free using the Kernel I/O */ if (listLength(server.clients) > server.maxclients) { char *err = "-ERR max number of clients reached\r\n"; @@ -941,7 +941,7 @@ int processMultibulkBuffer(redisClient *c) { /* Not enough data (+2 == trailing \r\n) */ break; } else { - /* Optimization: if the buffer contanins JUST our bulk element + /* Optimization: if the buffer containns JUST our bulk element * instead of creating a new object by *copying* the sds we * just use the current sds string. */ if (pos == 0 && diff --git a/src/object.c b/src/object.c index c2b89709d92..00cf023b07f 100644 --- a/src/object.c +++ b/src/object.c @@ -72,7 +72,7 @@ robj *createStringObjectFromLongDouble(long double value) { int len; /* We use 17 digits precision since with 128 bit floats that precision - * after rouding is able to represent most small decimal numbers in a way + * after rounding is able to represent most small decimal numbers in a way * that is "non surprising" for the user (that is, most small decimal * numbers will be represented in a way that when converted back into * a string are exactly the same as what the user typed.) */ diff --git a/src/pubsub.c b/src/pubsub.c index 8cd17df4f26..973692948ef 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -117,7 +117,7 @@ int pubsubUnsubscribeChannel(redisClient *c, robj *channel, int notify) { return retval; } -/* Subscribe a client to a pattern. Returns 1 if the operation succeeded, or 0 if the clinet was already subscribed to that pattern. */ +/* Subscribe a client to a pattern. Returns 1 if the operation succeeded, or 0 if the client was already subscribed to that pattern. */ int pubsubSubscribePattern(redisClient *c, robj *pattern) { int retval = 0; diff --git a/src/rdb.c b/src/rdb.c index 3d5ac2043a2..17f2ef840e6 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -265,7 +265,7 @@ robj *rdbLoadLzfStringObject(rio *rdb) { return NULL; } -/* Save a string objet as [len][data] on disk. If the object is a string +/* Save a string object as [len][data] on disk. If the object is a string * representation of an integer value we try to save it in a special form */ int rdbSaveRawString(rio *rdb, unsigned char *s, size_t len) { int enclen; @@ -321,7 +321,7 @@ int rdbSaveLongLongAsStringObject(rio *rdb, long long value) { /* Like rdbSaveStringObjectRaw() but handle encoded objects */ int rdbSaveStringObject(rio *rdb, robj *obj) { /* Avoid to decode the object, then encode it again, if the - * object is alrady integer encoded. */ + * object is already integer encoded. */ if (obj->encoding == REDIS_ENCODING_INT) { return rdbSaveLongLongAsStringObject(rdb,(long)obj->ptr); } else { @@ -367,7 +367,7 @@ robj *rdbLoadEncodedStringObject(rio *rdb) { } /* Save a double value. Doubles are saved as strings prefixed by an unsigned - * 8 bit integer specifing the length of the representation. + * 8 bit integer specifying the length of the representation. * This 8 bit integer has special values in order to specify the following * conditions: * 253: not a number @@ -606,7 +606,7 @@ off_t rdbSavedObjectLen(robj *o) { /* Save a key-value pair, with expire time, type, key, value. * On error -1 is returned. - * On success if the key was actaully saved 1 is returned, otherwise 0 + * On success if the key was actually saved 1 is returned, otherwise 0 * is returned (the key was already expired). */ int rdbSaveKeyValuePair(rio *rdb, robj *key, robj *val, long long expiretime, long long now) @@ -1109,7 +1109,7 @@ int rdbLoad(char *filename) { /* We read the time so we need to read the object type again. */ if ((type = rdbLoadType(&rdb)) == -1) goto eoferr; /* the EXPIRETIME opcode specifies time in seconds, so convert - * into milliesconds. */ + * into milliseconds. */ expiretime *= 1000; } else if (type == REDIS_RDB_OPCODE_EXPIRETIME_MS) { /* Milliseconds precision expire times introduced with RDB diff --git a/src/rdb.h b/src/rdb.h index d0f2aad86ba..54ee4e5142a 100644 --- a/src/rdb.h +++ b/src/rdb.h @@ -51,7 +51,7 @@ * number specify the kind of object that follows. * See the REDIS_RDB_ENC_* defines. * - * Lenghts up to 63 are stored using a single byte, most DB keys, and may + * Lengths up to 63 are stored using a single byte, most DB keys, and may * values, will fit inside. */ #define REDIS_RDB_6BITLEN 0 #define REDIS_RDB_14BITLEN 1 diff --git a/src/redis-check-dump.c b/src/redis-check-dump.c index 950655a024e..d0952778156 100644 --- a/src/redis-check-dump.c +++ b/src/redis-check-dump.c @@ -79,7 +79,7 @@ * number specify the kind of object that follows. * See the REDIS_RDB_ENC_* defines. * - * Lenghts up to 63 are stored using a single byte, most DB keys, and may + * Lengths up to 63 are stored using a single byte, most DB keys, and may * values, will fit inside. */ #define REDIS_RDB_6BITLEN 0 #define REDIS_RDB_14BITLEN 1 @@ -133,7 +133,7 @@ typedef struct { char success; } entry; -/* Global vars that are actally used as constants. The following double +/* Global vars that are actually used as constants. The following double * values are used for double on-disk serialization, and are initialized * at runtime to avoid strange compiler optimizations. */ static double R_Zero, R_PosInf, R_NegInf, R_Nan; diff --git a/src/redis-cli.c b/src/redis-cli.c index 1fb076fa54f..318f1822ab7 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -308,7 +308,7 @@ static int cliSelect() { return REDIS_ERR; } -/* Connect to the client. If force is not zero the connection is performed +/* Connect to the server. If force is not zero the connection is performed * even if there is already a connected socket. */ static int cliConnect(int force) { if (context == NULL || force) { @@ -976,7 +976,7 @@ static void slaveMode(void) { char buf[1024]; fprintf(stderr,"SYNC with master, discarding %llu " - "bytes of bulk tranfer...\n", payload); + "bytes of bulk transfer...\n", payload); /* Discard the payload. */ while(payload) { @@ -1141,7 +1141,7 @@ static void pipeMode(void) { int j; eof = 1; - /* Everything transfered, so we queue a special + /* Everything transferred, so we queue a special * ECHO command that we can match in the replies * to make sure everything was read from the server. */ for (j = 0; j < 20; j++) diff --git a/src/redis.c b/src/redis.c index 2ae567ddf5c..2a56ad2865a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -99,7 +99,7 @@ struct redisCommand *commandTable; * m: may increase memory usage once called. Don't allow if out of memory. * a: admin command, like SAVE or SHUTDOWN. * p: Pub/Sub related command. - * f: force replication of this command, regarless of server.dirty. + * f: force replication of this command, regardless of server.dirty. * s: command not allowed in scripts. * R: random command. Command is not deterministic, that is, the same command * with the same arguments, with the same key space, may have different @@ -288,7 +288,7 @@ void redisLogRaw(int level, const char *msg) { if (server.syslog_enabled) syslog(syslogLevelMap[level], "%s", msg); } -/* Like redisLogRaw() but with printf-alike support. This is the funciton that +/* Like redisLogRaw() but with printf-alike support. This is the function that * is used across the code. The raw version is only used in order to dump * the INFO output on crash. */ void redisLog(int level, const char *fmt, ...) { @@ -363,7 +363,7 @@ void exitFromChild(int retcode) { /*====================== Hash table type implementation ==================== */ -/* This is an hash table type that uses the SDS dynamic strings libary as +/* This is an hash table type that uses the SDS dynamic strings library as * keys and radis objects as values (objects can hold SDS strings, * lists, sets). */ @@ -537,7 +537,7 @@ dictType commandTableDictType = { NULL /* val destructor */ }; -/* Hash type hash table (note that small hashes are represented with zimpaps) */ +/* Hash type hash table (note that small hashes are represented with zipmaps) */ dictType hashDictType = { dictEncObjHash, /* hash function */ NULL, /* key dup */ @@ -738,7 +738,7 @@ int clientsCronHandleTimeout(redisClient *c) { /* The client query buffer is an sds.c string that can end with a lot of * free space not used, this function reclaims space if needed. * - * The funciton always returns 0 as it never terminates the client. */ + * The function always returns 0 as it never terminates the client. */ int clientsCronResizeQueryBuffer(redisClient *c) { size_t querybuf_size = sdsAllocSize(c->querybuf); time_t idletime = server.unixtime - c->lastinteraction; @@ -796,11 +796,11 @@ void clientsCron(void) { * * - Active expired keys collection (it is also performed in a lazy way on * lookup). - * - Software watchdong. + * - Software watchdog. * - Update some statistic. * - Incremental rehashing of the DBs hash tables. * - Triggering BGSAVE / AOF rewrite, and handling of terminated children. - * - Clients timeout of differnet kinds. + * - Clients timeout of different kinds. * - Replication reconnection. * - Many more... * @@ -829,7 +829,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* We have just 22 bits per object for LRU information. * So we use an (eventually wrapping) LRU clock with 10 seconds resolution. - * 2^22 bits with 10 seconds resoluton is more or less 1.5 years. + * 2^22 bits with 10 seconds resolution is more or less 1.5 years. * * Note that even if this will wrap after 1.5 years it's not a problem, * everything will still work but just some object will appear younger @@ -867,7 +867,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } } - /* We don't want to resize the hash tables while a bacground saving + /* We don't want to resize the hash tables while a background saving * is in progress: the saving child is created using fork() that is * implemented with a copy-on-write semantic in most modern systems, so * if we resize the HT while there is the saving child at work actually @@ -1179,7 +1179,7 @@ void initServerConfig() { R_NegInf = -1.0/R_Zero; R_Nan = R_Zero/R_Zero; - /* Command table -- we intiialize it here as it is part of the + /* Command table -- we initiialize it here as it is part of the * initial configuration, since command names may be changed via * redis.conf using the rename-command directive. */ server.commands = dictCreate(&commandTableDictType,NULL); @@ -1489,7 +1489,7 @@ void call(redisClient *c, int flags) { long long dirty, start = ustime(), duration; /* Sent the command to clients in MONITOR mode, only if the commands are - * not geneated from reading an AOF. */ + * not generated from reading an AOF. */ if (listLength(server.monitors) && !server.loading && !(c->cmd->flags & REDIS_CMD_SKIP_MONITOR)) @@ -1551,8 +1551,8 @@ void call(redisClient *c, int flags) { * server for a bulk read from the client. * * If 1 is returned the client is still alive and valid and - * and other operations can be performed by the caller. Otherwise - * if 0 is returned the client was destroied (i.e. after QUIT). */ + * other operations can be performed by the caller. Otherwise + * if 0 is returned the client was destroyed (i.e. after QUIT). */ int processCommand(redisClient *c) { /* The QUIT command is handled separately. Normal command procs will * go through checking for replication and QUIT will cause trouble @@ -1805,7 +1805,7 @@ void echoCommand(redisClient *c) { void timeCommand(redisClient *c) { struct timeval tv; - /* gettimeofday() can only fail if &tv is a bad addresss so we + /* gettimeofday() can only fail if &tv is a bad address so we * don't check for errors. */ gettimeofday(&tv,NULL); addReplyMultiBulkLen(c,2); diff --git a/src/redis.h b/src/redis.h index 524bc960117..334d7b65d10 100644 --- a/src/redis.h +++ b/src/redis.h @@ -144,12 +144,12 @@ * * 00|000000 => if the two MSB are 00 the len is the 6 bits of this byte * 01|000000 00000000 => 01, the len is 14 byes, 6 bits + 8 bits of next byte - * 10|000000 [32 bit integer] => if it's 01, a full 32 bit len will follow + * 10|000000 [32 bit integer] => if it's 10, a full 32 bit len will follow * 11|000000 this means: specially encoded object will follow. The six bits * number specify the kind of object that follows. * See the REDIS_RDB_ENC_* defines. * - * Lenghts up to 63 are stored using a single byte, most DB keys, and may + * Lengths up to 63 are stored using a single byte, most DB keys, and may * values, will fit inside. */ #define REDIS_RDB_6BITLEN 0 #define REDIS_RDB_14BITLEN 1 @@ -319,7 +319,7 @@ typedef struct redisObject { void *ptr; } robj; -/* Macro used to initalize a Redis object allocated on the stack. +/* Macro used to initialize a Redis object allocated on the stack. * Note that this macro is taken near the structure definition to make sure * we'll update it when the structure is changed, to avoid bugs like * bug #85 introduced exactly in this way. */ @@ -376,7 +376,7 @@ typedef struct readyList { robj *key; } readyList; -/* With multiplexing we need to take per-clinet state. +/* With multiplexing we need to take per-client state. * Clients are taken in a liked list. */ typedef struct redisClient { int fd; @@ -539,7 +539,7 @@ struct redisServer { long long stat_keyspace_hits; /* Number of successful lookups of keys */ long long stat_keyspace_misses; /* Number of failed lookups of keys */ size_t stat_peak_memory; /* Max used memory record */ - long long stat_fork_time; /* Time needed to perform latets fork() */ + long long stat_fork_time; /* Time needed to perform latest fork() */ long long stat_rejected_conn; /* Clients rejected because of maxclients */ list *slowlog; /* SLOWLOG list of commands */ long long slowlog_entry_id; /* SLOWLOG current entry ID */ @@ -588,7 +588,7 @@ struct redisServer { char *rdb_filename; /* Name of RDB file */ int rdb_compression; /* Use compression in RDB? */ int rdb_checksum; /* Use RDB checksum? */ - time_t lastsave; /* Unix time of last save succeeede */ + time_t lastsave; /* Unix time of last successful save */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ time_t rdb_save_time_start; /* Current RDB save start time. */ int lastbgsave_status; /* REDIS_OK or REDIS_ERR */ @@ -623,7 +623,7 @@ struct redisServer { /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ unsigned long long maxmemory; /* Max number of memory bytes to use */ - int maxmemory_policy; /* Policy for key evition */ + int maxmemory_policy; /* Policy for key eviction */ int maxmemory_samples; /* Pricision of random sampling */ /* Blocked clients */ unsigned int bpop_blocked_clients; /* Number of clients blocked by lists */ @@ -660,7 +660,7 @@ struct redisServer { int lua_timedout; /* True if we reached the time limit for script execution. */ int lua_kill; /* Kill the script if true. */ - /* Assert & bug reportign */ + /* Assert & bug reporting */ char *assert_failed; char *assert_file; int assert_line; @@ -679,13 +679,13 @@ struct redisCommand { char *name; redisCommandProc *proc; int arity; - char *sflags; /* Flags as string represenation, one char per flag. */ + char *sflags; /* Flags as string representation, one char per flag. */ int flags; /* The actual flags, obtained from the 'sflags' field. */ /* Use a function to determine keys arguments in a command line. */ redisGetKeysProc *getkeys_proc; /* What keys should be loaded in background when calling this command? */ int firstkey; /* The first argument that's a key (0 = no keys) */ - int lastkey; /* THe last argument that's a key */ + int lastkey; /* The last argument that's a key */ int keystep; /* The step between first and last key */ long long microseconds, calls; }; @@ -732,7 +732,7 @@ typedef struct { dictIterator *di; } setTypeIterator; -/* Structure to hold hash iteration abstration. Note that iteration over +/* Structure to hold hash iteration abstraction. Note that iteration over * hashes involves both fields and values. Because it is possible that * not both are required, store pointers in the iterator to avoid * unnecessary memory allocation for fields/values. */ diff --git a/src/release.c b/src/release.c index 34c3d813cc5..4c5e1f7452e 100644 --- a/src/release.c +++ b/src/release.c @@ -27,8 +27,8 @@ * POSSIBILITY OF SUCH DAMAGE. */ -/* Every time the Redis Git SHA1 or Dirty status changes only this file - * small file is recompiled, as we access this information in all the other +/* Every time the Redis Git SHA1 or Dirty status changes only this small + * file is recompiled, as we access this information in all the other * files using this functions. */ #include diff --git a/src/replication.c b/src/replication.c index 15fc71e7dd0..2f0cba70158 100644 --- a/src/replication.c +++ b/src/replication.c @@ -52,7 +52,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_START) continue; /* Feed slaves that are waiting for the initial SYNC (so these commands - * are queued in the output buffer until the intial SYNC completes), + * are queued in the output buffer until the initial SYNC completes), * or are already in sync with the master. */ if (slave->slaveseldb != dictid) { robj *selectcmd; @@ -115,7 +115,7 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** } void syncCommand(redisClient *c) { - /* ignore SYNC if aleady slave or in monitor mode */ + /* ignore SYNC if already slave or in monitor mode */ if (c->flags & REDIS_SLAVE) return; /* Refuse SYNC requests if we are a slave but the link with our master @@ -229,7 +229,7 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { if (slave->repldboff == 0) { /* Write the bulk write count before to transfer the DB. In theory here * we don't know how much room there is in the output buffer of the - * socket, but in pratice SO_SNDLOWAT (the minimum count for output + * socket, but in practice SO_SNDLOWAT (the minimum count for output * operations) will never be smaller than the few bytes we need. */ sds bulkcount; @@ -272,7 +272,7 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { } } -/* This function is called at the end of every backgrond saving. +/* This function is called at the end of every background saving. * The argument bgsaveerr is REDIS_OK if the background saving succeeded * otherwise REDIS_ERR is passed to the function. * @@ -451,7 +451,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { stopAppendOnly(); while (retry-- && startAppendOnly() == REDIS_ERR) { - redisLog(REDIS_WARNING,"Failed enabling the AOF after successful master synchrnization! Trying it again in one second."); + redisLog(REDIS_WARNING,"Failed enabling the AOF after successful master synchronization! Trying it again in one second."); sleep(1); } if (!retry) { diff --git a/src/scripting.c b/src/scripting.c index 46301eb1130..6661f37485d 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -48,7 +48,7 @@ void sha1hex(char *digest, char *script, size_t len); /* Take a Redis reply in the Redis protocol format and convert it into a * Lua type. Thanks to this function, and the introduction of not connected - * clients, it is trvial to implement the redis() lua function. + * clients, it is trivial to implement the redis() lua function. * * Basically we take the arguments, execute the Redis command in the context * of a non connected client, then take the generated reply and convert it @@ -58,7 +58,7 @@ void sha1hex(char *digest, char *script, size_t len); * * Note: in this function we do not do any sanity check as the reply is * generated by Redis directly. This allows us to go faster. - * The reply string can be altered during the parsing as it is discared + * The reply string can be altered during the parsing as it is discarded * after the conversion is completed. * * Errors are returned as a table with a single 'err' field set to the @@ -597,7 +597,7 @@ void scriptingInit(void) { lua_setglobal(lua,"math"); - /* Add a helper funciton that we use to sort the multi bulk output of non + /* Add a helper function that we use to sort the multi bulk output of non * deterministic commands, when containing 'false' elements. */ { char *compare_func = "function __redis__compare_helper(a,b)\n" @@ -638,7 +638,7 @@ void scriptingReset(void) { scriptingInit(); } -/* Perform the SHA1 of the input string. We use this both for hasing script +/* Perform the SHA1 of the input string. We use this both for hashing script * bodies in order to obtain the Lua function name, and in the implementation * of redis.sha1(). * @@ -677,7 +677,7 @@ void luaReplyToRedisReply(redisClient *c, lua_State *lua) { case LUA_TTABLE: /* We need to check if it is an array, an error, or a status reply. * Error are returned as a single element table with 'err' field. - * Status replies are returned as single elment table with 'ok' field */ + * Status replies are returned as single element table with 'ok' field */ lua_pushstring(lua,"err"); lua_gettable(lua,-2); t = lua_type(lua,-1); @@ -834,7 +834,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { if (lua_isnil(lua,1)) { lua_pop(lua,1); /* remove the nil from the stack */ /* Function not defined... let's define it if we have the - * body of the funciton. If this is an EVALSHA call we can just + * body of the function. If this is an EVALSHA call we can just * return an error. */ if (evalsha) { addReply(c, shared.noscripterr); diff --git a/src/sds.c b/src/sds.c index e8491acfae9..85858a4f055 100644 --- a/src/sds.c +++ b/src/sds.c @@ -141,7 +141,7 @@ size_t sdsAllocSize(sds s) { * right-trim the string. * * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the - * following schema to cat bytes coming from the kerenl to the end of an + * following schema to cat bytes coming from the kernel to the end of an * sds string new things without copying into an intermediate buffer: * * oldlen = sdslen(s); @@ -596,7 +596,7 @@ void sdssplitargs_free(sds *argv, int argc) { } /* Modify the string substituting all the occurrences of the set of - * characters specifed in the 'from' string to the corresponding character + * characters specified in the 'from' string to the corresponding character * in the 'to' array. * * For instance: sdsmapchars(mystring, "ho", "01", 2) diff --git a/src/sentinel.c b/src/sentinel.c index d8a960713f0..8009e5ed9b6 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -969,9 +969,9 @@ const char *sentinelRedisInstanceTypeStr(sentinelRedisInstance *ri) { * a master's Sentinels dictionary, we want to be very sure about not * having duplicated instances for any reason. This is so important because * we use those other sentinels in order to run our quorum protocol to - * understand if it's time to proceeed with the fail over. + * understand if it's time to proceed with the fail over. * - * Making sure no duplication is possible we greately improve the robustness + * Making sure no duplication is possible we greatly improve the robustness * of the quorum (otherwise we may end counting the same instance multiple * times for some reason). * @@ -1238,7 +1238,7 @@ void sentinelKillLink(sentinelRedisInstance *ri, redisAsyncContext *c) { * cleanup needed. * * Note: we don't free the hiredis context as hiredis will do it for us - * for async conenctions. */ + * for async connections. */ void sentinelDisconnectInstanceFromContext(const redisAsyncContext *c) { sentinelRedisInstance *ri = c->data; int pubsub; @@ -1647,7 +1647,7 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd /* Update the last activity in the pubsub channel. Note that since we * receive our messages as well this timestamp can be used to detect - * if the link is probably diconnected even if it seems otherwise. */ + * if the link is probably disconnected even if it seems otherwise. */ ri->pc_last_activity = mstime(); /* Sanity check in the reply we expect, so that the code that follows @@ -1939,7 +1939,7 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { setDeferredMultiBulkLength(c,mbl,fields*2); } -/* Output a number of instances contanined inside a dictionary as +/* Output a number of instances contained inside a dictionary as * Redis protocol. */ void addReplyDictOfRedisInstances(redisClient *c, dict *instances) { dictIterator *di; @@ -2535,7 +2535,7 @@ void sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { * 3) info_refresh more recent than SENTINEL_INFO_VALIDITY_TIME. * 4) master_link_down_time no more than: * (now - master->s_down_since_time) + (master->down_after_period * 10). - * 5) Slave priority can't be zero, otherwise the slave is discareded. + * 5) Slave priority can't be zero, otherwise the slave is discarded. * * Among all the slaves matching the above conditions we select the slave * with lower slave_priority. If priority is the same we select the slave @@ -2611,10 +2611,10 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { /* If we in "wait start" but the master is no longer in ODOWN nor in * SDOWN condition we abort the failover. This is important as it * prevents a useless failover in a a notable case of netsplit, where - * the senitnels are split from the redis instances. In this case + * the sentinels are split from the redis instances. In this case * the failover will not start while there is the split because no * good slave can be reached. However when the split is resolved, we - * can go to waitstart if the slave is back rechable a few milliseconds + * can go to waitstart if the slave is back reachable a few milliseconds * before the master is. In that case when the master is back online * we cancel the failover. */ if ((ri->flags & (SRI_S_DOWN|SRI_O_DOWN|SRI_FORCE_FAILOVER)) == 0) { @@ -3026,13 +3026,13 @@ void sentinelHandleDictOfRedisInstances(dict *instances) { * following conditions happen: * * 1) The Sentiel process for some time is blocked, for every kind of - * random reason: the load is huge, the computer was freezed for some time + * random reason: the load is huge, the computer was frozen for some time * in I/O or alike, the process was stopped by a signal. Everything. * 2) The system clock was altered significantly. * * Under both this conditions we'll see everything as timed out and failing * without good reasons. Instead we enter the TILT mode and wait - * for SENTIENL_TILT_PERIOD to elapse before starting to act again. + * for SENTINEL_TILT_PERIOD to elapse before starting to act again. * * During TILT time we still collect information, we just do not act. */ void sentinelCheckTiltCondition(void) { diff --git a/src/sha1.c b/src/sha1.c index 26a5565ee0a..59e6f461d38 100644 --- a/src/sha1.c +++ b/src/sha1.c @@ -57,13 +57,13 @@ A million repetitions of "a" void SHA1Transform(u_int32_t state[5], const unsigned char buffer[64]) { -u_int32_t a, b, c, d, e; -typedef union { - unsigned char c[64]; - u_int32_t l[16]; -} CHAR64LONG16; + u_int32_t a, b, c, d, e; + typedef union { + unsigned char c[64]; + u_int32_t l[16]; + } CHAR64LONG16; #ifdef SHA1HANDSOFF -CHAR64LONG16 block[1]; /* use array to appear as a pointer */ + CHAR64LONG16 block[1]; /* use array to appear as a pointer */ memcpy(block, buffer, 64); #else /* The following had better never be used because it causes the @@ -71,7 +71,7 @@ CHAR64LONG16 block[1]; /* use array to appear as a pointer */ * And the result is written through. I threw a "const" in, hoping * this will cause a diagnostic. */ -CHAR64LONG16* block = (const CHAR64LONG16*)buffer; + CHAR64LONG16* block = (const CHAR64LONG16*)buffer; #endif /* Copy context->state[] to working vars */ a = state[0]; @@ -132,12 +132,11 @@ void SHA1Init(SHA1_CTX* context) void SHA1Update(SHA1_CTX* context, const unsigned char* data, u_int32_t len) { -u_int32_t i; -u_int32_t j; + u_int32_t i, j; j = context->count[0]; if ((context->count[0] += len << 3) < j) - context->count[1]++; + context->count[1]++; context->count[1] += (len>>29); j = (j >> 3) & 63; if ((j + len) > 63) { @@ -157,9 +156,9 @@ u_int32_t j; void SHA1Final(unsigned char digest[20], SHA1_CTX* context) { -unsigned i; -unsigned char finalcount[8]; -unsigned char c; + unsigned i; + unsigned char finalcount[8]; + unsigned char c; #if 0 /* untested "improvement" by DHR */ /* Convert context->count to a sequence of bytes @@ -170,12 +169,12 @@ unsigned char c; unsigned char *fcp = &finalcount[8]; for (i = 0; i < 2; i++) - { - u_int32_t t = context->count[i]; - int j; + { + u_int32_t t = context->count[i]; + int j; - for (j = 0; j < 4; t >>= 8, j++) - *--fcp = (unsigned char) t + for (j = 0; j < 4; t >>= 8, j++) + *--fcp = (unsigned char) t; } #else for (i = 0; i < 8; i++) { @@ -226,3 +225,4 @@ main(int argc, char **argv) } #endif + diff --git a/src/sort.c b/src/sort.c index 39505b13670..cd54072f3e9 100644 --- a/src/sort.c +++ b/src/sort.c @@ -45,7 +45,7 @@ redisSortOperation *createSortOperation(int type, robj *pattern) { /* Return the value associated to the key with a name obtained using * the following rules: * - * 1) The first occurence of '*' in 'pattern' is substituted with 'subst'. + * 1) The first occurrence of '*' in 'pattern' is substituted with 'subst'. * * 2) If 'pattern' matches the "->" string, everything on the left of * the arrow is treated as the name of an hash field, and the part on the @@ -147,7 +147,7 @@ int sortCompare(const void *s1, const void *s2) { cmp = -1; } else { /* Objects have the same score, but we don't want the comparison - * to be undefined, so we compare objects lexicographycally. + * to be undefined, so we compare objects lexicographically. * This way the result of SORT is deterministic. */ cmp = compareStringObjects(so1->obj,so2->obj); } @@ -205,7 +205,7 @@ void sortCommand(redisClient *c) { /* Now we need to protect sortval incrementing its count, in the future * SORT may have options able to overwrite/delete keys during the sorting - * and the sorted key itself may get destroied */ + * and the sorted key itself may get destroyed */ if (sortval) incrRefCount(sortval); else diff --git a/src/t_list.c b/src/t_list.c index 50db6c536b7..16b5e1be522 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -45,10 +45,10 @@ void listTypeTryConversion(robj *subject, robj *value) { listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST); } -/* The function pushes an elmenet to the specified list object 'subject', +/* The function pushes an element to the specified list object 'subject', * at head or tail position as specified by 'where'. * - * There is no need for the caller to incremnet the refcount of 'value' as + * There is no need for the caller to increment the refcount of 'value' as * the function takes care of it if needed. */ void listTypePush(robj *subject, robj *value, int where) { /* Check if we need to convert the ziplist */ @@ -825,7 +825,7 @@ void unblockClientWaitingData(redisClient *c) { /* If the specified key has clients blocked waiting for list pushes, this * function will put the key reference into the server.ready_keys list. * Note that db->ready_keys is an hash table that allows us to avoid putting - * the same key agains and again in the list in case of multiple pushes + * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * * The list will be finally processed by handleClientsBlockedOnLists() */ @@ -858,7 +858,7 @@ void signalListAsReady(redisClient *c, robj *key) { * * 1) Provide the client with the 'value' element. * 2) If the dstkey is not NULL (we are serving a BRPOPLPUSH) also push the - * 'value' element on the destionation list (the LPUSH side of the command). + * 'value' element on the destination list (the LPUSH side of the command). * 3) Propagate the resulting BRPOP, BLPOP and additional LPUSH if any into * the AOF and replication channel. * @@ -868,7 +868,7 @@ void signalListAsReady(redisClient *c, robj *key) { * * The function returns REDIS_OK if we are able to serve the client, otherwise * REDIS_ERR is returned to signal the caller that the list POP operation - * should be undoed as the client was not served: This only happens for + * should be undone as the client was not served: This only happens for * BRPOPLPUSH that fails to push the value to the destination key as it is * of the wrong type. */ int serveClientBlockedOnList(redisClient *receiver, robj *key, robj *dstkey, redisDb *db, robj *value, int where) diff --git a/src/t_set.c b/src/t_set.c index 01ac92aa468..0909d03010a 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -188,7 +188,7 @@ robj *setTypeNextObject(setTypeIterator *si) { * The caller provides both pointers to be populated with the right * object. The return value of the function is the object->encoding * field of the object and is used by the caller to check if the - * int64_t pointer or the redis object pointere was populated. + * int64_t pointer or the redis object pointer was populated. * * When an object is returned (the set was a real set) the ref count * of the object is not incremented so this function can be considered @@ -606,7 +606,7 @@ void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, sets[j] = setobj; } /* Sort sets from the smallest to largest, this will improve our - * algorithm's performace */ + * algorithm's performance */ qsort(sets,setnum,sizeof(robj*),qsortCompareSetsByCardinality); /* The first thing we should output is the total number of elements... diff --git a/src/t_string.c b/src/t_string.c index 00510c71047..8ba915a587b 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -43,7 +43,7 @@ static int checkStringLength(redisClient *c, long long size) { } void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expire, int unit) { - long long milliseconds = 0; /* initialized to avoid an harmness warning */ + long long milliseconds = 0; /* initialized to avoid any harmness warning */ if (expire) { if (getLongLongFromObjectOrReply(c, expire, &milliseconds, NULL) != REDIS_OK) @@ -340,7 +340,7 @@ void incrbyfloatCommand(redisClient *c) { addReplyBulk(c,new); /* Always replicate INCRBYFLOAT as a SET command with the final value - * in order to make sure that differences in float pricision or formatting + * in order to make sure that differences in float prrcision or formatting * will not create differences in replicas or after an AOF restart. */ aux = createStringObject("SET",3); rewriteClientCommandArgument(c,0,aux); diff --git a/src/t_zset.c b/src/t_zset.c index f5cdec3e4bc..0d92f681f92 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -125,7 +125,7 @@ zskiplistNode *zslInsert(zskiplist *zsl, double score, robj *obj) { } /* we assume the key is not already inside, since we allow duplicated * scores, and the re-insertion of score and redis object should never - * happpen since the caller of zslInsert() should test in the hash table + * happen since the caller of zslInsert() should test in the hash table * if the element is already inside or not. */ level = zslRandomLevel(); if (level > zsl->level) { @@ -285,7 +285,7 @@ zskiplistNode *zslLastInRange(zskiplist *zsl, zrangespec range) { } /* Delete all the elements with score between min and max from the skiplist. - * Min and mx are inclusive, so a score >= min || score <= max is deleted. + * Min and max are inclusive, so a score >= min || score <= max is deleted. * Note that this function takes the reference to the hash table view of the * sorted set, in order to remove the elements from the hash table too. */ unsigned long zslDeleteRangeByScore(zskiplist *zsl, zrangespec range, dict *dict) { diff --git a/src/zmalloc.c b/src/zmalloc.c index a9ba09435e4..21042582874 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -250,7 +250,7 @@ void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) { * * For this kind of "fast RSS reporting" usages use instead the * function RedisEstimateRSS() that is a much faster (and less precise) - * version of the funciton. */ + * version of the function. */ #if defined(HAVE_PROC_STAT) #include diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 3dbfc902bc2..e42f87725d9 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -34,7 +34,7 @@ start_server {tags {"scripting"}} { r eval {return {1,2,3,'ciao',{1,2}}} 0 } {1 2 3 ciao {1 2}} - test {EVAL - Are the KEYS and ARGS arrays populated correctly?} { + test {EVAL - Are the KEYS and ARGV arrays populated correctly?} { r eval {return {KEYS[1],KEYS[2],ARGV[1],ARGV[2]}} 2 a b c d } {a b c d} From d766907cfb1824b62a3c46acadc87c60dee950ec Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 19 Jan 2013 12:52:05 +0100 Subject: [PATCH 0441/1752] Slowlog: don't log EXEC but just the executed commands. The Redis Slow Log always used to log the slow commands executed inside a MULTI/EXEC block. However also EXEC was logged at the end, which is perfectly useless. Now EXEC is no longer logged and a test was added to test this behavior. This fixes issue #759. --- src/redis.c | 2 +- tests/unit/slowlog.tcl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 2a56ad2865a..81d74741dd0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1511,7 +1511,7 @@ void call(redisClient *c, int flags) { /* Log the command into the Slow log if needed, and populate the * per-command statistics that we show in INFO commandstats. */ - if (flags & REDIS_CALL_SLOWLOG) + if (flags & REDIS_CALL_SLOWLOG && c->cmd->proc != execCommand) slowlogPushEntryIfNeeded(c->argv,c->argc,duration); if (flags & REDIS_CALL_STATS) { c->cmd->microseconds += duration; diff --git a/tests/unit/slowlog.tcl b/tests/unit/slowlog.tcl index 2216e925a70..b25b91e2ce7 100644 --- a/tests/unit/slowlog.tcl +++ b/tests/unit/slowlog.tcl @@ -55,4 +55,16 @@ start_server {tags {"slowlog"} overrides {slowlog-log-slower-than 1000000}} { set e [lindex [r slowlog get] 0] lindex $e 3 } {sadd set foo {AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA... (1 more bytes)}} + + test {SLOWLOG - EXEC is not logged, just executed commands} { + r config set slowlog-log-slower-than 100000 + r slowlog reset + assert_equal [r slowlog len] 0 + r multi + r debug sleep 0.2 + r exec + assert_equal [r slowlog len] 1 + set e [lindex [r slowlog get] 0] + assert_equal [lindex $e 3] {debug sleep 0.2} + } } From 1e20c939fe2ee1980f363dfe0fa5a95588c5da35 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 19 Jan 2013 13:19:41 +0100 Subject: [PATCH 0442/1752] Clear server.shutdown_asap on failed shutdown. When a SIGTERM is received Redis schedules a shutdown. However if it fails to perform the shutdown it must be clear the shutdown_asap flag otehrwise it will try again and again possibly making the server unusable. --- src/redis.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis.c b/src/redis.c index 81d74741dd0..00774bae311 100644 --- a/src/redis.c +++ b/src/redis.c @@ -850,6 +850,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (server.shutdown_asap) { if (prepareForShutdown(0) == REDIS_OK) exit(0); redisLog(REDIS_WARNING,"SIGTERM received but errors trying to shut down the server, check the logs for more information"); + server.shutdown_asap = 0; } /* Show some info about non-empty databases */ From 39f0a33f78bb61d8afb3d08b770247ad1673a1ea Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 14 Jan 2013 10:29:14 +0100 Subject: [PATCH 0443/1752] Whitelist SIGUSR1 to avoid auto-triggering errors. This commit fixes issue #875 that was caused by the following events: 1) There is an active child doing BGSAVE. 2) flushall is called (or any other condition that makes Redis killing the saving child process). 3) An error is sensed by Redis as the child exited with an error (killed by a singal), that stops accepting write commands until a BGSAVE happens to be executed with success. Whitelisting SIGUSR1 and making sure Redis always uses this signal in order to kill its own children fixes the issue. --- src/aof.c | 2 +- src/db.c | 2 +- src/rdb.c | 5 ++++- src/redis.c | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/aof.c b/src/aof.c index 7e1512ebdbd..8dc1dd1881f 100644 --- a/src/aof.c +++ b/src/aof.c @@ -177,7 +177,7 @@ void stopAppendOnly(void) { redisLog(REDIS_NOTICE,"Killing running AOF rewrite child: %ld", (long) server.aof_child_pid); - if (kill(server.aof_child_pid,SIGKILL) != -1) + if (kill(server.aof_child_pid,SIGUSR1) != -1) wait3(&statloc,0,NULL); /* reset the buffer accumulating changes while the child saves */ aofRewriteBufferReset(); diff --git a/src/db.c b/src/db.c index 1cb89e3c2bc..a7b196f5a84 100644 --- a/src/db.c +++ b/src/db.c @@ -219,7 +219,7 @@ void flushallCommand(redisClient *c) { server.dirty += emptyDb(); addReply(c,shared.ok); if (server.rdb_child_pid != -1) { - kill(server.rdb_child_pid,SIGKILL); + kill(server.rdb_child_pid,SIGUSR1); rdbRemoveTempFile(server.rdb_child_pid); } if (server.saveparamslen > 0) { diff --git a/src/rdb.c b/src/rdb.c index 17f2ef840e6..a5e4c47af42 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1194,7 +1194,10 @@ void backgroundSaveDoneHandler(int exitcode, int bysignal) { redisLog(REDIS_WARNING, "Background saving terminated by signal %d", bysignal); rdbRemoveTempFile(server.rdb_child_pid); - server.lastbgsave_status = REDIS_ERR; + /* SIGUSR1 is whitelisted, so we have a way to kill a child without + * tirggering an error conditon. */ + if (bysignal != SIGUSR1) + server.lastbgsave_status = REDIS_ERR; } server.rdb_child_pid = -1; server.rdb_save_time_last = time(NULL)-server.rdb_save_time_start; diff --git a/src/redis.c b/src/redis.c index 00774bae311..cb78de7ed75 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1695,7 +1695,7 @@ int prepareForShutdown(int flags) { overwrite the synchronous saving did by SHUTDOWN. */ if (server.rdb_child_pid != -1) { redisLog(REDIS_WARNING,"There is a child saving an .rdb. Killing it!"); - kill(server.rdb_child_pid,SIGKILL); + kill(server.rdb_child_pid,SIGUSR1); rdbRemoveTempFile(server.rdb_child_pid); } if (server.aof_state != REDIS_AOF_OFF) { @@ -1704,7 +1704,7 @@ int prepareForShutdown(int flags) { if (server.aof_child_pid != -1) { redisLog(REDIS_WARNING, "There is a child rewriting the AOF. Killing it!"); - kill(server.aof_child_pid,SIGKILL); + kill(server.aof_child_pid,SIGUSR1); } /* Append only file: fsync() the AOF and exit */ redisLog(REDIS_NOTICE,"Calling fsync() on the AOF file."); From 635c532c89b2235d3de329f1ee45d4ccafaeed3e Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 19 Jan 2013 13:46:14 +0100 Subject: [PATCH 0444/1752] Additionally two typos fixed thanks to @jodal --- src/networking.c | 2 +- src/t_string.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index f48d27c0723..6e5d3248986 100644 --- a/src/networking.c +++ b/src/networking.c @@ -941,7 +941,7 @@ int processMultibulkBuffer(redisClient *c) { /* Not enough data (+2 == trailing \r\n) */ break; } else { - /* Optimization: if the buffer containns JUST our bulk element + /* Optimization: if the buffer contains JUST our bulk element * instead of creating a new object by *copying* the sds we * just use the current sds string. */ if (pos == 0 && diff --git a/src/t_string.c b/src/t_string.c index 8ba915a587b..0a7f2258391 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -340,7 +340,7 @@ void incrbyfloatCommand(redisClient *c) { addReplyBulk(c,new); /* Always replicate INCRBYFLOAT as a SET command with the final value - * in order to make sure that differences in float prrcision or formatting + * in order to make sure that differences in float precision or formatting * will not create differences in replicas or after an AOF restart. */ aux = createStringObject("SET",3); rewriteClientCommandArgument(c,0,aux); From f6f43d7da0aa8edcc4770c0cb79758568b6ff14b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Jan 2013 12:06:54 +0100 Subject: [PATCH 0445/1752] Not every __sun has backtrace(). I don't know how to test for Open Solaris that has support for backtrace() so for now removing the #ifdef that breaks compilation under other Solaris flavors. --- src/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.h b/src/config.h index 472f10a3a4b..68e99b427d2 100644 --- a/src/config.h +++ b/src/config.h @@ -56,7 +56,7 @@ #endif /* Test for backtrace() */ -#if defined(__APPLE__) || defined(__linux__) || defined(__sun) +#if defined(__APPLE__) || defined(__linux__) #define HAVE_BACKTRACE 1 #endif From 2f1318ab29546baa3bb07277e163d0fa6e03449e Mon Sep 17 00:00:00 2001 From: Bilal Husain Date: Wed, 9 Jan 2013 21:46:58 +0530 Subject: [PATCH 0446/1752] s/adiacent/adjacent/ fixed typo in a comment (step 2 memcheck) --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index 897a71acea2..d6df93c62e9 100644 --- a/src/debug.c +++ b/src/debug.c @@ -714,7 +714,7 @@ int memtest_test_linux_anonymous_maps(void) { crc1 = crc64(crc1,(void*)start_vect[j],size_vect[j]); } - /* 2) Invert bits, swap adiacent words, swap again, invert bits. + /* 2) Invert bits, swap adjacent words, swap again, invert bits. * This is the error amplification step. */ for (j = 0; j < regions; j++) memtest_non_destructive_invert((void*)start_vect[j],size_vect[j]); From 850117a8a5897b3231af3068960c13304f2fdb69 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Jan 2013 12:34:22 +0100 Subject: [PATCH 0447/1752] Fixed a bug in memtest progress bar, that had no actual effects. This closes issue #859, thanks to @erbenmo. --- src/memtest.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/memtest.c b/src/memtest.c index 754d0202450..cabfb5c8934 100644 --- a/src/memtest.c +++ b/src/memtest.c @@ -79,10 +79,8 @@ void memtest_progress_end(void) { void memtest_progress_step(size_t curr, size_t size, char c) { size_t chars = ((unsigned long long)curr*progress_full)/size, j; - for (j = 0; j < chars-progress_printed; j++) { - printf("%c",c); - progress_printed++; - } + for (j = 0; j < chars-progress_printed; j++) printf("%c",c); + progress_printed = chars; fflush(stdout); } From 3ff75e58e8d39cf008ee26e026faf16f79631b73 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Jan 2013 18:50:16 +0100 Subject: [PATCH 0448/1752] UNSUBSCRIBE and PUNSUBSCRIBE: always provide a reply. UNSUBSCRIBE and PUNSUBSCRIBE commands are designed to mass-unsubscribe the client respectively all the channels and patters if called without arguments. However when these functions are called without arguments, but there are no channels or patters we are subscribed to, the old behavior was to don't reply at all. This behavior is broken, as every command should always reply. Also it is possible that we are no longer subscribed to a channels but we are subscribed to patters or the other way around, and the client should be notified with the correct number of subscriptions. Also it is not pretty that sometimes we did not receive a reply at all in a redis-cli session from these commands, blocking redis-cli trying to read the reply. This fixes issue #714. --- src/pubsub.c | 16 ++++++++++++++++ tests/unit/pubsub.tcl | 12 +++++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/pubsub.c b/src/pubsub.c index 973692948ef..3e58e3857eb 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -179,6 +179,14 @@ int pubsubUnsubscribeAllChannels(redisClient *c, int notify) { count += pubsubUnsubscribeChannel(c,channel,notify); } + /* We were subscribed to nothing? Still reply to the client. */ + if (notify && count == 0) { + addReply(c,shared.mbulkhdr[3]); + addReply(c,shared.unsubscribebulk); + addReply(c,shared.nullbulk); + addReplyLongLong(c,dictSize(c->pubsub_channels)+ + listLength(c->pubsub_patterns)); + } dictReleaseIterator(di); return count; } @@ -196,6 +204,14 @@ int pubsubUnsubscribeAllPatterns(redisClient *c, int notify) { count += pubsubUnsubscribePattern(c,pattern,notify); } + if (notify && count == 0) { + /* We were subscribed to nothing? Still reply to the client. */ + addReply(c,shared.mbulkhdr[3]); + addReply(c,shared.punsubscribebulk); + addReply(c,shared.nullbulk); + addReplyLongLong(c,dictSize(c->pubsub_channels)+ + listLength(c->pubsub_patterns)); + } return count; } diff --git a/tests/unit/pubsub.tcl b/tests/unit/pubsub.tcl index c8b547b4f1b..7691516009d 100644 --- a/tests/unit/pubsub.tcl +++ b/tests/unit/pubsub.tcl @@ -192,4 +192,14 @@ start_server {tags {"pubsub"}} { # clean up clients $rd1 close } -} \ No newline at end of file + + test "PUNSUBSCRIBE and UNSUBSCRIBE should always reply." { + # Make sure we are not subscribed to any channel at all. + r punsubscribe + r unsubscribe + # Now check if the commands still reply correctly. + set reply1 [r punsubscribe] + set reply2 [r unsubscribe] + concat $reply1 $reply2 + } {punsubscribe {} 0 unsubscribe {} 0} +} From 767a53aa845bcd93c46809e243dffe35d8389960 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 21 Jan 2013 19:15:51 +0100 Subject: [PATCH 0449/1752] redis-cli --bigkeys output is now simpler to understand. --- src/redis-cli.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 318f1822ab7..c182ac17d1b 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1183,6 +1183,7 @@ static void findBigKeys(void) { unsigned long long samples = 0; redisReply *reply1, *reply2, *reply3 = NULL; char *sizecmd, *typename[] = {"string","list","set","hash","zset"}; + char *typeunit[] = {"bytes","items","members","fields","members"}; int type; printf("\n# Press ctrl+c when you have had enough of it... :)\n"); @@ -1234,9 +1235,10 @@ static void findBigKeys(void) { reply3 = redisCommand(context,"%s %s", sizecmd, reply1->str); if (reply3 && reply3->type == REDIS_REPLY_INTEGER) { if (biggest[type] < reply3->integer) { - printf("[%6s] %s | biggest so far with size %llu\n", + printf("Biggest %-6s found so far '%s' with %llu %s.\n", typename[type], reply1->str, - (unsigned long long) reply3->integer); + (unsigned long long) reply3->integer, + typeunit[type]); biggest[type] = reply3->integer; } } From 153766e1378f83ed223a11f7f712d1e918e0dfbe Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Jan 2013 10:48:59 +0100 Subject: [PATCH 0450/1752] Lua struct library updated to version 0.2. There was a bug in the previous version of this library that caused a crash under the circumstances described in issue #901. The newer version of the library appears to be fixed (I tested it manually with valgrind and everything seems fine now). For more information about this library please visit this web site: http://www.inf.puc-rio.br/~roberto/struct/ --- deps/lua/src/lua_struct.c | 171 ++++++++++++++++++++++++++------------ 1 file changed, 119 insertions(+), 52 deletions(-) diff --git a/deps/lua/src/lua_struct.c b/deps/lua/src/lua_struct.c index 6807c838748..ec78bcbc02b 100644 --- a/deps/lua/src/lua_struct.c +++ b/deps/lua/src/lua_struct.c @@ -1,18 +1,7 @@ - -#include -#include -#include -#include - - -#include "lua.h" -#include "lauxlib.h" - - /* ** {====================================================== ** Library for packing/unpacking structures. -** $Id: struct.c,v 1.2 2008/04/18 20:06:01 roberto Exp $ +** $Id: struct.c,v 1.4 2012/07/04 18:54:29 roberto Exp $ ** See Copyright Notice at the end of this file ** ======================================================= */ @@ -25,6 +14,7 @@ ** b/B - signed/unsigned byte ** h/H - signed/unsigned short ** l/L - signed/unsigned long +** T - size_t ** i/In - signed/unsigned integer with size `n' (default is size of int) ** cn - sequence of `n' chars (from/to a string); when packing, n==0 means the whole string; when unpacking, n==0 means use the previous @@ -36,6 +26,38 @@ */ +#include +#include +#include +#include +#include + + +#include "lua.h" +#include "lauxlib.h" + + +#if (LUA_VERSION_NUM >= 502) + +#define luaL_register(L,n,f) luaL_newlib(L,f) + +#endif + + +/* basic integer type */ +#if !defined(STRUCT_INT) +#define STRUCT_INT long +#endif + +typedef STRUCT_INT Inttype; + +/* corresponding unsigned version */ +typedef unsigned STRUCT_INT Uinttype; + + +/* maximum size (in bytes) for integral types */ +#define MAXINTSIZE 32 + /* is 'x' a power of 2? */ #define isp2(x) ((x) > 0 && ((x) & ((x) - 1)) == 0) @@ -67,11 +89,11 @@ typedef struct Header { } Header; -static size_t getnum (const char **fmt, size_t df) { +static int getnum (const char **fmt, int df) { if (!isdigit(**fmt)) /* no number? */ return df; /* return default value */ else { - size_t a = 0; + int a = 0; do { a = a*10 + *((*fmt)++) - '0'; } while (isdigit(**fmt)); @@ -89,33 +111,40 @@ static size_t optsize (lua_State *L, char opt, const char **fmt) { case 'B': case 'b': return sizeof(char); case 'H': case 'h': return sizeof(short); case 'L': case 'l': return sizeof(long); + case 'T': return sizeof(size_t); case 'f': return sizeof(float); case 'd': return sizeof(double); case 'x': return 1; case 'c': return getnum(fmt, 1); - case 's': case ' ': case '<': case '>': case '!': return 0; case 'i': case 'I': { int sz = getnum(fmt, sizeof(int)); - if (!isp2(sz)) - luaL_error(L, "integral size %d is not a power of 2", sz); + if (sz > MAXINTSIZE) + luaL_error(L, "integral size %d is larger than limit of %d", + sz, MAXINTSIZE); return sz; } - default: { - const char *msg = lua_pushfstring(L, "invalid format option [%c]", opt); - return luaL_argerror(L, 1, msg); - } + default: return 0; /* other cases do not need alignment */ } } +/* +** return number of bytes needed to align an element of size 'size' +** at current position 'len' +*/ static int gettoalign (size_t len, Header *h, int opt, size_t size) { if (size == 0 || opt == 'c') return 0; - if (size > (size_t)h->align) size = h->align; /* respect max. alignment */ - return (size - (len & (size - 1))) & (size - 1); + if (size > (size_t)h->align) + size = h->align; /* respect max. alignment */ + return (size - (len & (size - 1))) & (size - 1); } -static void commoncases (lua_State *L, int opt, const char **fmt, Header *h) { +/* +** options to control endianess and alignment +*/ +static void controloptions (lua_State *L, int opt, const char **fmt, + Header *h) { switch (opt) { case ' ': return; /* ignore white spaces */ case '>': h->endian = BIG; return; @@ -127,7 +156,10 @@ static void commoncases (lua_State *L, int opt, const char **fmt, Header *h) { h->align = a; return; } - default: assert(0); + default: { + const char *msg = lua_pushfstring(L, "invalid format option '%c'", opt); + luaL_argerror(L, 1, msg); + } } } @@ -135,21 +167,27 @@ static void commoncases (lua_State *L, int opt, const char **fmt, Header *h) { static void putinteger (lua_State *L, luaL_Buffer *b, int arg, int endian, int size) { lua_Number n = luaL_checknumber(L, arg); - unsigned long value; - if (n < (lua_Number)LONG_MAX) - value = (long)n; + Uinttype value; + char buff[MAXINTSIZE]; + if (n < 0) + value = (Uinttype)(Inttype)n; else - value = (unsigned long)n; + value = (Uinttype)n; if (endian == LITTLE) { int i; - for (i = 0; i < size; i++) - luaL_addchar(b, (value >> 8*i) & 0xff); + for (i = 0; i < size; i++) { + buff[i] = (value & 0xff); + value >>= 8; + } } else { int i; - for (i = size - 1; i >= 0; i--) - luaL_addchar(b, (value >> 8*i) & 0xff); + for (i = size - 1; i >= 0; i--) { + buff[i] = (value & 0xff); + value >>= 8; + } } + luaL_addlstring(b, buff, size); } @@ -179,15 +217,15 @@ static int b_pack (lua_State *L) { size_t size = optsize(L, opt, &fmt); int toalign = gettoalign(totalsize, &h, opt, size); totalsize += toalign; - while (toalign-- > 0) luaL_putchar(&b, '\0'); + while (toalign-- > 0) luaL_addchar(&b, '\0'); switch (opt) { case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'i': case 'I': { /* integer types */ + case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ putinteger(L, &b, arg++, h.endian, size); break; } case 'x': { - luaL_putchar(&b, '\0'); + luaL_addchar(&b, '\0'); break; } case 'f': { @@ -209,12 +247,12 @@ static int b_pack (lua_State *L) { luaL_argcheck(L, l >= (size_t)size, arg, "string too short"); luaL_addlstring(&b, s, size); if (opt == 's') { - luaL_putchar(&b, '\0'); /* add zero at the end */ + luaL_addchar(&b, '\0'); /* add zero at the end */ size++; } break; } - default: commoncases(L, opt, &fmt, &h); + default: controloptions(L, opt, &fmt, &h); } totalsize += size; } @@ -225,24 +263,27 @@ static int b_pack (lua_State *L) { static lua_Number getinteger (const char *buff, int endian, int issigned, int size) { - unsigned long l = 0; + Uinttype l = 0; + int i; if (endian == BIG) { - int i; - for (i = 0; i < size; i++) - l |= (unsigned long)(unsigned char)buff[size - i - 1] << (i*8); + for (i = 0; i < size; i++) { + l <<= 8; + l |= (Uinttype)(unsigned char)buff[i]; + } } else { - int i; - for (i = 0; i < size; i++) - l |= (unsigned long)(unsigned char)buff[i] << (i*8); + for (i = size - 1; i >= 0; i--) { + l <<= 8; + l |= (Uinttype)(unsigned char)buff[i]; + } } if (!issigned) return (lua_Number)l; else { /* signed format */ - unsigned long mask = ~(0UL) << (size*8 - 1); + Uinttype mask = (Uinttype)(~((Uinttype)0)) << (size*8 - 1); if (l & mask) /* negative value? */ l |= mask; /* signal extension */ - return (lua_Number)(long)l; + return (lua_Number)(Inttype)l; } } @@ -260,9 +301,10 @@ static int b_unpack (lua_State *L) { size_t size = optsize(L, opt, &fmt); pos += gettoalign(pos, &h, opt, size); luaL_argcheck(L, pos+size <= ld, 2, "data string too short"); + luaL_checkstack(L, 1, "too many results"); switch (opt) { case 'b': case 'B': case 'h': case 'H': - case 'l': case 'L': case 'i': case 'I': { /* integer types */ + case 'l': case 'L': case 'T': case 'i': case 'I': { /* integer types */ int issigned = islower(opt); lua_Number res = getinteger(data+pos, h.endian, issigned, size); lua_pushnumber(L, res); @@ -304,7 +346,7 @@ static int b_unpack (lua_State *L) { lua_pushlstring(L, data+pos, size - 1); break; } - default: commoncases(L, opt, &fmt, &h); + default: controloptions(L, opt, &fmt, &h); } pos += size; } @@ -312,26 +354,50 @@ static int b_unpack (lua_State *L) { return lua_gettop(L) - 2; } + +static int b_size (lua_State *L) { + Header h; + const char *fmt = luaL_checkstring(L, 1); + size_t pos = 0; + defaultoptions(&h); + while (*fmt) { + int opt = *fmt++; + size_t size = optsize(L, opt, &fmt); + pos += gettoalign(pos, &h, opt, size); + if (opt == 's') + luaL_argerror(L, 1, "option 's' has no fixed size"); + else if (opt == 'c' && size == 0) + luaL_argerror(L, 1, "option 'c0' has no fixed size"); + if (!isalnum(opt)) + controloptions(L, opt, &fmt, &h); + pos += size; + } + lua_pushinteger(L, pos); + return 1; +} + /* }====================================================== */ -static const struct luaL_reg thislib[] = { +static const struct luaL_Reg thislib[] = { {"pack", b_pack}, {"unpack", b_unpack}, + {"size", b_size}, {NULL, NULL} }; +LUALIB_API int luaopen_struct (lua_State *L); + LUALIB_API int luaopen_struct (lua_State *L) { luaL_register(L, "struct", thislib); return 1; } - /****************************************************************************** -* Copyright (C) 2010 Lua.org, PUC-Rio. All rights reserved. +* Copyright (C) 2010-2012 Lua.org, PUC-Rio. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the @@ -352,3 +418,4 @@ LUALIB_API int luaopen_struct (lua_State *L) { * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ******************************************************************************/ + From 212edbc409b7e9d37fb3a9ebab2988ccd1837bc6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Jan 2013 16:23:33 +0100 Subject: [PATCH 0451/1752] Keyspace events notification API. --- redis.conf | 17 +++++++++++ src/Makefile | 2 +- src/Makefile.dep | 3 ++ src/config.c | 9 ++++-- src/notify.c | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ src/redis.c | 1 + src/redis.h | 2 ++ 7 files changed, 108 insertions(+), 3 deletions(-) create mode 100644 src/notify.c diff --git a/redis.conf b/redis.conf index 797347c69ea..d7bdcfd53d4 100644 --- a/redis.conf +++ b/redis.conf @@ -445,6 +445,23 @@ slowlog-log-slower-than 10000 # You can reclaim memory used by the slow log with SLOWLOG RESET. slowlog-max-len 128 +############################# Event notification ############################## + +# Redis can notify Pub/Sub clients about events happening in the key space. +# This feature is documented at http://redis.io/topics/keyspace-events +# +# For instance if keyspace events notification is enabled, and a client +# performs a DEL operation on key "foo" stored in the Database 0, two +# messages will be published via Pub/Sub: +# +# PUBLISH __keyspace@0__:foo del +# PUBLISH __keyevent@0__:del foo +# +# While the overhead of this feature is relatively small most users don't +# need it so it is disabled by default. You can enable it setting the +# following configuration option to yes. +notify-keyspace-events no + ############################### ADVANCED CONFIG ############################### # Hashes are encoded using a memory efficient data structure when they have a diff --git a/src/Makefile b/src/Makefile index 9d0ec4eb85c..242cf099fcd 100644 --- a/src/Makefile +++ b/src/Makefile @@ -99,7 +99,7 @@ endif REDIS_SERVER_NAME= redis-server REDIS_SENTINEL_NAME= redis-sentinel -REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o +REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o REDIS_CLI_NAME= redis-cli REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o REDIS_BENCHMARK_NAME= redis-benchmark diff --git a/src/Makefile.dep b/src/Makefile.dep index 4e07a563d78..35025070c7e 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -40,6 +40,9 @@ networking.o: networking.c redis.h fmacros.h config.h \ ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ rio.h +notify.o: notify.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ + ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ + ziplist.h intset.h version.h util.h rdb.h rio.h object.o: object.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h diff --git a/src/config.c b/src/config.c index 1fa498d3c8c..bfe73482fc7 100644 --- a/src/config.c +++ b/src/config.c @@ -384,6 +384,10 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { server.slave_priority = atoi(argv[1]); + } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { + if ((server.notify_keyspace_events = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"sentinel")) { /* argc == 1 is handled by main() as we need to enter the sentinel * mode ASAP. */ @@ -702,11 +706,11 @@ void configSetCommand(redisClient *c) { if (yn == -1) goto badfmt; server.rdb_compression = yn; - } else if (!strcasecmp(c->argv[2]->ptr,"rdbchecksum")) { + } else if (!strcasecmp(c->argv[2]->ptr,"notify-keyspace-events")) { int yn = yesnotoi(o->ptr); if (yn == -1) goto badfmt; - server.rdb_checksum = yn; + server.notify_keyspace_events = yn; } else if (!strcasecmp(c->argv[2]->ptr,"slave-priority")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; @@ -816,6 +820,7 @@ void configGetCommand(redisClient *c) { config_get_bool_field("rdbcompression", server.rdb_compression); config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); + config_get_bool_field("notify-keyspace-events", server.notify_keyspace_events); /* Everything we can't handle with macros follows. */ diff --git a/src/notify.c b/src/notify.c new file mode 100644 index 00000000000..70c96566389 --- /dev/null +++ b/src/notify.c @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2013, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "redis.h" + +/* This file implements keyspace events notification via Pub/Sub ad + * described at http://redis.io/topics/keyspace-events. + * + * The API provided to the rest of the Redis core is a simple function: + * + * notifyKeyspaceEvent(char *event, robj *key, int dbid); + * + * 'event' is a C string representing the event name. + * 'key' is a Redis object representing the key name. + * 'dbid' is the database ID where the key lives. + */ + +void notifyKeyspaceEvent(char *event, robj *key, int dbid) { + sds keyspace_chan, keyevent_chan; + int len; + char buf[24]; + robj *chan1, *chan2, *eventobj; + + /* The prefix of the two channels is identical if not for + * 'keyspace' that is 'keyevent' in the event channel name, so + * we build a single prefix and overwrite 'event' with 'space'. */ + keyspace_chan = sdsnewlen("__keyspace@",11); + len = ll2string(buf,sizeof(buf),dbid); + keyspace_chan = sdscatlen(keyspace_chan, buf, len); + keyspace_chan = sdscatlen(keyspace_chan, "__:", 3); + keyevent_chan = sdsdup(keyspace_chan); /* Dup the prefix. */ + memcpy(keyevent_chan+5,"event",5); /* Fix it. */ + + eventobj = createStringObject(event,strlen(event)); + + /* The keyspace channel name has a trailing key name, while + * the keyevent channel name has a trailing event name. */ + keyspace_chan = sdscatsds(keyspace_chan, key->ptr); + keyevent_chan = sdscatsds(keyspace_chan, eventobj->ptr); + chan1 = createObject(REDIS_STRING, keyspace_chan); + chan2 = createObject(REDIS_STRING, keyevent_chan); + + /* Finally publish the two notifications. */ + pubsubPublishMessage(chan1, eventobj); + pubsubPublishMessage(chan2, key); + + /* Release objects. */ + decrRefCount(eventobj); + decrRefCount(keyspace_chan); + decrRefCount(keyevent_chan); +} diff --git a/src/redis.c b/src/redis.c index cb78de7ed75..ce72dd93d7e 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1125,6 +1125,7 @@ void initServerConfig() { server.rdb_compression = 1; server.rdb_checksum = 1; server.activerehashing = 1; + server.notify_keyspace_events = 0; server.maxclients = REDIS_MAX_CLIENTS; server.bpop_blocked_clients = 0; server.maxmemory = 0; diff --git a/src/redis.h b/src/redis.h index 334d7b65d10..601725a3351 100644 --- a/src/redis.h +++ b/src/redis.h @@ -646,6 +646,7 @@ struct redisServer { /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ + int notify_keyspace_events; /* Propagate keyspace events via Pub/Sub. */ /* Scripting */ lua_State *lua; /* The Lua interpreter. We use just one for all clients */ redisClient *lua_client; /* The "fake client" to query Redis from Lua */ @@ -999,6 +1000,7 @@ int pubsubUnsubscribeAllPatterns(redisClient *c, int notify); void freePubsubPattern(void *p); int listMatchPubsubPattern(void *a, void *b); int pubsubPublishMessage(robj *channel, robj *message); +void notifyKeyspaceEvent(char *event, robj *key, int dbid); /* Configuration */ void loadServerConfig(char *filename, char *options); From 6e64525cc2595363144dee749699fe9eee152fe2 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Jan 2013 16:38:47 +0100 Subject: [PATCH 0452/1752] Two fixes to initial keyspace notifications API. --- src/notify.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/notify.c b/src/notify.c index 70c96566389..fc462d80fa0 100644 --- a/src/notify.c +++ b/src/notify.c @@ -47,6 +47,8 @@ void notifyKeyspaceEvent(char *event, robj *key, int dbid) { char buf[24]; robj *chan1, *chan2, *eventobj; + if (!server.notify_keyspace_events) return; + /* The prefix of the two channels is identical if not for * 'keyspace' that is 'keyevent' in the event channel name, so * we build a single prefix and overwrite 'event' with 'space'. */ @@ -62,7 +64,7 @@ void notifyKeyspaceEvent(char *event, robj *key, int dbid) { /* The keyspace channel name has a trailing key name, while * the keyevent channel name has a trailing event name. */ keyspace_chan = sdscatsds(keyspace_chan, key->ptr); - keyevent_chan = sdscatsds(keyspace_chan, eventobj->ptr); + keyevent_chan = sdscatsds(keyevent_chan, eventobj->ptr); chan1 = createObject(REDIS_STRING, keyspace_chan); chan2 = createObject(REDIS_STRING, keyevent_chan); From dd2ce743ca1b5cd6d7bcd06b64db388b10bda019 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Jan 2013 16:44:45 +0100 Subject: [PATCH 0453/1752] Fixed over-80-cols comment in db.c --- src/db.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index a7b196f5a84..43bff1d500b 100644 --- a/src/db.c +++ b/src/db.c @@ -376,7 +376,8 @@ void renameGenericCommand(redisClient *c, int nx) { addReply(c,shared.czero); return; } - /* Overwrite: delete the old key before creating the new one with the same name. */ + /* Overwrite: delete the old key before creating the new one + * with the same name. */ dbDelete(c->db,c->argv[2]); } dbAdd(c->db,c->argv[2],o); From fdfb59beaea5a12b62f4db8f81281fff1c4a0ad8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 23 Jan 2013 17:04:18 +0100 Subject: [PATCH 0454/1752] Initial test events for the new keyspace notification API. --- src/db.c | 3 +++ src/redis.c | 2 ++ src/t_string.c | 6 ++++++ 3 files changed, 11 insertions(+) diff --git a/src/db.c b/src/db.c index 43bff1d500b..8e6232d144e 100644 --- a/src/db.c +++ b/src/db.c @@ -238,6 +238,7 @@ void delCommand(redisClient *c) { for (j = 1; j < c->argc; j++) { if (dbDelete(c->db,c->argv[j])) { signalModifiedKey(c->db,c->argv[j]); + notifyKeyspaceEvent("del",c->argv[j],c->db->id); server.dirty++; deleted++; } @@ -385,6 +386,8 @@ void renameGenericCommand(redisClient *c, int nx) { dbDelete(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[2]); + notifyKeyspaceEvent("rename_from",c->argv[1],c->db->id); + notifyKeyspaceEvent("rename_to",c->argv[2],c->db->id); server.dirty++; addReply(c,nx ? shared.cone : shared.ok); } diff --git a/src/redis.c b/src/redis.c index ce72dd93d7e..7bbb65fc601 100644 --- a/src/redis.c +++ b/src/redis.c @@ -666,6 +666,7 @@ void activeExpireCycle(void) { propagateExpire(db,keyobj); dbDelete(db,keyobj); + notifyKeyspaceEvent("expired",keyobj,db->id); decrRefCount(keyobj); expired++; server.stat_expiredkeys++; @@ -2359,6 +2360,7 @@ int freeMemoryIfNeeded(void) { delta -= (long long) zmalloc_used_memory(); mem_freed += delta; server.stat_evictedkeys++; + notifyKeyspaceEvent("evicted",keyobj,db->id); decrRefCount(keyobj); keys_freed++; diff --git a/src/t_string.c b/src/t_string.c index 0a7f2258391..a3e58e5b9ee 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -62,6 +62,8 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir setKey(c->db,key,val); server.dirty++; if (expire) setExpire(c->db,key,mstime()+milliseconds); + notifyKeyspaceEvent("set",key,c->db->id); + if (expire) notifyKeyspaceEvent("expire",key,c->db->id); addReply(c, nx ? shared.cone : shared.ok); } @@ -253,6 +255,7 @@ void msetGenericCommand(redisClient *c, int nx) { for (j = 1; j < c->argc; j += 2) { c->argv[j+1] = tryObjectEncoding(c->argv[j+1]); setKey(c->db,c->argv[j],c->argv[j+1]); + notifyKeyspaceEvent("set",c->argv[j+1],c->db->id); } server.dirty += (c->argc-1)/2; addReply(c, nx ? shared.cone : shared.ok); @@ -287,6 +290,7 @@ void incrDecrCommand(redisClient *c, long long incr) { else dbAdd(c->db,c->argv[1],new); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("incrby",c->argv[1],c->db->id); server.dirty++; addReply(c,shared.colon); addReply(c,new); @@ -336,6 +340,7 @@ void incrbyfloatCommand(redisClient *c) { else dbAdd(c->db,c->argv[1],new); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("incrbyfloat",c->argv[1],c->db->id); server.dirty++; addReplyBulk(c,new); @@ -383,6 +388,7 @@ void appendCommand(redisClient *c) { totlen = sdslen(o->ptr); } signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("append",c->argv[1],c->db->id); server.dirty++; addReplyLongLong(c,totlen); } From a64383c8610bee2db6a3089bb9a7c6a39c431481 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Jan 2013 11:24:40 +0100 Subject: [PATCH 0455/1752] notifyKeyspaceEvent(): release channel names using the right pointers. --- src/notify.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/notify.c b/src/notify.c index fc462d80fa0..da1d4d8943d 100644 --- a/src/notify.c +++ b/src/notify.c @@ -74,6 +74,6 @@ void notifyKeyspaceEvent(char *event, robj *key, int dbid) { /* Release objects. */ decrRefCount(eventobj); - decrRefCount(keyspace_chan); - decrRefCount(keyevent_chan); + decrRefCount(chan1); + decrRefCount(chan2); } From 2825f21fd80f05865cc977cdcf20d6be159f586f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Jan 2013 11:27:10 +0100 Subject: [PATCH 0456/1752] Fix decrRefCount() prototype from void to robj pointer. decrRefCount used to get its argument as a void* pointer in order to be used as destructor where a 'void free_object(void*)' prototype is expected. However this made simpler to introduce bugs by freeing the wrong pointer. This commit fixes the argument type and introduces a new wrapper called decrRefCountVoid() that can be used when the void* argument is needed. --- src/aof.c | 2 +- src/networking.c | 6 +++--- src/object.c | 11 ++++++++--- src/redis.h | 3 ++- src/t_list.c | 2 +- 5 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/aof.c b/src/aof.c index 8dc1dd1881f..2265222cc6d 100644 --- a/src/aof.c +++ b/src/aof.c @@ -456,7 +456,7 @@ struct redisClient *createFakeClient(void) { c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); - listSetFreeMethod(c->reply,decrRefCount); + listSetFreeMethod(c->reply,decrRefCountVoid); listSetDupMethod(c->reply,dupClientReplyValue); initClientMultiState(c); return c; diff --git a/src/networking.c b/src/networking.c index 6e5d3248986..0853ad92300 100644 --- a/src/networking.c +++ b/src/networking.c @@ -89,17 +89,17 @@ redisClient *createClient(int fd) { c->reply = listCreate(); c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; - listSetFreeMethod(c->reply,decrRefCount); + listSetFreeMethod(c->reply,decrRefCountVoid); listSetDupMethod(c->reply,dupClientReplyValue); c->bpop.keys = dictCreate(&setDictType,NULL); c->bpop.timeout = 0; c->bpop.target = NULL; c->io_keys = listCreate(); c->watched_keys = listCreate(); - listSetFreeMethod(c->io_keys,decrRefCount); + listSetFreeMethod(c->io_keys,decrRefCountVoid); c->pubsub_channels = dictCreate(&setDictType,NULL); c->pubsub_patterns = listCreate(); - listSetFreeMethod(c->pubsub_patterns,decrRefCount); + listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); if (fd != -1) listAddNodeTail(server.clients,c); initClientMultiState(c); diff --git a/src/object.c b/src/object.c index 00cf023b07f..bb5f6729d80 100644 --- a/src/object.c +++ b/src/object.c @@ -215,9 +215,7 @@ void incrRefCount(robj *o) { o->refcount++; } -void decrRefCount(void *obj) { - robj *o = obj; - +void decrRefCount(robj *o) { if (o->refcount <= 0) redisPanic("decrRefCount against refcount <= 0"); if (o->refcount == 1) { switch(o->type) { @@ -234,6 +232,13 @@ void decrRefCount(void *obj) { } } +/* This variant of decrRefCount() gets its argument as void, and is useful + * as free method in data structures that expect a 'void free_object(void*)' + * prototype for the free method. */ +void decrRefCountVoid(void *o) { + decrRefCount(o); +} + /* This function set the ref count to zero without freeing the object. * It is useful in order to pass a new object to functions incrementing * the ref count of the received object. Example: diff --git a/src/redis.h b/src/redis.h index 601725a3351..f645fddcd95 100644 --- a/src/redis.h +++ b/src/redis.h @@ -854,7 +854,8 @@ void discardTransaction(redisClient *c); void flagTransaction(redisClient *c); /* Redis object implementation */ -void decrRefCount(void *o); +void decrRefCount(robj *o); +void decrRefCountVoid(void *o); void incrRefCount(robj *o); robj *resetRefCount(robj *obj); void freeStringObject(robj *o); diff --git a/src/t_list.c b/src/t_list.c index 16b5e1be522..d3d3f6c69f5 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -275,7 +275,7 @@ void listTypeConvert(robj *subject, int enc) { if (enc == REDIS_ENCODING_LINKEDLIST) { list *l = listCreate(); - listSetFreeMethod(l,decrRefCount); + listSetFreeMethod(l,decrRefCountVoid); /* listTypeGet returns a robj with incremented refcount */ li = listTypeInitIterator(subject,0,REDIS_TAIL); From 4ff138fbbe519e5e18ac8dac8c19177d7a12e7a1 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Jan 2013 16:20:53 +0100 Subject: [PATCH 0457/1752] Keyspace events added for more commands. --- src/bitops.c | 3 ++ src/db.c | 2 ++ src/sort.c | 4 +-- src/t_hash.c | 10 +++++- src/t_list.c | 46 +++++++++++++++++++++------ src/t_set.c | 31 +++++++++++++++--- src/t_string.c | 4 ++- src/t_zset.c | 85 ++++++++++++++++++++++++++++++-------------------- 8 files changed, 134 insertions(+), 51 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 75f3317a93e..cda3adc7df5 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -153,6 +153,7 @@ void setbitCommand(redisClient *c) { byteval |= ((on & 0x1) << bit); ((uint8_t*)o->ptr)[byte] = byteval; signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("setbit",c->argv[1],c->db->id); server.dirty++; addReply(c, bitval ? shared.cone : shared.czero); } @@ -346,9 +347,11 @@ void bitopCommand(redisClient *c) { if (maxlen) { o = createObject(REDIS_STRING,res); setKey(c->db,targetkey,o); + notifyKeyspaceEvent("set",targetkey,c->db->id); decrRefCount(o); } else if (dbDelete(c->db,targetkey)) { signalModifiedKey(c->db,targetkey); + notifyKeyspaceEvent("del",targetkey,c->db->id); } server.dirty++; addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ diff --git a/src/db.c b/src/db.c index 8e6232d144e..4d838b05151 100644 --- a/src/db.c +++ b/src/db.c @@ -576,12 +576,14 @@ void expireGenericCommand(redisClient *c, long long basetime, int unit) { rewriteClientCommandVector(c,2,aux,key); decrRefCount(aux); signalModifiedKey(c->db,key); + notifyKeyspaceEvent("del",key,c->db->id); addReply(c, shared.cone); return; } else { setExpire(c->db,key,when); addReply(c,shared.cone); signalModifiedKey(c->db,key); + notifyKeyspaceEvent("expire",key,c->db->id); server.dirty++; return; } diff --git a/src/sort.c b/src/sort.c index cd54072f3e9..a0971473d81 100644 --- a/src/sort.c +++ b/src/sort.c @@ -504,9 +504,11 @@ void sortCommand(redisClient *c) { } if (outputlen) { setKey(c->db,storekey,sobj); + notifyKeyspaceEvent("sortstore",storekey,c->db->id); server.dirty += outputlen; } else if (dbDelete(c->db,storekey)) { signalModifiedKey(c->db,storekey); + notifyKeyspaceEvent("del",storekey,c->db->id); server.dirty++; } decrRefCount(sobj); @@ -525,5 +527,3 @@ void sortCommand(redisClient *c) { } zfree(vector); } - - diff --git a/src/t_hash.c b/src/t_hash.c index 414fb1bf839..e395ee265cb 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -474,6 +474,7 @@ void hsetCommand(redisClient *c) { update = hashTypeSet(o,c->argv[2],c->argv[3]); addReply(c, update ? shared.czero : shared.cone); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("hset",c->argv[1],c->db->id); server.dirty++; } @@ -489,6 +490,7 @@ void hsetnxCommand(redisClient *c) { hashTypeSet(o,c->argv[2],c->argv[3]); addReply(c, shared.cone); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("hset",c->argv[1],c->db->id); server.dirty++; } } @@ -510,6 +512,7 @@ void hmsetCommand(redisClient *c) { } addReply(c, shared.ok); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("hset",c->argv[1],c->db->id); server.dirty++; } @@ -543,6 +546,7 @@ void hincrbyCommand(redisClient *c) { decrRefCount(new); addReplyLongLong(c,value); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("hincrby",c->argv[1],c->db->id); server.dirty++; } @@ -569,6 +573,7 @@ void hincrbyfloatCommand(redisClient *c) { hashTypeSet(o,c->argv[2],new); addReplyBulk(c,new); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("hincrbyfloat",c->argv[1],c->db->id); server.dirty++; /* Always replicate HINCRBYFLOAT as an HSET command with the final value @@ -649,7 +654,7 @@ void hmgetCommand(redisClient *c) { void hdelCommand(redisClient *c) { robj *o; - int j, deleted = 0; + int j, deleted = 0, keyremoved = 0; if ((o = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || checkType(c,o,REDIS_HASH)) return; @@ -659,12 +664,15 @@ void hdelCommand(redisClient *c) { deleted++; if (hashTypeLength(o) == 0) { dbDelete(c->db,c->argv[1]); + keyremoved = 1; break; } } } if (deleted) { signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("hdel",c->argv[1],c->db->id); + if (keyremoved) notifyKeyspaceEvent("del",c->argv[1],c->db->id); server.dirty += deleted; } addReplyLongLong(c,deleted); diff --git a/src/t_list.c b/src/t_list.c index d3d3f6c69f5..ea5f4718dc1 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -316,7 +316,12 @@ void pushGenericCommand(redisClient *c, int where) { pushed++; } addReplyLongLong(c, waiting + (lobj ? listTypeLength(lobj) : 0)); - if (pushed) signalModifiedKey(c->db,c->argv[1]); + if (pushed) { + char *event = (where == REDIS_HEAD) ? "lpush" : "rpush"; + + signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent(event,c->argv[1],c->db->id); + } server.dirty += pushed; } @@ -338,10 +343,6 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { checkType(c,subject,REDIS_LIST)) return; if (refval != NULL) { - /* Note: we expect refval to be string-encoded because it is *not* the - * last argument of the multi-bulk LINSERT. */ - redisAssertWithInfo(c,refval,refval->encoding == REDIS_ENCODING_RAW); - /* We're not sure if this value can be inserted yet, but we cannot * convert the list inside the iterator. We don't want to loop over * the list twice (once to see if the value can be inserted and once @@ -366,6 +367,7 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { ziplistLen(subject->ptr) > server.list_max_ziplist_entries) listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("linsert",c->argv[1],c->db->id); server.dirty++; } else { /* Notify client of a failed insert */ @@ -373,8 +375,11 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { return; } } else { + char *event = (where == REDIS_HEAD) ? "lpush" : "rpush"; + listTypePush(subject,val,where); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent(event,c->argv[1],c->db->id); server.dirty++; } @@ -469,6 +474,7 @@ void lsetCommand(redisClient *c) { decrRefCount(value); addReply(c,shared.ok); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("lset",c->argv[1],c->db->id); server.dirty++; } } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { @@ -481,6 +487,7 @@ void lsetCommand(redisClient *c) { incrRefCount(value); addReply(c,shared.ok); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("lset",c->argv[1],c->db->id); server.dirty++; } } else { @@ -496,9 +503,15 @@ void popGenericCommand(redisClient *c, int where) { if (value == NULL) { addReply(c,shared.nullbulk); } else { + char *event = (where == REDIS_HEAD) ? "lpop" : "rpop"; + addReplyBulk(c,value); decrRefCount(value); - if (listTypeLength(o) == 0) dbDelete(c->db,c->argv[1]); + notifyKeyspaceEvent(event,c->argv[1],c->db->id); + if (listTypeLength(o) == 0) { + notifyKeyspaceEvent("del",c->argv[1],c->db->id); + dbDelete(c->db,c->argv[1]); + } signalModifiedKey(c->db,c->argv[1]); server.dirty++; } @@ -618,7 +631,12 @@ void ltrimCommand(redisClient *c) { } else { redisPanic("Unknown list encoding"); } - if (listTypeLength(o) == 0) dbDelete(c->db,c->argv[1]); + + notifyKeyspaceEvent("ltrim",c->argv[1],c->db->id); + if (listTypeLength(o) == 0) { + dbDelete(c->db,c->argv[1]); + notifyKeyspaceEvent("del",c->argv[1],c->db->id); + } signalModifiedKey(c->db,c->argv[1]); server.dirty++; addReply(c,shared.ok); @@ -693,6 +711,7 @@ void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value } signalModifiedKey(c->db,dstkey); listTypePush(dstobj,value,REDIS_HEAD); + notifyKeyspaceEvent("lpush",dstkey,c->db->id); /* Always send the pushed value to the client. */ addReplyBulk(c,value); } @@ -722,7 +741,11 @@ void rpoplpushCommand(redisClient *c) { decrRefCount(value); /* Delete the source list when it is empty */ - if (listTypeLength(sobj) == 0) dbDelete(c->db,touchedkey); + notifyKeyspaceEvent("rpop",touchedkey,c->db->id); + if (listTypeLength(sobj) == 0) { + dbDelete(c->db,touchedkey); + notifyKeyspaceEvent("del",touchedkey,c->db->id); + } signalModifiedKey(c->db,touchedkey); decrRefCount(touchedkey); server.dirty++; @@ -1046,6 +1069,7 @@ void blockingPopGenericCommand(redisClient *c, int where) { } else { if (listTypeLength(o) != 0) { /* Non empty list, this is like a non normal [LR]POP. */ + char *event = (where == REDIS_HEAD) ? "lpop" : "rpop"; robj *value = listTypePop(o,where); redisAssert(value != NULL); @@ -1053,7 +1077,11 @@ void blockingPopGenericCommand(redisClient *c, int where) { addReplyBulk(c,c->argv[j]); addReplyBulk(c,value); decrRefCount(value); - if (listTypeLength(o) == 0) dbDelete(c->db,c->argv[j]); + notifyKeyspaceEvent(event,c->argv[j],c->db->id); + if (listTypeLength(o) == 0) { + dbDelete(c->db,c->argv[j]); + notifyKeyspaceEvent("del",c->argv[j],c->db->id); + } signalModifiedKey(c->db,c->argv[j]); server.dirty++; diff --git a/src/t_set.c b/src/t_set.c index 0909d03010a..70052a55e55 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -266,14 +266,17 @@ void saddCommand(redisClient *c) { c->argv[j] = tryObjectEncoding(c->argv[j]); if (setTypeAdd(set,c->argv[j])) added++; } - if (added) signalModifiedKey(c->db,c->argv[1]); + if (added) { + signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("sadd",c->argv[1],c->db->id); + } server.dirty += added; addReplyLongLong(c,added); } void sremCommand(redisClient *c) { robj *set; - int j, deleted = 0; + int j, deleted = 0, keyremoved = 0; if ((set = lookupKeyWriteOrReply(c,c->argv[1],shared.czero)) == NULL || checkType(c,set,REDIS_SET)) return; @@ -283,12 +286,15 @@ void sremCommand(redisClient *c) { deleted++; if (setTypeSize(set) == 0) { dbDelete(c->db,c->argv[1]); + keyremoved = 1; break; } } } if (deleted) { signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("srem",c->argv[1],c->db->id); + if (keyremoved) notifyKeyspaceEvent("del",c->argv[1],c->db->id); server.dirty += deleted; } addReplyLongLong(c,deleted); @@ -322,9 +328,13 @@ void smoveCommand(redisClient *c) { addReply(c,shared.czero); return; } + notifyKeyspaceEvent("srem",c->argv[1],c->db->id); /* Remove the src set from the database when empty */ - if (setTypeSize(srcset) == 0) dbDelete(c->db,c->argv[1]); + if (setTypeSize(srcset) == 0) { + dbDelete(c->db,c->argv[1]); + notifyKeyspaceEvent("del",c->argv[1],c->db->id); + } signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[2]); server.dirty++; @@ -336,7 +346,10 @@ void smoveCommand(redisClient *c) { } /* An extra key has changed when ele was successfully added to dstset */ - if (setTypeAdd(dstset,ele)) server.dirty++; + if (setTypeAdd(dstset,ele)) { + server.dirty++; + notifyKeyspaceEvent("sadd",c->argv[2],c->db->id); + } addReply(c,shared.cone); } @@ -378,6 +391,7 @@ void spopCommand(redisClient *c) { incrRefCount(ele); setTypeRemove(set,ele); } + notifyKeyspaceEvent("spop",c->argv[1],c->db->id); /* Replicate/AOF this command as an SREM operation */ aux = createStringObject("SREM",4); @@ -386,7 +400,10 @@ void spopCommand(redisClient *c) { decrRefCount(aux); addReplyBulk(c,ele); - if (setTypeSize(set) == 0) dbDelete(c->db,c->argv[1]); + if (setTypeSize(set) == 0) { + dbDelete(c->db,c->argv[1]); + notifyKeyspaceEvent("del",c->argv[1],c->db->id); + } signalModifiedKey(c->db,c->argv[1]); server.dirty++; } @@ -691,6 +708,7 @@ void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, if (setTypeSize(dstset) > 0) { dbAdd(c->db,dstkey,dstset); addReplyLongLong(c,setTypeSize(dstset)); + notifyKeyspaceEvent("sinterstore",dstkey,c->db->id); } else { decrRefCount(dstset); addReply(c,shared.czero); @@ -856,6 +874,9 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * if (setTypeSize(dstset) > 0) { dbAdd(c->db,dstkey,dstset); addReplyLongLong(c,setTypeSize(dstset)); + notifyKeyspaceEvent( + op == REDIS_OP_UNION ? "sunionstore" : "sdiffstore", + dstkey,c->db->id); } else { decrRefCount(dstset); addReply(c,shared.czero); diff --git a/src/t_string.c b/src/t_string.c index a3e58e5b9ee..ed7840467f2 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -110,6 +110,7 @@ void getsetCommand(redisClient *c) { if (getGenericCommand(c) == REDIS_ERR) return; c->argv[2] = tryObjectEncoding(c->argv[2]); setKey(c->db,c->argv[1],c->argv[2]); + notifyKeyspaceEvent("set",c->argv[1],c->db->id); server.dirty++; } @@ -171,6 +172,7 @@ void setrangeCommand(redisClient *c) { o->ptr = sdsgrowzero(o->ptr,offset+sdslen(value)); memcpy((char*)o->ptr+offset,value,sdslen(value)); signalModifiedKey(c->db,c->argv[1]); + notifyKeyspaceEvent("setrange",c->argv[1],c->db->id); server.dirty++; } addReplyLongLong(c,sdslen(o->ptr)); @@ -255,7 +257,7 @@ void msetGenericCommand(redisClient *c, int nx) { for (j = 1; j < c->argc; j += 2) { c->argv[j+1] = tryObjectEncoding(c->argv[j+1]); setKey(c->db,c->argv[j],c->argv[j+1]); - notifyKeyspaceEvent("set",c->argv[j+1],c->db->id); + notifyKeyspaceEvent("set",c->argv[j],c->db->id); } server.dirty += (c->argc-1)/2; addReply(c, nx ? shared.cone : shared.ok); diff --git a/src/t_zset.c b/src/t_zset.c index 0d92f681f92..4a9383c1298 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -844,9 +844,9 @@ void zaddGenericCommand(redisClient *c, int incr) { robj *ele; robj *zobj; robj *curobj; - double score = 0, *scores, curscore = 0.0; + double score = 0, *scores = NULL, curscore = 0.0; int j, elements = (c->argc-2)/2; - int added = 0; + int added = 0, updated = 0; if (c->argc % 2) { addReply(c,shared.syntaxerr); @@ -859,11 +859,7 @@ void zaddGenericCommand(redisClient *c, int incr) { scores = zmalloc(sizeof(double)*elements); for (j = 0; j < elements; j++) { if (getDoubleFromObjectOrReply(c,c->argv[2+j*2],&scores[j],NULL) - != REDIS_OK) - { - zfree(scores); - return; - } + != REDIS_OK) goto cleanup; } /* Lookup the key and create the sorted set if does not exist. */ @@ -880,8 +876,7 @@ void zaddGenericCommand(redisClient *c, int incr) { } else { if (zobj->type != REDIS_ZSET) { addReply(c,shared.wrongtypeerr); - zfree(scores); - return; + goto cleanup; } } @@ -898,10 +893,7 @@ void zaddGenericCommand(redisClient *c, int incr) { score += curscore; if (isnan(score)) { addReplyError(c,nanerr); - /* Don't need to check if the sorted set is empty - * because we know it has at least one element. */ - zfree(scores); - return; + goto cleanup; } } @@ -909,9 +901,8 @@ void zaddGenericCommand(redisClient *c, int incr) { if (score != curscore) { zobj->ptr = zzlDelete(zobj->ptr,eptr); zobj->ptr = zzlInsert(zobj->ptr,ele,score); - - signalModifiedKey(c->db,key); server.dirty++; + updated++; } } else { /* Optimize: check if the element is too large or the list @@ -921,10 +912,8 @@ void zaddGenericCommand(redisClient *c, int incr) { zsetConvert(zobj,REDIS_ENCODING_SKIPLIST); if (sdslen(ele->ptr) > server.zset_max_ziplist_value) zsetConvert(zobj,REDIS_ENCODING_SKIPLIST); - - signalModifiedKey(c->db,key); server.dirty++; - if (!incr) added++; + added++; } } else if (zobj->encoding == REDIS_ENCODING_SKIPLIST) { zset *zs = zobj->ptr; @@ -943,8 +932,7 @@ void zaddGenericCommand(redisClient *c, int incr) { addReplyError(c,nanerr); /* Don't need to check if the sorted set is empty * because we know it has at least one element. */ - zfree(scores); - return; + goto cleanup; } } @@ -956,29 +944,32 @@ void zaddGenericCommand(redisClient *c, int incr) { znode = zslInsert(zs->zsl,score,curobj); incrRefCount(curobj); /* Re-inserted in skiplist. */ dictGetVal(de) = &znode->score; /* Update score ptr. */ - - signalModifiedKey(c->db,key); server.dirty++; + updated++; } } else { znode = zslInsert(zs->zsl,score,ele); incrRefCount(ele); /* Inserted in skiplist. */ redisAssertWithInfo(c,NULL,dictAdd(zs->dict,ele,&znode->score) == DICT_OK); incrRefCount(ele); /* Added to dictionary. */ - - signalModifiedKey(c->db,key); server.dirty++; - if (!incr) added++; + added++; } } else { redisPanic("Unknown sorted set encoding"); } } - zfree(scores); if (incr) /* ZINCRBY */ addReplyDouble(c,score); else /* ZADD */ addReplyLongLong(c,added); + +cleanup: + zfree(scores); + if (added || updated) { + signalModifiedKey(c->db,key); + notifyKeyspaceEvent(incr ? "zincr" : "zadd", key, c->db->id); + } } void zaddCommand(redisClient *c) { @@ -992,7 +983,7 @@ void zincrbyCommand(redisClient *c) { void zremCommand(redisClient *c) { robj *key = c->argv[1]; robj *zobj; - int deleted = 0, j; + int deleted = 0, keyremoved = 0, j; if ((zobj = lookupKeyWriteOrReply(c,key,shared.czero)) == NULL || checkType(c,zobj,REDIS_ZSET)) return; @@ -1038,6 +1029,8 @@ void zremCommand(redisClient *c) { } if (deleted) { + notifyKeyspaceEvent("zrem",key,c->db->id); + if (keyremoved) notifyKeyspaceEvent("del",key,c->db->id); signalModifiedKey(c->db,key); server.dirty += deleted; } @@ -1048,6 +1041,7 @@ void zremrangebyscoreCommand(redisClient *c) { robj *key = c->argv[1]; robj *zobj; zrangespec range; + int keyremoved = 0; unsigned long deleted; /* Parse the range arguments. */ @@ -1061,17 +1055,27 @@ void zremrangebyscoreCommand(redisClient *c) { if (zobj->encoding == REDIS_ENCODING_ZIPLIST) { zobj->ptr = zzlDeleteRangeByScore(zobj->ptr,range,&deleted); - if (zzlLength(zobj->ptr) == 0) dbDelete(c->db,key); + if (zzlLength(zobj->ptr) == 0) { + dbDelete(c->db,key); + keyremoved = 1; + } } else if (zobj->encoding == REDIS_ENCODING_SKIPLIST) { zset *zs = zobj->ptr; deleted = zslDeleteRangeByScore(zs->zsl,range,zs->dict); if (htNeedsResize(zs->dict)) dictResize(zs->dict); - if (dictSize(zs->dict) == 0) dbDelete(c->db,key); + if (dictSize(zs->dict) == 0) { + dbDelete(c->db,key); + keyremoved = 1; + } } else { redisPanic("Unknown sorted set encoding"); } - if (deleted) signalModifiedKey(c->db,key); + if (deleted) { + signalModifiedKey(c->db,key); + notifyKeyspaceEvent("zrembyscore",key,c->db->id); + if (keyremoved) notifyKeyspaceEvent("del",key,c->db->id); + } server.dirty += deleted; addReplyLongLong(c,deleted); } @@ -1083,6 +1087,7 @@ void zremrangebyrankCommand(redisClient *c) { long end; int llen; unsigned long deleted; + int keyremoved = 0; if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != REDIS_OK) || (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != REDIS_OK)) return; @@ -1107,19 +1112,29 @@ void zremrangebyrankCommand(redisClient *c) { if (zobj->encoding == REDIS_ENCODING_ZIPLIST) { /* Correct for 1-based rank. */ zobj->ptr = zzlDeleteRangeByRank(zobj->ptr,start+1,end+1,&deleted); - if (zzlLength(zobj->ptr) == 0) dbDelete(c->db,key); + if (zzlLength(zobj->ptr) == 0) { + dbDelete(c->db,key); + keyremoved = 1; + } } else if (zobj->encoding == REDIS_ENCODING_SKIPLIST) { zset *zs = zobj->ptr; /* Correct for 1-based rank. */ deleted = zslDeleteRangeByRank(zs->zsl,start+1,end+1,zs->dict); if (htNeedsResize(zs->dict)) dictResize(zs->dict); - if (dictSize(zs->dict) == 0) dbDelete(c->db,key); + if (dictSize(zs->dict) == 0) { + dbDelete(c->db,key); + keyremoved = 1; + } } else { redisPanic("Unknown sorted set encoding"); } - if (deleted) signalModifiedKey(c->db,key); + if (deleted) { + signalModifiedKey(c->db,key); + notifyKeyspaceEvent("zrembyrank",key,c->db->id); + if (keyremoved) notifyKeyspaceEvent("del",key,c->db->id); + } server.dirty += deleted; addReplyLongLong(c,deleted); } @@ -1661,6 +1676,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { if (dbDelete(c->db,dstkey)) { signalModifiedKey(c->db,dstkey); + notifyKeyspaceEvent("del",dstkey,c->db->id); touched = 1; server.dirty++; } @@ -1673,6 +1689,9 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { dbAdd(c->db,dstkey,dstobj); addReplyLongLong(c,zsetLength(dstobj)); if (!touched) signalModifiedKey(c->db,dstkey); + notifyKeyspaceEvent( + (op == REDIS_OP_UNION) ? "zunionstore" : "zinterstore", + dstkey,c->db->id); server.dirty++; } else { decrRefCount(dstobj); From 81af92b02315baa35d032e4e11f28e770c9d18e3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 24 Jan 2013 16:27:48 +0100 Subject: [PATCH 0458/1752] Enable keyspace events notification when testing. --- tests/assets/default.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/assets/default.conf b/tests/assets/default.conf index 976852e9153..f264bed3c44 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -12,6 +12,9 @@ # # units are case insensitive so 1GB 1Gb 1gB are all the same. +# Enable keyspace events notification for testing so that we cover more code. +notify-keyspace-events yes + # By default Redis does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. daemonize no From 6de052cd774989864012154f045ac56df6aa4418 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Jan 2013 11:16:00 +0100 Subject: [PATCH 0459/1752] decrRefCount -> decrRefCountVoid in list constructor. --- src/object.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index bb5f6729d80..2554656a332 100644 --- a/src/object.c +++ b/src/object.c @@ -97,7 +97,7 @@ robj *dupStringObject(robj *o) { robj *createListObject(void) { list *l = listCreate(); robj *o = createObject(REDIS_LIST,l); - listSetFreeMethod(l,decrRefCount); + listSetFreeMethod(l,decrRefCountVoid); o->encoding = REDIS_ENCODING_LINKEDLIST; return o; } From b9bc4f9132e92b54cb0521511a7859dc34ad5cf9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Jan 2013 13:19:08 +0100 Subject: [PATCH 0460/1752] Keyspace events: it is now possible to select subclasses of events. When keyspace events are enabled, the overhead is not sever but noticeable, so this commit introduces the ability to select subclasses of events in order to avoid to generate events the user is not interested in. The events can be selected using redis.conf or CONFIG SET / GET. --- redis.conf | 37 ++++++++++-- src/bitops.c | 6 +- src/config.c | 24 ++++++-- src/db.c | 13 +++-- src/notify.c | 117 ++++++++++++++++++++++++++------------ src/redis.c | 6 +- src/redis.h | 23 +++++++- src/sort.c | 5 +- src/t_hash.c | 16 +++--- src/t_list.c | 33 ++++++----- src/t_set.c | 23 ++++---- src/t_string.c | 18 +++--- src/t_zset.c | 22 ++++--- tests/assets/default.conf | 2 +- 14 files changed, 237 insertions(+), 108 deletions(-) diff --git a/redis.conf b/redis.conf index d7bdcfd53d4..a6932cdf391 100644 --- a/redis.conf +++ b/redis.conf @@ -457,10 +457,39 @@ slowlog-max-len 128 # PUBLISH __keyspace@0__:foo del # PUBLISH __keyevent@0__:del foo # -# While the overhead of this feature is relatively small most users don't -# need it so it is disabled by default. You can enable it setting the -# following configuration option to yes. -notify-keyspace-events no +# It is possible to select the events that Redis will notify among a set +# of classes. Every class is identified by a single character: +# +# K Keyspace events, published with __keyspace@__ prefix. +# E Keyevent events, published with __keyevent@__ prefix. +# g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... +# $ String commands +# l List commands +# s Set commands +# h Hash commands +# z Sorted set commands +# x Expired events (events generated every time a key expires) +# e Evicted events (events generated when a key is evicted for maxmemory) +# A Alias for g$lshzxe, so that the "AKE" string means all the events. +# +# The "notify-keyspace-events" takes as argument a string that is composed +# by zero or multiple characters. The empty string means that notifications +# are disabled at all. +# +# Example: to enable list and generic events, from the point of view of the +# event name, use: +# +# notify-keyspace-events Elg +# +# Example 2: to get the stream of the expired keys subscribing to channel +# name __keyevent@0__:expired use: +# +# notify-keyspace-events Ex +# +# By default all notifications are disabled because most users don't need +# this feature and the feature has some overhead. Note that if you don't +# specify at least one of K or E, no events will be delivered. +notify-keyspace-events "" ############################### ADVANCED CONFIG ############################### diff --git a/src/bitops.c b/src/bitops.c index cda3adc7df5..f2d03f9b0f1 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -153,7 +153,7 @@ void setbitCommand(redisClient *c) { byteval |= ((on & 0x1) << bit); ((uint8_t*)o->ptr)[byte] = byteval; signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("setbit",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"setbit",c->argv[1],c->db->id); server.dirty++; addReply(c, bitval ? shared.cone : shared.czero); } @@ -347,11 +347,11 @@ void bitopCommand(redisClient *c) { if (maxlen) { o = createObject(REDIS_STRING,res); setKey(c->db,targetkey,o); - notifyKeyspaceEvent("set",targetkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"set",targetkey,c->db->id); decrRefCount(o); } else if (dbDelete(c->db,targetkey)) { signalModifiedKey(c->db,targetkey); - notifyKeyspaceEvent("del",targetkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",targetkey,c->db->id); } server.dirty++; addReplyLongLong(c,maxlen); /* Return the output string length in bytes. */ diff --git a/src/config.c b/src/config.c index bfe73482fc7..4ab19c1515a 100644 --- a/src/config.c +++ b/src/config.c @@ -385,9 +385,13 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { server.slave_priority = atoi(argv[1]); } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { - if ((server.notify_keyspace_events = yesnotoi(argv[1])) == -1) { - err = "argument must be 'yes' or 'no'"; goto loaderr; + int flags = keyspaceEventsStringToFlags(argv[1]); + + if (flags == -1) { + err = "Invalid event class character. Use 'g$lshzxeA'."; + goto loaderr; } + server.notify_keyspace_events = flags; } else if (!strcasecmp(argv[0],"sentinel")) { /* argc == 1 is handled by main() as we need to enter the sentinel * mode ASAP. */ @@ -707,10 +711,10 @@ void configSetCommand(redisClient *c) { if (yn == -1) goto badfmt; server.rdb_compression = yn; } else if (!strcasecmp(c->argv[2]->ptr,"notify-keyspace-events")) { - int yn = yesnotoi(o->ptr); + int flags = keyspaceEventsStringToFlags(o->ptr); - if (yn == -1) goto badfmt; - server.notify_keyspace_events = yn; + if (flags == -1) goto badfmt; + server.notify_keyspace_events = flags; } else if (!strcasecmp(c->argv[2]->ptr,"slave-priority")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; @@ -820,7 +824,6 @@ void configGetCommand(redisClient *c) { config_get_bool_field("rdbcompression", server.rdb_compression); config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); - config_get_bool_field("notify-keyspace-events", server.notify_keyspace_events); /* Everything we can't handle with macros follows. */ @@ -935,6 +938,15 @@ void configGetCommand(redisClient *c) { addReplyBulkCString(c,buf); matches++; } + if (stringmatch(pattern,"notify-keyspace-events",0)) { + robj *flagsobj = createObject(REDIS_STRING, + keyspaceEventsFlagsToString(server.notify_keyspace_events)); + + addReplyBulkCString(c,"notify-keyspace-events"); + addReplyBulk(c,flagsobj); + decrRefCount(flagsobj); + matches++; + } setDeferredMultiBulkLength(c,replylen,matches*2); } diff --git a/src/db.c b/src/db.c index 4d838b05151..f460f14634b 100644 --- a/src/db.c +++ b/src/db.c @@ -238,7 +238,8 @@ void delCommand(redisClient *c) { for (j = 1; j < c->argc; j++) { if (dbDelete(c->db,c->argv[j])) { signalModifiedKey(c->db,c->argv[j]); - notifyKeyspaceEvent("del",c->argv[j],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC, + "del",c->argv[j],c->db->id); server.dirty++; deleted++; } @@ -386,8 +387,10 @@ void renameGenericCommand(redisClient *c, int nx) { dbDelete(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[2]); - notifyKeyspaceEvent("rename_from",c->argv[1],c->db->id); - notifyKeyspaceEvent("rename_to",c->argv[2],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"rename_from", + c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"rename_to", + c->argv[2],c->db->id); server.dirty++; addReply(c,nx ? shared.cone : shared.ok); } @@ -576,14 +579,14 @@ void expireGenericCommand(redisClient *c, long long basetime, int unit) { rewriteClientCommandVector(c,2,aux,key); decrRefCount(aux); signalModifiedKey(c->db,key); - notifyKeyspaceEvent("del",key,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",key,c->db->id); addReply(c, shared.cone); return; } else { setExpire(c->db,key,when); addReply(c,shared.cone); signalModifiedKey(c->db,key); - notifyKeyspaceEvent("expire",key,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"expire",key,c->db->id); server.dirty++; return; } diff --git a/src/notify.c b/src/notify.c index da1d4d8943d..f8e01829503 100644 --- a/src/notify.c +++ b/src/notify.c @@ -30,50 +30,97 @@ #include "redis.h" /* This file implements keyspace events notification via Pub/Sub ad - * described at http://redis.io/topics/keyspace-events. + * described at http://redis.io/topics/keyspace-events. */ + +/* Turn a string representing notification classes into an integer + * representing notification classes flags xored. * - * The API provided to the rest of the Redis core is a simple function: + * The function returns -1 if the input contains characters not mapping to + * any class. */ +int keyspaceEventsStringToFlags(char *classes) { + char *p = classes; + int c, flags = 0; + + while((c = *p++) != '\0') { + switch(c) { + case 'A': flags |= REDIS_NOTIFY_ALL; break; + case 'g': flags |= REDIS_NOTIFY_GENERIC; break; + case '$': flags |= REDIS_NOTIFY_STRING; break; + case 'l': flags |= REDIS_NOTIFY_LIST; break; + case 's': flags |= REDIS_NOTIFY_SET; break; + case 'h': flags |= REDIS_NOTIFY_HASH; break; + case 'z': flags |= REDIS_NOTIFY_ZSET; break; + case 'x': flags |= REDIS_NOTIFY_EXPIRED; break; + case 'e': flags |= REDIS_NOTIFY_EVICTED; break; + case 'K': flags |= REDIS_NOTIFY_KEYSPACE; break; + case 'E': flags |= REDIS_NOTIFY_KEYEVENT; break; + default: return -1; + } + } + return flags; +} + +/* This function does exactly the revese of the function above: it gets + * as input an integer with the xored flags and returns a string representing + * the selected classes. The string returned is an sds string that needs to + * be released with sdsfree(). */ +sds keyspaceEventsFlagsToString(int flags) { + sds res; + + if ((flags & REDIS_NOTIFY_ALL) == REDIS_NOTIFY_ALL) + return sdsnew("A"); + res = sdsempty(); + if (flags & REDIS_NOTIFY_GENERIC) res = sdscatlen(res,"g",1); + if (flags & REDIS_NOTIFY_STRING) res = sdscatlen(res,"$",1); + if (flags & REDIS_NOTIFY_LIST) res = sdscatlen(res,"l",1); + if (flags & REDIS_NOTIFY_SET) res = sdscatlen(res,"s",1); + if (flags & REDIS_NOTIFY_HASH) res = sdscatlen(res,"h",1); + if (flags & REDIS_NOTIFY_ZSET) res = sdscatlen(res,"z",1); + if (flags & REDIS_NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); + if (flags & REDIS_NOTIFY_EVICTED) res = sdscatlen(res,"e",1); + if (flags & REDIS_NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); + if (flags & REDIS_NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); + return res; +} + +/* The API provided to the rest of the Redis core is a simple function: * * notifyKeyspaceEvent(char *event, robj *key, int dbid); * * 'event' is a C string representing the event name. * 'key' is a Redis object representing the key name. - * 'dbid' is the database ID where the key lives. - */ - -void notifyKeyspaceEvent(char *event, robj *key, int dbid) { - sds keyspace_chan, keyevent_chan; - int len; + * 'dbid' is the database ID where the key lives. */ +void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid) { + sds chan; + robj *chanobj; + int len = -1; char buf[24]; - robj *chan1, *chan2, *eventobj; - - if (!server.notify_keyspace_events) return; - - /* The prefix of the two channels is identical if not for - * 'keyspace' that is 'keyevent' in the event channel name, so - * we build a single prefix and overwrite 'event' with 'space'. */ - keyspace_chan = sdsnewlen("__keyspace@",11); - len = ll2string(buf,sizeof(buf),dbid); - keyspace_chan = sdscatlen(keyspace_chan, buf, len); - keyspace_chan = sdscatlen(keyspace_chan, "__:", 3); - keyevent_chan = sdsdup(keyspace_chan); /* Dup the prefix. */ - memcpy(keyevent_chan+5,"event",5); /* Fix it. */ - eventobj = createStringObject(event,strlen(event)); + /* If notifications for this class of events are off, return ASAP. */ + if (!(server.notify_keyspace_events & type)) return; - /* The keyspace channel name has a trailing key name, while - * the keyevent channel name has a trailing event name. */ - keyspace_chan = sdscatsds(keyspace_chan, key->ptr); - keyevent_chan = sdscatsds(keyevent_chan, eventobj->ptr); - chan1 = createObject(REDIS_STRING, keyspace_chan); - chan2 = createObject(REDIS_STRING, keyevent_chan); + /* __keyspace@__: notifications. */ + if (server.notify_keyspace_events & REDIS_NOTIFY_KEYSPACE) { + robj *eventobj; - /* Finally publish the two notifications. */ - pubsubPublishMessage(chan1, eventobj); - pubsubPublishMessage(chan2, key); + chan = sdsnewlen("__keyspace@",11); + len = ll2string(buf,sizeof(buf),dbid); + chan = sdscatlen(chan, buf, len); + chan = sdscatlen(chan, "__:", 3); + eventobj = createStringObject(event,strlen(event)); + chanobj = createObject(REDIS_STRING, chan); + pubsubPublishMessage(chanobj, eventobj); + decrRefCount(chanobj); + } - /* Release objects. */ - decrRefCount(eventobj); - decrRefCount(chan1); - decrRefCount(chan2); + /* __keyevente@__: notifications. */ + if (server.notify_keyspace_events & REDIS_NOTIFY_KEYEVENT) { + chan = sdsnewlen("__keyevent@",11); + if (len == -1) len = ll2string(buf,sizeof(buf),dbid); + chan = sdscatlen(chan, buf, len); + chan = sdscatlen(chan, "__:", 3); + chanobj = createObject(REDIS_STRING, chan); + pubsubPublishMessage(chanobj, key); + decrRefCount(chanobj); + } } diff --git a/src/redis.c b/src/redis.c index 7bbb65fc601..6786eeddc3a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -666,7 +666,8 @@ void activeExpireCycle(void) { propagateExpire(db,keyobj); dbDelete(db,keyobj); - notifyKeyspaceEvent("expired",keyobj,db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_EXPIRED, + "expired",keyobj,db->id); decrRefCount(keyobj); expired++; server.stat_expiredkeys++; @@ -2360,7 +2361,8 @@ int freeMemoryIfNeeded(void) { delta -= (long long) zmalloc_used_memory(); mem_freed += delta; server.stat_evictedkeys++; - notifyKeyspaceEvent("evicted",keyobj,db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_EVICTED, "evicted", + keyobj, db->id); decrRefCount(keyobj); keys_freed++; diff --git a/src/redis.h b/src/redis.h index f645fddcd95..635232f96cd 100644 --- a/src/redis.h +++ b/src/redis.h @@ -291,6 +291,20 @@ #define REDIS_PROPAGATE_AOF 1 #define REDIS_PROPAGATE_REPL 2 +/* Keyspace changes notification classes. Every class is associated with a + * character for configuration purposes. */ +#define REDIS_NOTIFY_KEYSPACE (1<<0) /* K */ +#define REDIS_NOTIFY_KEYEVENT (1<<1) /* E */ +#define REDIS_NOTIFY_GENERIC (1<<2) /* g */ +#define REDIS_NOTIFY_STRING (1<<3) /* $ */ +#define REDIS_NOTIFY_LIST (1<<4) /* l */ +#define REDIS_NOTIFY_SET (1<<5) /* s */ +#define REDIS_NOTIFY_HASH (1<<6) /* h */ +#define REDIS_NOTIFY_ZSET (1<<7) /* z */ +#define REDIS_NOTIFY_EXPIRED (1<<8) /* x */ +#define REDIS_NOTIFY_EVICTED (1<<9) /* e */ +#define REDIS_NOTIFY_ALL (REDIS_NOTIFY_GENERIC | REDIS_NOTIFY_STRING | REDIS_NOTIFY_LIST | REDIS_NOTIFY_SET | REDIS_NOTIFY_HASH | REDIS_NOTIFY_ZSET | REDIS_NOTIFY_EXPIRED | REDIS_NOTIFY_EVICTED) /* A */ + /* Using the following macro you can run code inside serverCron() with the * specified period, specified in milliseconds. * The actual resolution depends on server.hz. */ @@ -646,7 +660,8 @@ struct redisServer { /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ - int notify_keyspace_events; /* Propagate keyspace events via Pub/Sub. */ + int notify_keyspace_events; /* Events to propagate via Pub/Sub. This is an + xor of REDIS_NOTIFY... flags. */ /* Scripting */ lua_State *lua; /* The Lua interpreter. We use just one for all clients */ redisClient *lua_client; /* The "fake client" to query Redis from Lua */ @@ -1001,7 +1016,11 @@ int pubsubUnsubscribeAllPatterns(redisClient *c, int notify); void freePubsubPattern(void *p); int listMatchPubsubPattern(void *a, void *b); int pubsubPublishMessage(robj *channel, robj *message); -void notifyKeyspaceEvent(char *event, robj *key, int dbid); + +/* Keyspace events notification */ +void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid); +int keyspaceEventsStringToFlags(char *classes); +sds keyspaceEventsFlagsToString(int flags); /* Configuration */ void loadServerConfig(char *filename, char *options); diff --git a/src/sort.c b/src/sort.c index a0971473d81..4b5040250c6 100644 --- a/src/sort.c +++ b/src/sort.c @@ -504,11 +504,12 @@ void sortCommand(redisClient *c) { } if (outputlen) { setKey(c->db,storekey,sobj); - notifyKeyspaceEvent("sortstore",storekey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"sortstore",storekey, + c->db->id); server.dirty += outputlen; } else if (dbDelete(c->db,storekey)) { signalModifiedKey(c->db,storekey); - notifyKeyspaceEvent("del",storekey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",storekey,c->db->id); server.dirty++; } decrRefCount(sobj); diff --git a/src/t_hash.c b/src/t_hash.c index e395ee265cb..959c9ca81d3 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -474,7 +474,7 @@ void hsetCommand(redisClient *c) { update = hashTypeSet(o,c->argv[2],c->argv[3]); addReply(c, update ? shared.czero : shared.cone); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("hset",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty++; } @@ -490,7 +490,7 @@ void hsetnxCommand(redisClient *c) { hashTypeSet(o,c->argv[2],c->argv[3]); addReply(c, shared.cone); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("hset",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty++; } } @@ -512,7 +512,7 @@ void hmsetCommand(redisClient *c) { } addReply(c, shared.ok); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("hset",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_HASH,"hset",c->argv[1],c->db->id); server.dirty++; } @@ -546,7 +546,7 @@ void hincrbyCommand(redisClient *c) { decrRefCount(new); addReplyLongLong(c,value); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("hincrby",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_HASH,"hincrby",c->argv[1],c->db->id); server.dirty++; } @@ -573,7 +573,7 @@ void hincrbyfloatCommand(redisClient *c) { hashTypeSet(o,c->argv[2],new); addReplyBulk(c,new); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("hincrbyfloat",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_HASH,"hincrbyfloat",c->argv[1],c->db->id); server.dirty++; /* Always replicate HINCRBYFLOAT as an HSET command with the final value @@ -671,8 +671,10 @@ void hdelCommand(redisClient *c) { } if (deleted) { signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("hdel",c->argv[1],c->db->id); - if (keyremoved) notifyKeyspaceEvent("del",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_HASH,"hdel",c->argv[1],c->db->id); + if (keyremoved) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",c->argv[1], + c->db->id); server.dirty += deleted; } addReplyLongLong(c,deleted); diff --git a/src/t_list.c b/src/t_list.c index ea5f4718dc1..0413dc69b4f 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -320,7 +320,7 @@ void pushGenericCommand(redisClient *c, int where) { char *event = (where == REDIS_HEAD) ? "lpush" : "rpush"; signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent(event,c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,event,c->argv[1],c->db->id); } server.dirty += pushed; } @@ -367,7 +367,8 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { ziplistLen(subject->ptr) > server.list_max_ziplist_entries) listTypeConvert(subject,REDIS_ENCODING_LINKEDLIST); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("linsert",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"linsert", + c->argv[1],c->db->id); server.dirty++; } else { /* Notify client of a failed insert */ @@ -379,7 +380,7 @@ void pushxGenericCommand(redisClient *c, robj *refval, robj *val, int where) { listTypePush(subject,val,where); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent(event,c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,event,c->argv[1],c->db->id); server.dirty++; } @@ -474,7 +475,7 @@ void lsetCommand(redisClient *c) { decrRefCount(value); addReply(c,shared.ok); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("lset",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id); server.dirty++; } } else if (o->encoding == REDIS_ENCODING_LINKEDLIST) { @@ -487,7 +488,7 @@ void lsetCommand(redisClient *c) { incrRefCount(value); addReply(c,shared.ok); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("lset",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lset",c->argv[1],c->db->id); server.dirty++; } } else { @@ -507,9 +508,10 @@ void popGenericCommand(redisClient *c, int where) { addReplyBulk(c,value); decrRefCount(value); - notifyKeyspaceEvent(event,c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,event,c->argv[1],c->db->id); if (listTypeLength(o) == 0) { - notifyKeyspaceEvent("del",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del", + c->argv[1],c->db->id); dbDelete(c->db,c->argv[1]); } signalModifiedKey(c->db,c->argv[1]); @@ -632,10 +634,10 @@ void ltrimCommand(redisClient *c) { redisPanic("Unknown list encoding"); } - notifyKeyspaceEvent("ltrim",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"ltrim",c->argv[1],c->db->id); if (listTypeLength(o) == 0) { dbDelete(c->db,c->argv[1]); - notifyKeyspaceEvent("del",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } signalModifiedKey(c->db,c->argv[1]); server.dirty++; @@ -711,7 +713,7 @@ void rpoplpushHandlePush(redisClient *c, robj *dstkey, robj *dstobj, robj *value } signalModifiedKey(c->db,dstkey); listTypePush(dstobj,value,REDIS_HEAD); - notifyKeyspaceEvent("lpush",dstkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"lpush",dstkey,c->db->id); /* Always send the pushed value to the client. */ addReplyBulk(c,value); } @@ -741,10 +743,11 @@ void rpoplpushCommand(redisClient *c) { decrRefCount(value); /* Delete the source list when it is empty */ - notifyKeyspaceEvent("rpop",touchedkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,"rpop",touchedkey,c->db->id); if (listTypeLength(sobj) == 0) { dbDelete(c->db,touchedkey); - notifyKeyspaceEvent("del",touchedkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del", + touchedkey,c->db->id); } signalModifiedKey(c->db,touchedkey); decrRefCount(touchedkey); @@ -1077,10 +1080,12 @@ void blockingPopGenericCommand(redisClient *c, int where) { addReplyBulk(c,c->argv[j]); addReplyBulk(c,value); decrRefCount(value); - notifyKeyspaceEvent(event,c->argv[j],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_LIST,event, + c->argv[j],c->db->id); if (listTypeLength(o) == 0) { dbDelete(c->db,c->argv[j]); - notifyKeyspaceEvent("del",c->argv[j],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del", + c->argv[j],c->db->id); } signalModifiedKey(c->db,c->argv[j]); server.dirty++; diff --git a/src/t_set.c b/src/t_set.c index 70052a55e55..a22bbb67b95 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -268,7 +268,7 @@ void saddCommand(redisClient *c) { } if (added) { signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("sadd",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_SET,"sadd",c->argv[1],c->db->id); } server.dirty += added; addReplyLongLong(c,added); @@ -293,8 +293,10 @@ void sremCommand(redisClient *c) { } if (deleted) { signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("srem",c->argv[1],c->db->id); - if (keyremoved) notifyKeyspaceEvent("del",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_SET,"srem",c->argv[1],c->db->id); + if (keyremoved) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",c->argv[1], + c->db->id); server.dirty += deleted; } addReplyLongLong(c,deleted); @@ -328,12 +330,12 @@ void smoveCommand(redisClient *c) { addReply(c,shared.czero); return; } - notifyKeyspaceEvent("srem",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_SET,"srem",c->argv[1],c->db->id); /* Remove the src set from the database when empty */ if (setTypeSize(srcset) == 0) { dbDelete(c->db,c->argv[1]); - notifyKeyspaceEvent("del",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } signalModifiedKey(c->db,c->argv[1]); signalModifiedKey(c->db,c->argv[2]); @@ -348,7 +350,7 @@ void smoveCommand(redisClient *c) { /* An extra key has changed when ele was successfully added to dstset */ if (setTypeAdd(dstset,ele)) { server.dirty++; - notifyKeyspaceEvent("sadd",c->argv[2],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_SET,"sadd",c->argv[2],c->db->id); } addReply(c,shared.cone); } @@ -391,7 +393,7 @@ void spopCommand(redisClient *c) { incrRefCount(ele); setTypeRemove(set,ele); } - notifyKeyspaceEvent("spop",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_SET,"spop",c->argv[1],c->db->id); /* Replicate/AOF this command as an SREM operation */ aux = createStringObject("SREM",4); @@ -402,7 +404,7 @@ void spopCommand(redisClient *c) { addReplyBulk(c,ele); if (setTypeSize(set) == 0) { dbDelete(c->db,c->argv[1]); - notifyKeyspaceEvent("del",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",c->argv[1],c->db->id); } signalModifiedKey(c->db,c->argv[1]); server.dirty++; @@ -708,7 +710,8 @@ void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, if (setTypeSize(dstset) > 0) { dbAdd(c->db,dstkey,dstset); addReplyLongLong(c,setTypeSize(dstset)); - notifyKeyspaceEvent("sinterstore",dstkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_SET,"sinterstore", + dstkey,c->db->id); } else { decrRefCount(dstset); addReply(c,shared.czero); @@ -874,7 +877,7 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * if (setTypeSize(dstset) > 0) { dbAdd(c->db,dstkey,dstset); addReplyLongLong(c,setTypeSize(dstset)); - notifyKeyspaceEvent( + notifyKeyspaceEvent(REDIS_NOTIFY_SET, op == REDIS_OP_UNION ? "sunionstore" : "sdiffstore", dstkey,c->db->id); } else { diff --git a/src/t_string.c b/src/t_string.c index ed7840467f2..1d25e5ad131 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -62,8 +62,9 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir setKey(c->db,key,val); server.dirty++; if (expire) setExpire(c->db,key,mstime()+milliseconds); - notifyKeyspaceEvent("set",key,c->db->id); - if (expire) notifyKeyspaceEvent("expire",key,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"set",key,c->db->id); + if (expire) notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC, + "expire",key,c->db->id); addReply(c, nx ? shared.cone : shared.ok); } @@ -110,7 +111,7 @@ void getsetCommand(redisClient *c) { if (getGenericCommand(c) == REDIS_ERR) return; c->argv[2] = tryObjectEncoding(c->argv[2]); setKey(c->db,c->argv[1],c->argv[2]); - notifyKeyspaceEvent("set",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"set",c->argv[1],c->db->id); server.dirty++; } @@ -172,7 +173,8 @@ void setrangeCommand(redisClient *c) { o->ptr = sdsgrowzero(o->ptr,offset+sdslen(value)); memcpy((char*)o->ptr+offset,value,sdslen(value)); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("setrange",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING, + "setrange",c->argv[1],c->db->id); server.dirty++; } addReplyLongLong(c,sdslen(o->ptr)); @@ -257,7 +259,7 @@ void msetGenericCommand(redisClient *c, int nx) { for (j = 1; j < c->argc; j += 2) { c->argv[j+1] = tryObjectEncoding(c->argv[j+1]); setKey(c->db,c->argv[j],c->argv[j+1]); - notifyKeyspaceEvent("set",c->argv[j],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"set",c->argv[j],c->db->id); } server.dirty += (c->argc-1)/2; addReply(c, nx ? shared.cone : shared.ok); @@ -292,7 +294,7 @@ void incrDecrCommand(redisClient *c, long long incr) { else dbAdd(c->db,c->argv[1],new); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("incrby",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"incrby",c->argv[1],c->db->id); server.dirty++; addReply(c,shared.colon); addReply(c,new); @@ -342,7 +344,7 @@ void incrbyfloatCommand(redisClient *c) { else dbAdd(c->db,c->argv[1],new); signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("incrbyfloat",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"incrbyfloat",c->argv[1],c->db->id); server.dirty++; addReplyBulk(c,new); @@ -390,7 +392,7 @@ void appendCommand(redisClient *c) { totlen = sdslen(o->ptr); } signalModifiedKey(c->db,c->argv[1]); - notifyKeyspaceEvent("append",c->argv[1],c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"append",c->argv[1],c->db->id); server.dirty++; addReplyLongLong(c,totlen); } diff --git a/src/t_zset.c b/src/t_zset.c index 4a9383c1298..b458f88a149 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -968,7 +968,8 @@ void zaddGenericCommand(redisClient *c, int incr) { zfree(scores); if (added || updated) { signalModifiedKey(c->db,key); - notifyKeyspaceEvent(incr ? "zincr" : "zadd", key, c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_ZSET, + incr ? "zincr" : "zadd", key, c->db->id); } } @@ -1029,8 +1030,9 @@ void zremCommand(redisClient *c) { } if (deleted) { - notifyKeyspaceEvent("zrem",key,c->db->id); - if (keyremoved) notifyKeyspaceEvent("del",key,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_ZSET,"zrem",key,c->db->id); + if (keyremoved) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",key,c->db->id); signalModifiedKey(c->db,key); server.dirty += deleted; } @@ -1073,8 +1075,9 @@ void zremrangebyscoreCommand(redisClient *c) { if (deleted) { signalModifiedKey(c->db,key); - notifyKeyspaceEvent("zrembyscore",key,c->db->id); - if (keyremoved) notifyKeyspaceEvent("del",key,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_ZSET,"zrembyscore",key,c->db->id); + if (keyremoved) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",key,c->db->id); } server.dirty += deleted; addReplyLongLong(c,deleted); @@ -1132,8 +1135,9 @@ void zremrangebyrankCommand(redisClient *c) { if (deleted) { signalModifiedKey(c->db,key); - notifyKeyspaceEvent("zrembyrank",key,c->db->id); - if (keyremoved) notifyKeyspaceEvent("del",key,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_ZSET,"zrembyrank",key,c->db->id); + if (keyremoved) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",key,c->db->id); } server.dirty += deleted; addReplyLongLong(c,deleted); @@ -1676,7 +1680,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { if (dbDelete(c->db,dstkey)) { signalModifiedKey(c->db,dstkey); - notifyKeyspaceEvent("del",dstkey,c->db->id); + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",dstkey,c->db->id); touched = 1; server.dirty++; } @@ -1689,7 +1693,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { dbAdd(c->db,dstkey,dstobj); addReplyLongLong(c,zsetLength(dstobj)); if (!touched) signalModifiedKey(c->db,dstkey); - notifyKeyspaceEvent( + notifyKeyspaceEvent(REDIS_NOTIFY_ZSET, (op == REDIS_OP_UNION) ? "zunionstore" : "zinterstore", dstkey,c->db->id); server.dirty++; diff --git a/tests/assets/default.conf b/tests/assets/default.conf index f264bed3c44..8d68e74f4e1 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -13,7 +13,7 @@ # units are case insensitive so 1GB 1Gb 1gB are all the same. # Enable keyspace events notification for testing so that we cover more code. -notify-keyspace-events yes +notify-keyspace-events A # By default Redis does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. From 9db8fbcfdd8e9f0d98c71030cb4562f3b8f8af31 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Jan 2013 17:34:52 +0100 Subject: [PATCH 0461/1752] Keyspace notifications: fixed a leak and a bug introduced in the latest commit. --- src/notify.c | 10 ++++++---- tests/assets/default.conf | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/notify.c b/src/notify.c index f8e01829503..bb598c4f21c 100644 --- a/src/notify.c +++ b/src/notify.c @@ -92,22 +92,22 @@ sds keyspaceEventsFlagsToString(int flags) { * 'dbid' is the database ID where the key lives. */ void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid) { sds chan; - robj *chanobj; + robj *chanobj, *eventobj; int len = -1; char buf[24]; /* If notifications for this class of events are off, return ASAP. */ if (!(server.notify_keyspace_events & type)) return; + eventobj = createStringObject(event,strlen(event)); + /* __keyspace@__: notifications. */ if (server.notify_keyspace_events & REDIS_NOTIFY_KEYSPACE) { - robj *eventobj; - chan = sdsnewlen("__keyspace@",11); len = ll2string(buf,sizeof(buf),dbid); chan = sdscatlen(chan, buf, len); chan = sdscatlen(chan, "__:", 3); - eventobj = createStringObject(event,strlen(event)); + chan = sdscatsds(chan, key->ptr); chanobj = createObject(REDIS_STRING, chan); pubsubPublishMessage(chanobj, eventobj); decrRefCount(chanobj); @@ -119,8 +119,10 @@ void notifyKeyspaceEvent(int type, char *event, robj *key, int dbid) { if (len == -1) len = ll2string(buf,sizeof(buf),dbid); chan = sdscatlen(chan, buf, len); chan = sdscatlen(chan, "__:", 3); + chan = sdscatsds(chan, eventobj->ptr); chanobj = createObject(REDIS_STRING, chan); pubsubPublishMessage(chanobj, key); decrRefCount(chanobj); } + decrRefCount(eventobj); } diff --git a/tests/assets/default.conf b/tests/assets/default.conf index 8d68e74f4e1..241db2d60ee 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -13,7 +13,7 @@ # units are case insensitive so 1GB 1Gb 1gB are all the same. # Enable keyspace events notification for testing so that we cover more code. -notify-keyspace-events A +notify-keyspace-events KEA # By default Redis does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. From d3cf778d827150f0d870f955ca3de0cd3454dcde Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Jan 2013 13:00:03 +0100 Subject: [PATCH 0462/1752] Send 'expired' events when a key expires by lookup. --- src/db.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db.c b/src/db.c index f460f14634b..11d4f0b1eec 100644 --- a/src/db.c +++ b/src/db.c @@ -532,6 +532,8 @@ int expireIfNeeded(redisDb *db, robj *key) { /* Delete the key */ server.stat_expiredkeys++; propagateExpire(db,key); + notifyKeyspaceEvent(REDIS_NOTIFY_EXPIRED, + "expired",key,db->id); return dbDelete(db,key); } From 664de412afc14a72839d256cb6fdb31c03e3e039 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Jan 2013 13:04:23 +0100 Subject: [PATCH 0463/1752] Tests for keyspace notifications. --- tests/unit/pubsub.tcl | 156 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 155 insertions(+), 1 deletion(-) diff --git a/tests/unit/pubsub.tcl b/tests/unit/pubsub.tcl index 7691516009d..90166885100 100644 --- a/tests/unit/pubsub.tcl +++ b/tests/unit/pubsub.tcl @@ -193,7 +193,7 @@ start_server {tags {"pubsub"}} { $rd1 close } - test "PUNSUBSCRIBE and UNSUBSCRIBE should always reply." { + test "PUNSUBSCRIBE and UNSUBSCRIBE should always reply" { # Make sure we are not subscribed to any channel at all. r punsubscribe r unsubscribe @@ -202,4 +202,158 @@ start_server {tags {"pubsub"}} { set reply2 [r unsubscribe] concat $reply1 $reply2 } {punsubscribe {} 0 unsubscribe {} 0} + + ### Keyspace events notification tests + + test "Keyspace notifications: we receive keyspace notifications" { + r config set notify-keyspace-events KA + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r set foo bar + assert_equal {pmessage * __keyspace@9__:foo set} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: we receive keyevent notifications" { + r config set notify-keyspace-events EA + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r set foo bar + assert_equal {pmessage * __keyevent@9__:set foo} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: we can receive both kind of events" { + r config set notify-keyspace-events KEA + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r set foo bar + assert_equal {pmessage * __keyspace@9__:foo set} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:set foo} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: we are able to mask events" { + r config set notify-keyspace-events KEl + r del mylist + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r set foo bar + r lpush mylist a + # No notification for set, because only list commands are enabled. + assert_equal {pmessage * __keyspace@9__:mylist lpush} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:lpush mylist} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: general events test" { + r config set notify-keyspace-events KEg + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r set foo bar + r expire foo 1 + r del foo + assert_equal {pmessage * __keyspace@9__:foo expire} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:expire foo} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:foo del} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:del foo} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: list events test" { + r config set notify-keyspace-events KEl + r del mylist + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r lpush mylist a + r rpush mylist a + r rpop mylist + assert_equal {pmessage * __keyspace@9__:mylist lpush} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:lpush mylist} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:mylist rpush} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:rpush mylist} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:mylist rpop} [$rd1 read] + assert_equal {pmessage * __keyevent@9__:rpop mylist} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: set events test" { + r config set notify-keyspace-events Ks + r del myset + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r sadd myset a b c d + r srem myset x + r sadd myset x y z + r srem myset x + assert_equal {pmessage * __keyspace@9__:myset sadd} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:myset sadd} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:myset srem} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: zset events test" { + r config set notify-keyspace-events Kz + r del myzset + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r zadd myzset 1 a 2 b + r zrem myzset x + r zadd myzset 3 x 4 y 5 z + r zrem myzset x + assert_equal {pmessage * __keyspace@9__:myzset zadd} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:myzset zadd} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:myzset zrem} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: hash events test" { + r config set notify-keyspace-events Kh + r del myhash + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r hmset myhash yes 1 no 0 + r hincrby myhash yes 10 + assert_equal {pmessage * __keyspace@9__:myhash hset} [$rd1 read] + assert_equal {pmessage * __keyspace@9__:myhash hincrby} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: expired events (triggered expire)" { + r config set notify-keyspace-events Ex + r del foo + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r psetex foo 100 1 + wait_for_condition 50 100 { + [r exists foo] == 0 + } else { + fail "Key does not expire?!" + } + assert_equal {pmessage * __keyevent@9__:expired foo} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: expired events (background expire)" { + r config set notify-keyspace-events Ex + r del foo + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r psetex foo 100 1 + assert_equal {pmessage * __keyevent@9__:expired foo} [$rd1 read] + $rd1 close + } + + test "Keyspace notifications: evicted events" { + r config set notify-keyspace-events Ee + r config set maxmemory-policy allkeys-lru + r flushdb + set rd1 [redis_deferring_client] + assert_equal {1} [psubscribe $rd1 *] + r set foo bar + r config set maxmemory 1 + assert_equal {pmessage * __keyevent@9__:evicted foo} [$rd1 read] + r config set maxmemory 0 + $rd1 close + } } From ca8e7d4f8a1983bf3e4fb6b52f8caa06028474ac Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 29 Jan 2013 13:43:13 +0100 Subject: [PATCH 0464/1752] Generate del events when S*STORE commands delete the destination key. --- src/t_set.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/t_set.c b/src/t_set.c index a22bbb67b95..e56a5fd4edc 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -706,7 +706,7 @@ void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, if (dstkey) { /* Store the resulting set into the target, if the intersection * is not an empty set. */ - dbDelete(c->db,dstkey); + int deleted = dbDelete(c->db,dstkey); if (setTypeSize(dstset) > 0) { dbAdd(c->db,dstkey,dstset); addReplyLongLong(c,setTypeSize(dstset)); @@ -715,6 +715,9 @@ void sinterGenericCommand(redisClient *c, robj **setkeys, unsigned long setnum, } else { decrRefCount(dstset); addReply(c,shared.czero); + if (deleted) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del", + dstkey,c->db->id); } signalModifiedKey(c->db,dstkey); server.dirty++; @@ -873,7 +876,7 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * } else { /* If we have a target key where to store the resulting set * create this key with the result set inside */ - dbDelete(c->db,dstkey); + int deleted = dbDelete(c->db,dstkey); if (setTypeSize(dstset) > 0) { dbAdd(c->db,dstkey,dstset); addReplyLongLong(c,setTypeSize(dstset)); @@ -883,6 +886,9 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * } else { decrRefCount(dstset); addReply(c,shared.czero); + if (deleted) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del", + dstkey,c->db->id); } signalModifiedKey(c->db,dstkey); server.dirty++; From 8024de7eb07edb119a235a446a956a4a0281296b Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 29 Jan 2013 13:50:01 +0100 Subject: [PATCH 0465/1752] Z*STORE event fixed: generate del only if resulting sorted set is empty. --- src/t_zset.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/t_zset.c b/src/t_zset.c index b458f88a149..8ef9c5376b7 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1680,7 +1680,6 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { if (dbDelete(c->db,dstkey)) { signalModifiedKey(c->db,dstkey); - notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",dstkey,c->db->id); touched = 1; server.dirty++; } @@ -1700,6 +1699,8 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { } else { decrRefCount(dstobj); addReply(c,shared.czero); + if (touched) + notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC,"del",dstkey,c->db->id); } zfree(src); } From 57e411c687ada99fbf467c94ab09953e3cc4feb3 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Mon, 4 Feb 2013 14:01:08 +0800 Subject: [PATCH 0466/1752] Fix a bug in srandmemberWithCountCommand() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In CASE 2, the call sunionDiffGenericCommand will involve the string "srandmember" > sadd foo one (integer 1) > sadd srandmember two (integer 2) > srandmember foo 3 1)"one" 2)"two" --- src/t_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index e56a5fd4edc..9d4bab6701b 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -469,7 +469,7 @@ void srandmemberWithCountCommand(redisClient *c) { * The number of requested elements is greater than the number of * elements inside the set: simply return the whole set. */ if (count >= size) { - sunionDiffGenericCommand(c,c->argv,c->argc-1,NULL,REDIS_OP_UNION); + sunionDiffGenericCommand(c,c->argv+1,1,NULL,REDIS_OP_UNION); return; } From 44d0eac568ed057ad1cc4797d13abf35f7e2616d Mon Sep 17 00:00:00 2001 From: David Celis Date: Sun, 3 Feb 2013 11:40:07 -0800 Subject: [PATCH 0467/1752] Fix a few typos and improve grammar of redis.conf Make several edits to the example redis.conf configuration file for improved flow and grammar. Signed-off-by: David Celis --- redis.conf | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/redis.conf b/redis.conf index a6932cdf391..15391f3f6d0 100644 --- a/redis.conf +++ b/redis.conf @@ -39,8 +39,8 @@ port 6379 # Close the connection after a client is idle for N seconds (0 to disable) timeout 0 -# Set server verbosity to 'debug' -# it can be one of: +# Specify the server verbosity level. +# This can be one of: # debug (a lot of information, useful for development/testing) # verbose (many rarely useful info, but not a mess like the debug level) # notice (moderately verbose, what you want in production probably) @@ -59,7 +59,7 @@ logfile stdout # Specify the syslog identity. # syslog-ident redis -# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. +# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. # syslog-facility local0 # Set the number of databases. The default database is DB 0, you can select @@ -114,7 +114,7 @@ stop-writes-on-bgsave-error yes # the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes -# Since verison 5 of RDB a CRC64 checksum is placed at the end of the file. +# Since version 5 of RDB a CRC64 checksum is placed at the end of the file. # This makes the format more resistant to corruption but there is a performance # hit to pay (around 10%) when saving and loading RDB files, so you can disable it # for maximum performances. @@ -131,7 +131,7 @@ dbfilename dump.rdb # The DB will be written inside this directory, with the filename specified # above using the 'dbfilename' configuration directive. # -# Also the Append Only File will be created inside this directory. +# The Append Only File will also be created inside this directory. # # Note that you must specify a directory here, not a file name. dir ./ @@ -152,7 +152,7 @@ dir ./ # # masterauth -# When a slave lost the connection with the master, or when the replication +# When a slave loses its connection with the master, or when the replication # is still in progress, the slave can act in two different ways: # # 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will @@ -230,14 +230,14 @@ slave-priority 100 # # It is possible to change the name of dangerous commands in a shared # environment. For instance the CONFIG command may be renamed into something -# of hard to guess so that it will be still available for internal-use -# tools but not available for general clients. +# hard to guess so that it will still be available for internal-use tools +# but not available for general clients. # # Example: # # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 # -# It is also possible to completely kill a command renaming it into +# It is also possible to completely kill a command by renaming it into # an empty string: # # rename-command CONFIG "" @@ -281,7 +281,7 @@ slave-priority 100 # maxmemory # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached? You can select among five behavior: +# is reached. You can select among five behaviors: # # volatile-lru -> remove the key with an expire set using an LRU algorithm # allkeys-lru -> remove any key accordingly to the LRU algorithm @@ -290,7 +290,7 @@ slave-priority 100 # volatile-ttl -> remove the key with the nearest expire time (minor TTL) # noeviction -> don't expire at all, just return an error on write operations # -# Note: with all the kind of policies, Redis will return an error on write +# Note: with any of the above policies, Redis will return an error on write # operations, when there are not suitable keys for eviction. # # At the date of writing this commands are: set setnx setex append @@ -346,7 +346,7 @@ appendonly no # always: fsync after every write to the append only log . Slow, Safest. # everysec: fsync only one time every second. Compromise. # -# The default is "everysec" that's usually the right compromise between +# The default is "everysec", as that's usually the right compromise between # speed and data safety. It's up to you to understand if you can relax this to # "no" that will let the operating system flush the output buffer when # it wants, for better performances (but if you can live with the idea of @@ -374,9 +374,9 @@ appendfsync everysec # that will prevent fsync() from being called in the main process while a # BGSAVE or BGREWRITEAOF is in progress. # -# This means that while another child is saving the durability of Redis is -# the same as "appendfsync none", that in practical terms means that it is -# possible to lost up to 30 seconds of log in the worst scenario (with the +# This means that while another child is saving, the durability of Redis is +# the same as "appendfsync none". In practical terms, this means that it is +# possible to lose up to 30 seconds of log in the worst scenario (with the # default Linux settings). # # If you have latency problems turn this to "yes". Otherwise leave it as @@ -385,10 +385,10 @@ no-appendfsync-on-rewrite no # Automatic rewrite of the append only file. # Redis is able to automatically rewrite the log file implicitly calling -# BGREWRITEAOF when the AOF log size will growth by the specified percentage. +# BGREWRITEAOF when the AOF log size grows by the specified percentage. # # This is how it works: Redis remembers the size of the AOF file after the -# latest rewrite (or if no rewrite happened since the restart, the size of +# latest rewrite (if no rewrite has happened since the restart, the size of # the AOF at startup is used). # # This base size is compared to the current size. If the current size is @@ -570,7 +570,7 @@ activerehashing yes # Instead there is a default limit for pubsub and slave clients, since # subscribers and slaves receive data in a push fashion. # -# Both the hard or the soft limit can be disabled just setting it to zero. +# Both the hard or the soft limit can be disabled by setting them to zero. client-output-buffer-limit normal 0 0 0 client-output-buffer-limit slave 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 @@ -579,7 +579,7 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # closing connections of clients in timeot, purging expired keys that are # never requested, and so forth. # -# Not all tasks are perforemd with the same frequency, but Redis checks for +# Not all tasks are performed with the same frequency, but Redis checks for # tasks to perform accordingly to the specified "hz" value. # # By default "hz" is set to 10. Raising the value will use more CPU when From 2543fa647613303a5cbb6e3896f724923a86f48f Mon Sep 17 00:00:00 2001 From: Rock Li Date: Tue, 5 Feb 2013 15:56:04 +0800 Subject: [PATCH 0468/1752] retval doesn't initalized If each if conditions are all fail, variable retval will under uninitlized --- src/t_set.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/t_set.c b/src/t_set.c index 9d4bab6701b..a522cd88ae8 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -491,7 +491,7 @@ void srandmemberWithCountCommand(redisClient *c) { /* Add all the elements into the temporary dictionary. */ si = setTypeInitIterator(set); while((encoding = setTypeNext(si,&ele,&llele)) != -1) { - int retval; + int retval = DICT_ERR; if (encoding == REDIS_ENCODING_INTSET) { retval = dictAdd(d,createStringObjectFromLongLong(llele),NULL); From e92edd3cee4030c836e4a5d4daafd43b684cb98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Thu, 24 Jan 2013 09:25:47 +1100 Subject: [PATCH 0469/1752] Check available tcl versions --- runtest | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/runtest b/runtest index 0eb384c24dc..d8451df57eb 100755 --- a/runtest +++ b/runtest @@ -1,9 +1,14 @@ #!/bin/sh -TCL=tclsh8.5 -which $TCL -if [ "$?" != "0" ] +TCL_VERSIONS="8.5 8.6" +TCLSH="" + +for VERSION in $TCL_VERSIONS; do + TCL=`which tclsh$VERSION 2>/dev/null` && TCLSH=$TCL +done + +if [ -z $TCLSH ] then - echo "You need '$TCL' in order to run the Redis test" + echo "You need tcl 8.5 or newer in order to run the Redis test" exit 1 fi -$TCL tests/test_helper.tcl $* +$TCLSH tests/test_helper.tcl $* From 1124cda37a5dff6379afd611a095adfca87ed57a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Thu, 24 Jan 2013 09:36:59 +1100 Subject: [PATCH 0470/1752] Enforce tcl 8.5 or newer --- tests/test_helper.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 38fb1f5394d..aeff69f7e06 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -2,6 +2,8 @@ # This softare is released under the BSD License. See the COPYING file for # more information. +package require Tcl 8.5 + set tcl_precision 17 source tests/support/redis.tcl source tests/support/server.tcl From d2a8bca82bd05310934f75e7bdbf1f107c3d8676 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Thu, 24 Jan 2013 09:37:18 +1100 Subject: [PATCH 0471/1752] Use `info nameofexectuable` to find current executable --- tests/integration/replication-4.tcl | 3 ++- tests/integration/replication.tcl | 3 ++- tests/test_helper.tcl | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 69fcab37324..0a0dcdc2bdd 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -1,5 +1,6 @@ proc start_bg_complex_data {host port db ops} { - exec tclsh8.5 tests/helpers/bg_complex_data.tcl $host $port $db $ops & + set tclsh [info nameofexecutable] + exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops & } proc stop_bg_complex_data {handle} { diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index da94b088065..5ca6449f4fd 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -78,7 +78,8 @@ start_server {tags {"repl"}} { } proc start_write_load {host port seconds} { - exec tclsh8.5 tests/helpers/gen_write_load.tcl $host $port $seconds & + set tclsh [info nameofexecutable] + exec $tclsh tests/helpers/gen_write_load.tcl $host $port $seconds & } proc stop_write_load {handle} { diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index aeff69f7e06..483774219b4 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -178,6 +178,7 @@ proc find_available_port start { proc test_server_main {} { cleanup + set tclsh [info nameofexecutable] # Open a listening socket, trying different ports in order to find a # non busy one. set port [find_available_port 11111] @@ -191,7 +192,7 @@ proc test_server_main {} { set start_port [expr {$::port+100}] for {set j 0} {$j < $::numclients} {incr j} { set start_port [find_available_port $start_port] - set p [exec tclsh8.5 [info script] {*}$::argv \ + set p [exec $tclsh [info script] {*}$::argv \ --client $port --port $start_port &] lappend ::clients_pids $p incr start_port 10 From 14daa01c52bfa1ce6d6dc312e49ae131f70623be Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Feb 2013 12:02:36 +0100 Subject: [PATCH 0472/1752] Test: No clients timeout while testing. --- tests/assets/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/assets/default.conf b/tests/assets/default.conf index 241db2d60ee..d6da94e0d3d 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -38,7 +38,7 @@ port 6379 # unixsocket /tmp/redis.sock # Close the connection after a client is idle for N seconds (0 to disable) -timeout 300 +timeout 0 # Set server verbosity to 'debug' # it can be one of: From 1d80acae541f4287ea7416d4d526d65086bace44 Mon Sep 17 00:00:00 2001 From: charsyam Date: Thu, 31 Jan 2013 12:09:16 +0900 Subject: [PATCH 0473/1752] Turn off TCP_NODELAY on the slave socket after SYNC. Further details from @antirez: It was reported by @StopForumSpam on Twitter that the Redis replication link was strangely using multiple TCP packets for multiple commands. This wastes a lot of bandwidth and is due to the TCP_NODELAY option we enable on the socket after accepting a new connection. However the master -> slave channel is a one-way channel since Redis replication is asynchronous, so there is no point in trying to reduce the latency, we should aim to reduce the bandwidth. For this reason this commit introduces the ability to disable the nagle algorithm on the socket after a successful SYNC. This feature is off by default because the delay can be up to 40 milliseconds with normally configured Linux kernels. --- src/anet.c | 14 ++++++++++++-- src/anet.h | 1 + src/config.c | 7 +++++++ src/redis.c | 1 + src/redis.h | 1 + src/replication.c | 8 ++++++++ 6 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/anet.c b/src/anet.c index 4da3e28db7e..d002cb31ca3 100644 --- a/src/anet.c +++ b/src/anet.c @@ -75,9 +75,8 @@ int anetNonBlock(char *err, int fd) return ANET_OK; } -int anetTcpNoDelay(char *err, int fd) +static int _anetTcpNoDelay(char *err, int fd, int yes) { - int yes = 1; if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) { anetSetError(err, "setsockopt TCP_NODELAY: %s", strerror(errno)); @@ -86,6 +85,17 @@ int anetTcpNoDelay(char *err, int fd) return ANET_OK; } +int anetTcpNoDelay(char *err, int fd) +{ + return _anetTcpNoDelay(err, fd, 1); +} + +int anetTcpNoDelayOff(char *err, int fd) +{ + return _anetTcpNoDelay(err, fd, 0); +} + + int anetSetSendBuffer(char *err, int fd, int buffsize) { if (setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &buffsize, sizeof(buffsize)) == -1) diff --git a/src/anet.h b/src/anet.h index 062b22c5644..56ae50573eb 100644 --- a/src/anet.h +++ b/src/anet.h @@ -52,6 +52,7 @@ int anetUnixAccept(char *err, int serversock); int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetTcpNoDelay(char *err, int fd); +int anetTcpNoDelayOff(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetPeerToString(int fd, char *ip, int *port); diff --git a/src/config.c b/src/config.c index 4ab19c1515a..365e50476f9 100644 --- a/src/config.c +++ b/src/config.c @@ -382,6 +382,8 @@ void loadServerConfigFromString(char *config) { if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } + } else if (!strcasecmp(argv[0],"slave-tcp-nodelay-off") && argc == 2) { + server.slave_tcp_nodelay_off = atoi(argv[1]); } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { server.slave_priority = atoi(argv[1]); } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { @@ -715,6 +717,10 @@ void configSetCommand(redisClient *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; + } else if (!strcasecmp(c->argv[2]->ptr,"slave-tcp-nodelay-off")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR ) goto badfmt; + + server.slave_tcp_nodelay_off = ll; } else if (!strcasecmp(c->argv[2]->ptr,"slave-priority")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; @@ -808,6 +814,7 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("repl-timeout",server.repl_timeout); config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); + config_get_numerical_field("slave-tcp-nodelay-off",server.slave_tcp_nodelay_off); config_get_numerical_field("slave-priority",server.slave_priority); config_get_numerical_field("hz",server.hz); diff --git a/src/redis.c b/src/redis.c index 6786eeddc3a..c1bdd5197be 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1164,6 +1164,7 @@ void initServerConfig() { server.repl_serve_stale_data = 1; server.repl_slave_ro = 1; server.repl_down_since = time(NULL); + server.slave_tcp_nodelay_off = 1; server.slave_priority = REDIS_DEFAULT_SLAVE_PRIORITY; /* Client output buffer limits */ diff --git a/src/redis.h b/src/redis.h index 635232f96cd..dad5fc5a994 100644 --- a/src/redis.h +++ b/src/redis.h @@ -633,6 +633,7 @@ struct redisServer { int repl_serve_stale_data; /* Serve stale data when link is down? */ int repl_slave_ro; /* Slave is read only? */ time_t repl_down_since; /* Unix time at which link with master went down */ + int slave_tcp_nodelay_off; /* turn off slave's tcp nodelay */ int slave_priority; /* Reported in INFO and used by Sentinel. */ /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ diff --git a/src/replication.c b/src/replication.c index 2f0cba70158..fffac0e120d 100644 --- a/src/replication.c +++ b/src/replication.c @@ -118,6 +118,14 @@ void syncCommand(redisClient *c) { /* ignore SYNC if already slave or in monitor mode */ if (c->flags & REDIS_SLAVE) return; + if (server.slave_tcp_nodelay_off) { + redisLog(REDIS_NOTICE, "Turning off slave's :%d TCP NODELAY SETTING", c->fd); + char err[1024]; + if (anetTcpNoDelayOff(err, c->fd) == ANET_ERR) + redisLog(REDIS_WARNING, + "Can't turn off %d 's tcp nodelay setting: %s", c->fd, err); + } + /* Refuse SYNC requests if we are a slave but the link with our master * is not ok... */ if (server.masterhost && server.repl_state != REDIS_REPL_CONNECTED) { From 5f7dff4d16aae48db9ec4b71a5044f1c22100390 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Jan 2013 11:14:15 +0100 Subject: [PATCH 0474/1752] TCP_NODELAY after SYNC: changes to the implementation. --- redis.conf | 15 +++++++++++++++ src/anet.c | 12 ++++++------ src/anet.h | 4 ++-- src/config.c | 16 ++++++++++------ src/networking.c | 2 +- src/redis.c | 2 +- src/redis.h | 2 +- src/replication.c | 11 +++-------- 8 files changed, 39 insertions(+), 25 deletions(-) diff --git a/redis.conf b/redis.conf index 15391f3f6d0..ccb437c34d0 100644 --- a/redis.conf +++ b/redis.conf @@ -196,6 +196,21 @@ slave-read-only yes # # repl-timeout 60 +# Disable TCP_NODELAY on the slave socket after SYNC? +# +# If you select "yes" Redis will use a smaller number of TCP packets and +# less bandwidth to send data to slaves. But this can add a delay for +# the data to appear on the slave side, up to 40 milliseconds with +# Linux kernels using a default configuration. +# +# If you select "no" the delay for data to appear on the slave side will +# be reduced but more bandwidth will be used for replication. +# +# By default we optimize for low latency, but in very high traffic conditions +# or when the master and slaves are many hops away, turning this to "yes" may +# be a good idea. +repl-disable-tcp-nodelay no + # The slave priority is an integer number published by Redis in the INFO output. # It is used by Redis Sentinel in order to select a slave to promote into a # master if the master is no longer working correctly. diff --git a/src/anet.c b/src/anet.c index d002cb31ca3..3f037dcad58 100644 --- a/src/anet.c +++ b/src/anet.c @@ -75,9 +75,9 @@ int anetNonBlock(char *err, int fd) return ANET_OK; } -static int _anetTcpNoDelay(char *err, int fd, int yes) +static int anetSetTcpNoDelay(char *err, int fd, int val) { - if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)) == -1) + if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) == -1) { anetSetError(err, "setsockopt TCP_NODELAY: %s", strerror(errno)); return ANET_ERR; @@ -85,14 +85,14 @@ static int _anetTcpNoDelay(char *err, int fd, int yes) return ANET_OK; } -int anetTcpNoDelay(char *err, int fd) +int anetEnableTcpNoDelay(char *err, int fd) { - return _anetTcpNoDelay(err, fd, 1); + return anetSetTcpNoDelay(err, fd, 1); } -int anetTcpNoDelayOff(char *err, int fd) +int anetDisableTcpNoDelay(char *err, int fd) { - return _anetTcpNoDelay(err, fd, 0); + return anetSetTcpNoDelay(err, fd, 0); } diff --git a/src/anet.h b/src/anet.h index 56ae50573eb..dbf36dbd81f 100644 --- a/src/anet.h +++ b/src/anet.h @@ -51,8 +51,8 @@ int anetTcpAccept(char *err, int serversock, char *ip, int *port); int anetUnixAccept(char *err, int serversock); int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); -int anetTcpNoDelay(char *err, int fd); -int anetTcpNoDelayOff(char *err, int fd); +int anetEnableTcpNoDelay(char *err, int fd); +int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetPeerToString(int fd, char *ip, int *port); diff --git a/src/config.c b/src/config.c index 365e50476f9..3fc4e1135b4 100644 --- a/src/config.c +++ b/src/config.c @@ -230,6 +230,10 @@ void loadServerConfigFromString(char *config) { err = "repl-timeout must be 1 or greater"; goto loaderr; } + } else if (!strcasecmp(argv[0],"repl-disable-tcp-nodelay") && argc==2) { + if ((server.repl_disable_tcp_nodelay = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { server.masterauth = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"slave-serve-stale-data") && argc == 2) { @@ -382,8 +386,6 @@ void loadServerConfigFromString(char *config) { if ((server.stop_writes_on_bgsave_err = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; } - } else if (!strcasecmp(argv[0],"slave-tcp-nodelay-off") && argc == 2) { - server.slave_tcp_nodelay_off = atoi(argv[1]); } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { server.slave_priority = atoi(argv[1]); } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { @@ -717,10 +719,11 @@ void configSetCommand(redisClient *c) { if (flags == -1) goto badfmt; server.notify_keyspace_events = flags; - } else if (!strcasecmp(c->argv[2]->ptr,"slave-tcp-nodelay-off")) { - if (getLongLongFromObject(o,&ll) == REDIS_ERR ) goto badfmt; + } else if (!strcasecmp(c->argv[2]->ptr,"repl-disable-tcp-nodelay")) { + int yn = yesnotoi(o->ptr); - server.slave_tcp_nodelay_off = ll; + if (yn == -1) goto badfmt; + server.repl_disable_tcp_nodelay = yn; } else if (!strcasecmp(c->argv[2]->ptr,"slave-priority")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; @@ -814,7 +817,6 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("repl-timeout",server.repl_timeout); config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); - config_get_numerical_field("slave-tcp-nodelay-off",server.slave_tcp_nodelay_off); config_get_numerical_field("slave-priority",server.slave_priority); config_get_numerical_field("hz",server.hz); @@ -831,6 +833,8 @@ void configGetCommand(redisClient *c) { config_get_bool_field("rdbcompression", server.rdb_compression); config_get_bool_field("rdbchecksum", server.rdb_checksum); config_get_bool_field("activerehashing", server.activerehashing); + config_get_bool_field("repl-disable-tcp-nodelay", + server.repl_disable_tcp_nodelay); /* Everything we can't handle with macros follows. */ diff --git a/src/networking.c b/src/networking.c index 0853ad92300..a7d14b49146 100644 --- a/src/networking.c +++ b/src/networking.c @@ -58,7 +58,7 @@ redisClient *createClient(int fd) { * contexts (for instance a Lua script) we need a non connected client. */ if (fd != -1) { anetNonBlock(NULL,fd); - anetTcpNoDelay(NULL,fd); + anetEnableTcpNoDelay(NULL,fd); if (aeCreateFileEvent(server.el,fd,AE_READABLE, readQueryFromClient, c) == AE_ERR) { diff --git a/src/redis.c b/src/redis.c index c1bdd5197be..0f89ca76bc0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1164,7 +1164,7 @@ void initServerConfig() { server.repl_serve_stale_data = 1; server.repl_slave_ro = 1; server.repl_down_since = time(NULL); - server.slave_tcp_nodelay_off = 1; + server.repl_disable_tcp_nodelay = 0; server.slave_priority = REDIS_DEFAULT_SLAVE_PRIORITY; /* Client output buffer limits */ diff --git a/src/redis.h b/src/redis.h index dad5fc5a994..ae01acd595b 100644 --- a/src/redis.h +++ b/src/redis.h @@ -633,7 +633,7 @@ struct redisServer { int repl_serve_stale_data; /* Serve stale data when link is down? */ int repl_slave_ro; /* Slave is read only? */ time_t repl_down_since; /* Unix time at which link with master went down */ - int slave_tcp_nodelay_off; /* turn off slave's tcp nodelay */ + int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ diff --git a/src/replication.c b/src/replication.c index fffac0e120d..3819c885922 100644 --- a/src/replication.c +++ b/src/replication.c @@ -118,14 +118,6 @@ void syncCommand(redisClient *c) { /* ignore SYNC if already slave or in monitor mode */ if (c->flags & REDIS_SLAVE) return; - if (server.slave_tcp_nodelay_off) { - redisLog(REDIS_NOTICE, "Turning off slave's :%d TCP NODELAY SETTING", c->fd); - char err[1024]; - if (anetTcpNoDelayOff(err, c->fd) == ANET_ERR) - redisLog(REDIS_WARNING, - "Can't turn off %d 's tcp nodelay setting: %s", c->fd, err); - } - /* Refuse SYNC requests if we are a slave but the link with our master * is not ok... */ if (server.masterhost && server.repl_state != REDIS_REPL_CONNECTED) { @@ -180,6 +172,9 @@ void syncCommand(redisClient *c) { } c->replstate = REDIS_REPL_WAIT_BGSAVE_END; } + + if (server.repl_disable_tcp_nodelay) + anetDisableTcpNoDelay(NULL, c->fd); /* Non critical if it fails. */ c->repldbfd = -1; c->flags |= REDIS_SLAVE; c->slaveseldb = 0; From 5eb0ec142dac44bb15baf1f4075071d1049d266b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Feb 2013 19:13:00 +0100 Subject: [PATCH 0475/1752] LASTSAVE is a "random" command. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 0f89ca76bc0..0deffe72fab 100644 --- a/src/redis.c +++ b/src/redis.c @@ -214,7 +214,7 @@ struct redisCommand redisCommandTable[] = { {"bgsave",bgsaveCommand,1,"ar",0,NULL,0,0,0,0,0}, {"bgrewriteaof",bgrewriteaofCommand,1,"ar",0,NULL,0,0,0,0,0}, {"shutdown",shutdownCommand,-1,"ar",0,NULL,0,0,0,0,0}, - {"lastsave",lastsaveCommand,1,"r",0,NULL,0,0,0,0,0}, + {"lastsave",lastsaveCommand,1,"rR",0,NULL,0,0,0,0,0}, {"type",typeCommand,2,"r",0,NULL,1,1,1,0,0}, {"multi",multiCommand,1,"rs",0,NULL,0,0,0,0,0}, {"exec",execCommand,1,"sM",0,NULL,0,0,0,0,0}, From cc55a4525a5ab4d4b3a173c0867b568551215842 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2013 10:26:19 +0100 Subject: [PATCH 0476/1752] Make all WATCHers dirty when the slave reloads the DB. --- src/replication.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/replication.c b/src/replication.c index 3819c885922..a6abd0b9493 100644 --- a/src/replication.c +++ b/src/replication.c @@ -427,6 +427,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { return; } redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Loading DB in memory"); + signalFlushedDb(-1); emptyDb(); /* Before loading the DB into memory we need to delete the readable * handler, otherwise it will get called recursively since From 0d470b4c4321194eeb076a81ca68f938f3e658d5 Mon Sep 17 00:00:00 2001 From: Pierre Chapuis Date: Mon, 28 Jan 2013 11:07:17 +0100 Subject: [PATCH 0477/1752] fix comments forgotten in #285 (zipmap -> ziplist) --- src/redis.c | 2 +- src/ziplist.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0deffe72fab..61e9267ba13 100644 --- a/src/redis.c +++ b/src/redis.c @@ -537,7 +537,7 @@ dictType commandTableDictType = { NULL /* val destructor */ }; -/* Hash type hash table (note that small hashes are represented with zipmaps) */ +/* Hash type hash table (note that small hashes are represented with ziplists) */ dictType hashDictType = { dictEncObjHash, /* hash function */ NULL, /* key dup */ diff --git a/src/ziplist.c b/src/ziplist.c index d4ac4f9b4bc..8a6b54a7c09 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -739,10 +739,10 @@ unsigned char *ziplistPrev(unsigned char *zl, unsigned char *p) { } } -/* Get entry pointer to by 'p' and store in either 'e' or 'v' depending +/* Get entry pointed to by 'p' and store in either 'e' or 'v' depending * on the encoding of the entry. 'e' is always set to NULL to be able * to find out whether the string pointer or the integer value was set. - * Return 0 if 'p' points to the end of the zipmap, 1 otherwise. */ + * Return 0 if 'p' points to the end of the ziplist, 1 otherwise. */ unsigned int ziplistGet(unsigned char *p, unsigned char **sstr, unsigned int *slen, long long *sval) { zlentry entry; if (p == NULL || p[0] == ZIP_END) return 0; From fcfdbda1046215d0cadb9cc4d120d36d97171ee5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Jan 2013 17:19:21 +0100 Subject: [PATCH 0478/1752] Sentinel: advertise the promoted slave address only after successful setup. --- src/sentinel.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 8009e5ed9b6..fc857344cc0 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2035,8 +2035,16 @@ void sentinelCommand(redisClient *c) { } else { sentinelAddr *addr = ri->addr; - if ((ri->flags & SRI_FAILOVER_IN_PROGRESS) && ri->promoted_slave) + /* If we are in the middle of a failover, and the slave was + * already successfully switched to master role, we can advertise + * the new address as slave in order to allow clients to talk + * with the new master ASAP. */ + if ((ri->flags & SRI_FAILOVER_IN_PROGRESS) && + ri->promoted_slave && + ri->failover_state >= SENTINEL_FAILOVER_STATE_RECONF_SLAVES) + { addr = ri->promoted_slave->addr; + } addReplyMultiBulkLen(c,2); addReplyBulkCString(c,addr->ip); addReplyBulkLongLong(c,addr->port); From f2817cbd9eabf6fce0f8108db771eb9c83a57e80 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2013 16:30:21 +0100 Subject: [PATCH 0479/1752] Add SO_KEEPALIVE support to anet.c. --- src/anet.c | 43 +++++++++++++++++++++++++++++++++++++++++++ src/anet.h | 1 + 2 files changed, 44 insertions(+) diff --git a/src/anet.c b/src/anet.c index 3f037dcad58..82bca5534d7 100644 --- a/src/anet.c +++ b/src/anet.c @@ -75,6 +75,49 @@ int anetNonBlock(char *err, int fd) return ANET_OK; } +/* Set TCP keep alive option to detect dead peers. The interval option + * is only used for Linux as we are using Linux-specific APIs to set + * the probe send time, interval, and count. */ +int anetKeepAlive(char *err, int fd, int interval) +{ + int val = 1; + + if (setsockopt(fd, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)) == -1) + { + anetSetError(err, "setsockopt SO_KEEPALIVE: %s", strerror(errno)); + return ANET_ERR; + } + +#ifdef __linux__ + /* Default settings are more or less garbage, with the keepalive time + * set to 7200 by default on Linux. Modify settings to make the feature + * actually useful. */ + + /* Send first probe after interval. */ + val = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPIDLE, &val, sizeof(val)) < 0) { + anetSetError(err, "setsockopt TCP_KEEPIDLE: %s\n", strerror(errno)); + return ANET_ERR; + } + + /* Send next probes after interval. */ + val = interval; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { + anetSetError(err, "setsockopt TCP_KEEPINTVL: %s\n", strerror(errno)); + return ANET_ERR; + } + + /* Consider the socket in error state after just one missing ACK reply. */ + val = 1; + if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { + anetSetError(err, "setsockopt TCP_KEEPCNT: %s\n", strerror(errno)); + return ANET_ERR; + } +#endif + + return ANET_OK; +} + static int anetSetTcpNoDelay(char *err, int fd, int val) { if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof(val)) == -1) diff --git a/src/anet.h b/src/anet.h index dbf36dbd81f..696c2c22586 100644 --- a/src/anet.h +++ b/src/anet.h @@ -55,5 +55,6 @@ int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetPeerToString(int fd, char *ip, int *port); +int anetKeepAlive(char *err, int fd, int interval); #endif From 2ed3fc1502e530c42f052448b4155cf6e0135974 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2013 16:40:59 +0100 Subject: [PATCH 0480/1752] Set SO_KEEPALIVE on client sockets if configured to do so. --- src/config.c | 10 ++++++++++ src/networking.c | 2 ++ src/redis.c | 1 + src/redis.h | 1 + 4 files changed, 14 insertions(+) diff --git a/src/config.c b/src/config.c index 3fc4e1135b4..4644530bd0c 100644 --- a/src/config.c +++ b/src/config.c @@ -81,6 +81,11 @@ void loadServerConfigFromString(char *config) { if (server.maxidletime < 0) { err = "Invalid timeout value"; goto loaderr; } + } else if (!strcasecmp(argv[0],"tcp-keepalive") && argc == 2) { + server.tcpkeepalive = atoi(argv[1]); + if (server.tcpkeepalive < 0) { + err = "Invalid tcp-keepalive value"; goto loaderr; + } } else if (!strcasecmp(argv[0],"port") && argc == 2) { server.port = atoi(argv[1]); if (server.port < 0 || server.port > 65535) { @@ -521,6 +526,10 @@ void configSetCommand(redisClient *c) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0 || ll > LONG_MAX) goto badfmt; server.maxidletime = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"tcp-keepalive")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0 || ll > INT_MAX) goto badfmt; + server.tcpkeepalive = ll; } else if (!strcasecmp(c->argv[2]->ptr,"appendfsync")) { if (!strcasecmp(o->ptr,"no")) { server.aof_fsync = AOF_FSYNC_NO; @@ -788,6 +797,7 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("maxmemory",server.maxmemory); config_get_numerical_field("maxmemory-samples",server.maxmemory_samples); config_get_numerical_field("timeout",server.maxidletime); + config_get_numerical_field("tcp-keepalive",server.tcpkeepalive); config_get_numerical_field("auto-aof-rewrite-percentage", server.aof_rewrite_perc); config_get_numerical_field("auto-aof-rewrite-min-size", diff --git a/src/networking.c b/src/networking.c index a7d14b49146..66a2702c518 100644 --- a/src/networking.c +++ b/src/networking.c @@ -59,6 +59,8 @@ redisClient *createClient(int fd) { if (fd != -1) { anetNonBlock(NULL,fd); anetEnableTcpNoDelay(NULL,fd); + if (server.tcpkeepalive) + anetKeepAlive(NULL,fd,server.tcpkeepalive); if (aeCreateFileEvent(server.el,fd,AE_READABLE, readQueryFromClient, c) == AE_ERR) { diff --git a/src/redis.c b/src/redis.c index 61e9267ba13..1e0e0fbca98 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1097,6 +1097,7 @@ void initServerConfig() { server.dbnum = REDIS_DEFAULT_DBNUM; server.verbosity = REDIS_NOTICE; server.maxidletime = REDIS_MAXIDLETIME; + server.tcpkeepalive = 0; server.client_max_querybuf_len = REDIS_MAX_QUERYBUF_LEN; server.saveparams = NULL; server.loading = 0; diff --git a/src/redis.h b/src/redis.h index ae01acd595b..f1a38cebc0c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -568,6 +568,7 @@ struct redisServer { /* Configuration */ int verbosity; /* Loglevel in redis.conf */ int maxidletime; /* Client timeout in seconds */ + int tcpkeepalive; /* Set SO_KEEPALIVE if non-zero. */ size_t client_max_querybuf_len; /* Limit for client query buffer length */ int dbnum; /* Total number of configured DBs */ int daemonize; /* True if running as a daemon */ From 0a14a6542c8901f9868a7ffb5a7b393573176cb8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2013 17:03:11 +0100 Subject: [PATCH 0481/1752] tcp-keepalive option documented in redis.conf. --- redis.conf | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/redis.conf b/redis.conf index ccb437c34d0..676f3492e4a 100644 --- a/redis.conf +++ b/redis.conf @@ -39,6 +39,17 @@ port 6379 # Close the connection after a client is idle for N seconds (0 to disable) timeout 0 +# TCP keepalive. +# +# When this option is set to a non-zero value, SO_KEEPALIVE option will be +# enabled in order to send ACKs just to avoid connection drops or to detect +# dead peers. +# +# The value you specify with this option is the period (in seconds) we use +# in order to refresh the connection with TCP ACKs, however the period is +# only actually set on Linux. Other kernels will use the system-wide default. +tcp-keepalive 0 + # Specify the server verbosity level. # This can be one of: # debug (a lot of information, useful for development/testing) From b06b90b5d8ebe12ff47846caf694ce967f3e913e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Feb 2013 17:06:01 +0100 Subject: [PATCH 0482/1752] Tcp keep-alive: send three probes before detectin an error. Otherwise we end with less reliable connections because it's too easy that a single packet gets lost. --- src/anet.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/anet.c b/src/anet.c index 82bca5534d7..963b6688e53 100644 --- a/src/anet.c +++ b/src/anet.c @@ -100,15 +100,19 @@ int anetKeepAlive(char *err, int fd, int interval) return ANET_ERR; } - /* Send next probes after interval. */ - val = interval; + /* Send next probes after the specified interval. Note that we set the + * delay as interval / 3, as we send three probes before detecting + * an error (see the next setsockopt call). */ + val = interval/3; + if (val == 0) val = 1; if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPINTVL, &val, sizeof(val)) < 0) { anetSetError(err, "setsockopt TCP_KEEPINTVL: %s\n", strerror(errno)); return ANET_ERR; } - /* Consider the socket in error state after just one missing ACK reply. */ - val = 1; + /* Consider the socket in error state after three we send three ACK + * probes without getting a reply. */ + val = 3; if (setsockopt(fd, IPPROTO_TCP, TCP_KEEPCNT, &val, sizeof(val)) < 0) { anetSetError(err, "setsockopt TCP_KEEPCNT: %s\n", strerror(errno)); return ANET_ERR; From 8a22934c6871e51d11378e28fccecab19173e7e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 9 Feb 2013 01:17:59 +0100 Subject: [PATCH 0483/1752] TCP keep-alive. Better documentation in redis.conf. --- redis.conf | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/redis.conf b/redis.conf index 676f3492e4a..36d18b7ad3c 100644 --- a/redis.conf +++ b/redis.conf @@ -41,13 +41,18 @@ timeout 0 # TCP keepalive. # -# When this option is set to a non-zero value, SO_KEEPALIVE option will be -# enabled in order to send ACKs just to avoid connection drops or to detect -# dead peers. +# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence +# of communication. This is useful for two reasons: # -# The value you specify with this option is the period (in seconds) we use -# in order to refresh the connection with TCP ACKs, however the period is -# only actually set on Linux. Other kernels will use the system-wide default. +# 1) Detect dead peers. +# 2) Take the connection alive from the point of view of network +# equipment in the middle. +# +# On Linux, the specified value (in seconds) is the period used to send ACKs. +# Note that to close the connection the double of the time is needed. +# On other kernels the period depends on the kernel configuration. +# +# A reasonable value for this option is 60 seconds. tcp-keepalive 0 # Specify the server verbosity level. From c970816e4d8c9ad1b77d22f3b29b44aba2f4b3e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Feb 2013 12:11:21 +0100 Subject: [PATCH 0484/1752] Makefile: valgrind target added (forces -O0 and libc malloc). --- src/Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 242cf099fcd..0aa409de53b 100644 --- a/src/Makefile +++ b/src/Makefile @@ -217,7 +217,10 @@ gcov: $(MAKE) REDIS_CFLAGS="-fprofile-arcs -ftest-coverage -DCOVERAGE_TEST" REDIS_LDFLAGS="-fprofile-arcs -ftest-coverage" noopt: - $(MAKE) OPT="-O0" + $(MAKE) OPTIMIZATION="-O0" + +valgrind: + $(MAKE) OPTIMIZATION="-O0" MALLOC="libc" src/help.h: @../utils/generate-command-help.rb > help.h From 5a35e485f936a2e22e3df12052fd9182b9d408ef Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 2 Nov 2012 16:31:28 +0100 Subject: [PATCH 0485/1752] Emit SELECT to slaves in a centralized way. Before this commit every Redis slave had its own selected database ID state. This was not actually useful as the emitted stream of commands is identical for all the slaves. Now the the currently selected database is a global state that is set to -1 when a new slave is attached, in order to force the SELECT command to be re-emitted for all the slaves. This change is useful in order to implement replication partial resynchronization in the future, as makes sure that the stream of commands received by slaves, including SELECT commands, are exactly the same for every slave connected, at any time. In this way we could have a global offset that can identify a specific piece of the master -> slaves stream of commands. --- src/redis.c | 2 +- src/redis.h | 2 +- src/replication.c | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/redis.c b/src/redis.c index 1e0e0fbca98..a4654d3a327 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1269,6 +1269,7 @@ void initServer() { server.clients_to_close = listCreate(); server.slaves = listCreate(); server.monitors = listCreate(); + server.slaveseldb = -1; /* Force to emit the first SELECT command. */ server.unblocked_clients = listCreate(); server.ready_keys = listCreate(); @@ -2215,7 +2216,6 @@ void monitorCommand(redisClient *c) { if (c->flags & REDIS_SLAVE) return; c->flags |= (REDIS_SLAVE|REDIS_MONITOR); - c->slaveseldb = 0; listAddNodeTail(server.monitors,c); addReply(c,shared.ok); } diff --git a/src/redis.h b/src/redis.h index f1a38cebc0c..267d1c38b6d 100644 --- a/src/redis.h +++ b/src/redis.h @@ -412,7 +412,6 @@ typedef struct redisClient { time_t lastinteraction; /* time of the last interaction, used for timeout */ time_t obuf_soft_limit_reached_time; int flags; /* REDIS_SLAVE | REDIS_MONITOR | REDIS_MULTI ... */ - int slaveseldb; /* slave selected db, if this client is a slave */ int authenticated; /* when requirepass is non-NULL */ int replstate; /* replication state if this is a slave */ int repldbfd; /* replication DB file descriptor */ @@ -534,6 +533,7 @@ struct redisServer { list *clients; /* List of active clients */ list *clients_to_close; /* Clients to close asynchronously */ list *slaves, *monitors; /* List of slaves and MONITORs */ + int slaveseldb; /* Last SELECTed DB in replication output */ redisClient *current_client; /* Current client, only used on crash report */ char neterr[ANET_ERR_LEN]; /* Error buffer for anet.c */ /* RDB / AOF loading information */ diff --git a/src/replication.c b/src/replication.c index a6abd0b9493..71b2921f042 100644 --- a/src/replication.c +++ b/src/replication.c @@ -54,7 +54,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* Feed slaves that are waiting for the initial SYNC (so these commands * are queued in the output buffer until the initial SYNC completes), * or are already in sync with the master. */ - if (slave->slaveseldb != dictid) { + if (server.slaveseldb != dictid) { robj *selectcmd; if (dictid >= 0 && dictid < REDIS_SHARED_SELECT_CMDS) { @@ -66,11 +66,11 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { } addReply(slave,selectcmd); decrRefCount(selectcmd); - slave->slaveseldb = dictid; } addReplyMultiBulkLen(slave,argc); for (j = 0; j < argc; j++) addReplyBulk(slave,argv[j]); } + server.slaveseldb = dictid; } void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj **argv, int argc) { @@ -177,7 +177,7 @@ void syncCommand(redisClient *c) { anetDisableTcpNoDelay(NULL, c->fd); /* Non critical if it fails. */ c->repldbfd = -1; c->flags |= REDIS_SLAVE; - c->slaveseldb = 0; + server.slaveseldb = -1; /* Force to re-emit the SELECT command. */ listAddNodeTail(server.slaves,c); return; } From 01c21f9943f484722a877c5ee696e208b2cdf8db Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 3 Nov 2012 11:56:28 +0100 Subject: [PATCH 0486/1752] Use replicationFeedSlaves() to send PING to slaves. A Redis master sends PING commands to slaves from time to time: doing this ensures that even if absence of writes, the master->slave channel remains active and the slave can feel the master presence, instead of closing the connection for timeout. This commit changes the way PINGs are sent to slaves in order to use the standard interface used to replicate all the other commands, that is, the function replicationFeedSlaves(). With this change the stream of commands sent to every slave is exactly the same regardless of their exact state (Transferring RDB for first synchronization or slave already online). With the previous implementation the PING was only sent to online slaves, with the result that the output stream from master to slaves was not identical for all the slaves: this is a problem if we want to implement partial resyncs in the future using a global replication stream offset. TL;DR: this commit should not change the behaviour in practical terms, but is just something in preparation for partial resynchronization support. --- src/redis.h | 20 ++++++++++---------- src/replication.c | 24 ++++++++++++------------ 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/redis.h b/src/redis.h index 267d1c38b6d..0134efef746 100644 --- a/src/redis.h +++ b/src/redis.h @@ -197,7 +197,7 @@ #define REDIS_CLIENT_LIMIT_CLASS_PUBSUB 2 #define REDIS_CLIENT_LIMIT_NUM_CLASSES 3 -/* Slave replication state - slave side */ +/* Slave replication state - from the point of view of the slave. */ #define REDIS_REPL_NONE 0 /* No active replication */ #define REDIS_REPL_CONNECT 1 /* Must connect to master */ #define REDIS_REPL_CONNECTING 2 /* Connecting to master */ @@ -205,17 +205,17 @@ #define REDIS_REPL_TRANSFER 4 /* Receiving .rdb from master */ #define REDIS_REPL_CONNECTED 5 /* Connected to master */ -/* Synchronous read timeout - slave side */ -#define REDIS_REPL_SYNCIO_TIMEOUT 5 - -/* Slave replication state - from the point of view of master - * Note that in SEND_BULK and ONLINE state the slave receives new updates +/* Slave replication state - from the point of view of the master. + * In SEND_BULK and ONLINE state the slave receives new updates * in its output queue. In the WAIT_BGSAVE state instead the server is waiting * to start the next background saving in order to send updates to it. */ -#define REDIS_REPL_WAIT_BGSAVE_START 3 /* master waits bgsave to start feeding it */ -#define REDIS_REPL_WAIT_BGSAVE_END 4 /* master waits bgsave to start bulk DB transmission */ -#define REDIS_REPL_SEND_BULK 5 /* master is sending the bulk DB */ -#define REDIS_REPL_ONLINE 6 /* bulk DB already transmitted, receive updates */ +#define REDIS_REPL_WAIT_BGSAVE_START 6 /* We need to produce a new RDB file. */ +#define REDIS_REPL_WAIT_BGSAVE_END 7 /* Waiting RDB file creation to finish. */ +#define REDIS_REPL_SEND_BULK 8 /* Sending RDB file to slave. */ +#define REDIS_REPL_ONLINE 9 /* RDB file transmitted, sending just updates. */ + +/* Synchronous read timeout - slave side */ +#define REDIS_REPL_SYNCIO_TIMEOUT 5 /* List related stuff */ #define REDIS_HEAD 0 diff --git a/src/replication.c b/src/replication.c index 71b2921f042..ea9c0104e78 100644 --- a/src/replication.c +++ b/src/replication.c @@ -803,23 +803,23 @@ void replicationCron(void) { if (!(server.cronloops % (server.repl_ping_slave_period * server.hz))) { listIter li; listNode *ln; + robj *ping_argv[1]; + /* First, send PING */ + ping_argv[0] = createStringObject("PING",4); + replicationFeedSlaves(server.slaves, server.slaveseldb, ping_argv, 1); + decrRefCount(ping_argv[0]); + + /* Second, send a newline to all the slaves in pre-synchronization stage, + * that is, slaves waiting for the master to create the RDB file. + * The newline will be ignored by the slave but will refresh the + * last-io timer preventing a timeout. */ listRewind(server.slaves,&li); while((ln = listNext(&li))) { redisClient *slave = ln->value; - /* Don't ping slaves that are in the middle of a bulk transfer - * with the master for first synchronization. */ - if (slave->replstate == REDIS_REPL_SEND_BULK) continue; - if (slave->replstate == REDIS_REPL_ONLINE) { - /* If the slave is online send a normal ping */ - addReplySds(slave,sdsnew("*1\r\n$4\r\nPING\r\n")); - } else { - /* Otherwise we are in the pre-synchronization stage. - * Just a newline will do the work of refreshing the - * connection last interaction time, and at the same time - * we'll be sure that being a single char there are no - * short-write problems. */ + if (slave->replstate == REDIS_REPL_WAIT_BGSAVE_START || + slave->replstate == REDIS_REPL_WAIT_BGSAVE_END) { if (write(slave->fd, "\n", 1) == -1) { /* Don't worry, it's just a ping. */ } From 4d8655cfd3ae656378b32ae56d58455be72cdc70 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 3 Nov 2012 12:15:29 +0100 Subject: [PATCH 0487/1752] Use the new unified protocol to send SELECT to slaves. SELECT was still transmitted to slaves using the inline protocol, that is conceived mostly for humans to type into telnet sessions, and is notably not understood by redis-cli --slave. Now the new protocol is used instead. --- src/redis.c | 8 +++++++- src/replication.c | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index a4654d3a327..b6ffb2a00d6 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1058,8 +1058,14 @@ void createSharedObjects(void) { shared.plus = createObject(REDIS_STRING,sdsnew("+")); for (j = 0; j < REDIS_SHARED_SELECT_CMDS; j++) { + char dictid_str[64]; + int dictid_len; + + dictid_len = ll2string(dictid_str,sizeof(dictid_str),j); shared.select[j] = createObject(REDIS_STRING, - sdscatprintf(sdsempty(),"select %d\r\n", j)); + sdscatprintf(sdsempty(), + "*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + dictid_len, dictid_str)); } shared.messagebulk = createStringObject("$7\r\nmessage\r\n",13); shared.pmessagebulk = createStringObject("$8\r\npmessage\r\n",14); diff --git a/src/replication.c b/src/replication.c index ea9c0104e78..872cf473528 100644 --- a/src/replication.c +++ b/src/replication.c @@ -61,8 +61,14 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { selectcmd = shared.select[dictid]; incrRefCount(selectcmd); } else { + char dictid_str[64]; + int dictid_len; + + dictid_len = ll2string(dictid_str,sizeof(dictid_str),dictid); selectcmd = createObject(REDIS_STRING, - sdscatprintf(sdsempty(),"select %d\r\n",dictid)); + sdscatprintf(sdsempty(), + "*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + dictid_len, dictid_str)); } addReply(slave,selectcmd); decrRefCount(selectcmd); From 700e5eb4fc1655fd2de27ecc721b9016fa9185b0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Jan 2013 18:33:16 +0100 Subject: [PATCH 0488/1752] PSYNC: work in progress, preview #2, rebased to unstable. --- redis.conf | 22 ++ src/config.c | 20 ++ src/db.c | 3 +- src/multi.c | 3 +- src/networking.c | 68 +++-- src/redis.c | 30 ++- src/redis.h | 34 ++- src/replication.c | 651 ++++++++++++++++++++++++++++++++++++++++++---- 8 files changed, 758 insertions(+), 73 deletions(-) diff --git a/redis.conf b/redis.conf index 36d18b7ad3c..2862a81fcb8 100644 --- a/redis.conf +++ b/redis.conf @@ -227,6 +227,28 @@ slave-read-only yes # be a good idea. repl-disable-tcp-nodelay no +# Set the replication backlog size. The backlog is a buffer that accumulates +# slave data when slaves are disconnected for some time, so that when a slave +# wants to reconnect again, often a full resync is not needed, but a partial +# resync is enough, just passing the portion of data the slave missed while +# disconnected. +# +# The biggest the replication backlog, the longer the time the slave can be +# disconnected and later be able to perform a partial resynchronization. +# +# The backlog is only allocated once there is at least a slave connected. +# +# repl-backlog-size 1mb + +# After a master has no longer connected slaves for some time, the backlog +# will be freed. The following option configures the amount of seconds that +# need to elapse, starting from the time the last slave disconnected, for +# the backlog buffer to be freed. +# +# A value of 0 means to never release the backlog. +# +# repl-backlog-ttl 3600 + # The slave priority is an integer number published by Redis in the INFO output. # It is used by Redis Sentinel in order to select a slave to promote into a # master if the master is no longer working correctly. diff --git a/src/config.c b/src/config.c index 4644530bd0c..f84ce191aeb 100644 --- a/src/config.c +++ b/src/config.c @@ -238,6 +238,18 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"repl-disable-tcp-nodelay") && argc==2) { if ((server.repl_disable_tcp_nodelay = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; + } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { + long long size = strtoll(argv[0],NULL,10); + if (size <= 0) { + err = "repl-backlog-size must be 1 or greater."; + goto loaderr; + } + resizeReplicationBacklog(size); + } else if (!strcasecmp(argv[0],"repl-backlog-ttl") && argc == 2) { + server.repl_backlog_time_limit = atoi(argv[1]); + if (server.repl_backlog_time_limit < 0) { + err = "repl-backlog-ttl can't be negative "; + goto loaderr; } } else if (!strcasecmp(argv[0],"masterauth") && argc == 2) { server.masterauth = zstrdup(argv[1]); @@ -712,6 +724,12 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"repl-timeout")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; server.repl_timeout = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"repl-backlog-size")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; + resizeReplicationBacklog(ll); + } else if (!strcasecmp(c->argv[2]->ptr,"repl-backlog-ttl")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.repl_backlog_time_limit = ll; } else if (!strcasecmp(c->argv[2]->ptr,"watchdog-period")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; if (ll) @@ -825,6 +843,8 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("databases",server.dbnum); config_get_numerical_field("repl-ping-slave-period",server.repl_ping_slave_period); config_get_numerical_field("repl-timeout",server.repl_timeout); + config_get_numerical_field("repl-backlog-size",server.repl_backlog_size); + config_get_numerical_field("repl-backlog-ttl",server.repl_backlog_time_limit); config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); config_get_numerical_field("slave-priority",server.slave_priority); diff --git a/src/db.c b/src/db.c index 11d4f0b1eec..f448e0330fb 100644 --- a/src/db.c +++ b/src/db.c @@ -500,8 +500,7 @@ void propagateExpire(redisDb *db, robj *key) { if (server.aof_state != REDIS_AOF_OFF) feedAppendOnlyFile(server.delCommand,db->id,argv,2); - if (listLength(server.slaves)) - replicationFeedSlaves(server.slaves,db->id,argv,2); + replicationFeedSlaves(server.slaves,db->id,argv,2); decrRefCount(argv[0]); decrRefCount(argv[1]); diff --git a/src/multi.c b/src/multi.c index dfac15c344f..ba8baf0a22b 100644 --- a/src/multi.c +++ b/src/multi.c @@ -108,8 +108,7 @@ void execCommandReplicateMulti(redisClient *c) { if (server.aof_state != REDIS_AOF_OFF) feedAppendOnlyFile(server.multiCommand,c->db->id,&multistring,1); - if (listLength(server.slaves)) - replicationFeedSlaves(server.slaves,c->db->id,&multistring,1); + replicationFeedSlaves(server.slaves,c->db->id,&multistring,1); decrRefCount(multistring); } diff --git a/src/networking.c b/src/networking.c index 66a2702c518..b909360115e 100644 --- a/src/networking.c +++ b/src/networking.c @@ -87,6 +87,7 @@ redisClient *createClient(int fd) { c->ctime = c->lastinteraction = server.unixtime; c->authenticated = 0; c->replstate = REDIS_REPL_NONE; + c->reploff = 0; c->slave_listening_port = 0; c->reply = listCreate(); c->reply_bytes = 0; @@ -595,12 +596,42 @@ void disconnectSlaves(void) { } } +/* This function is called when the slave lose the connection with the + * master into an unexpected way. */ +void replicationHandleMasterDisconnection(void) { + server.master = NULL; + server.repl_state = REDIS_REPL_CONNECT; + server.repl_down_since = server.unixtime; + /* We lost connection with our master, force our slaves to resync + * with us as well to load the new data set. + * + * If server.masterhost is NULL the user called SLAVEOF NO ONE so + * slave resync is not needed. */ + if (server.masterhost != NULL) disconnectSlaves(); +} + void freeClient(redisClient *c) { listNode *ln; /* If this is marked as current client unset it */ if (server.current_client == c) server.current_client = NULL; + /* If it is our master that's beging disconnected we should make sure + * to cache the state to try a partial resynchronization later. + * + * Note that before doing this we make sure that the client is not in + * some unexpected state, by checking its flags. */ + if (server.master && + (c->flags & REDIS_MASTER) && + !(c->flags & (REDIS_CLOSE_AFTER_REPLY| + REDIS_CLOSE_ASAP| + REDIS_BLOCKED| + REDIS_UNBLOCKED))) + { + replicationCacheMaster(c); + return; + } + /* Note that if the client we are freeing is blocked into a blocking * call, we have to set querybuf to NULL *before* to call * unblockClientWaitingData() to avoid processInputBuffer() will get @@ -620,16 +651,21 @@ void freeClient(redisClient *c) { pubsubUnsubscribeAllPatterns(c,0); dictRelease(c->pubsub_channels); listRelease(c->pubsub_patterns); - /* Obvious cleanup */ - aeDeleteFileEvent(server.el,c->fd,AE_READABLE); - aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + /* Close socket, unregister events, and remove list of replies and + * accumulated arguments. */ + if (c->fd != -1) { + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + close(c->fd); + } listRelease(c->reply); freeClientArgv(c); - close(c->fd); /* Remove from the list of clients */ - ln = listSearchKey(server.clients,c); - redisAssert(ln != NULL); - listDelNode(server.clients,ln); + if (c->fd != -1) { + ln = listSearchKey(server.clients,c); + redisAssert(ln != NULL); + listDelNode(server.clients,ln); + } /* When client was just unblocked because of a blocking operation, * remove it from the list with unblocked clients. */ if (c->flags & REDIS_UNBLOCKED) { @@ -647,20 +683,15 @@ void freeClient(redisClient *c) { ln = listSearchKey(l,c); redisAssert(ln != NULL); listDelNode(l,ln); + /* We need to remember the time when we started to have zero + * attached slaves, as after some time we'll free the replication + * backlog. */ + if (c->flags & REDIS_SLAVE && listLength(server.slaves) == 0) + server.repl_no_slaves_since = server.unixtime; } /* Case 2: we lost the connection with the master. */ - if (c->flags & REDIS_MASTER) { - server.master = NULL; - server.repl_state = REDIS_REPL_CONNECT; - server.repl_down_since = server.unixtime; - /* We lost connection with our master, force our slaves to resync - * with us as well to load the new data set. - * - * If server.masterhost is NULL the user called SLAVEOF NO ONE so - * slave resync is not needed. */ - if (server.masterhost != NULL) disconnectSlaves(); - } + if (c->flags & REDIS_MASTER) replicationHandleMasterDisconnection(); /* If this client was scheduled for async freeing we need to remove it * from the queue. */ @@ -1059,6 +1090,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { if (nread) { sdsIncrLen(c->querybuf,nread); c->lastinteraction = server.unixtime; + if (c->flags & REDIS_MASTER) c->reploff += nread; } else { server.current_client = NULL; return; diff --git a/src/redis.c b/src/redis.c index b6ffb2a00d6..d2bfe1b3b17 100644 --- a/src/redis.c +++ b/src/redis.c @@ -220,6 +220,7 @@ struct redisCommand redisCommandTable[] = { {"exec",execCommand,1,"sM",0,NULL,0,0,0,0,0}, {"discard",discardCommand,1,"rs",0,NULL,0,0,0,0,0}, {"sync",syncCommand,1,"ars",0,NULL,0,0,0,0,0}, + {"psync",syncCommand,3,"ars",0,NULL,0,0,0,0,0}, {"replconf",replconfCommand,-1,"ars",0,NULL,0,0,0,0,0}, {"flushdb",flushdbCommand,1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,1,"w",0,NULL,0,0,0,0,0}, @@ -1166,6 +1167,8 @@ void initServerConfig() { server.masterhost = NULL; server.masterport = 6379; server.master = NULL; + server.cached_master = NULL; + server.repl_master_initial_offset = -1; server.repl_state = REDIS_REPL_NONE; server.repl_syncio_timeout = REDIS_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = 1; @@ -1173,6 +1176,16 @@ void initServerConfig() { server.repl_down_since = time(NULL); server.repl_disable_tcp_nodelay = 0; server.slave_priority = REDIS_DEFAULT_SLAVE_PRIORITY; + server.master_repl_offset = 0; + + /* Replication partial resync backlog */ + server.repl_backlog = NULL; + server.repl_backlog_size = REDIS_DEFAULT_REPL_BACKLOG_SIZE; + server.repl_backlog_histlen = 0; + server.repl_backlog_idx = 0; + server.repl_backlog_off = 0; + server.repl_backlog_time_limit = REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT; + server.repl_no_slaves_since = time(NULL); /* Client output buffer limits */ server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].hard_limit_bytes = 0; @@ -1485,7 +1498,7 @@ void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, { if (server.aof_state != REDIS_AOF_OFF && flags & REDIS_PROPAGATE_AOF) feedAppendOnlyFile(cmd,dbid,argv,argc); - if (flags & REDIS_PROPAGATE_REPL && listLength(server.slaves)) + if (flags & REDIS_PROPAGATE_REPL) replicationFeedSlaves(server.slaves,dbid,argv,argc); } @@ -2088,13 +2101,15 @@ sds genRedisInfoString(char *section) { "master_link_status:%s\r\n" "master_last_io_seconds_ago:%d\r\n" "master_sync_in_progress:%d\r\n" + "slave_repl_offset:%lld\r\n" ,server.masterhost, server.masterport, (server.repl_state == REDIS_REPL_CONNECTED) ? "up" : "down", server.master ? ((int)(server.unixtime-server.master->lastinteraction)) : -1, - server.repl_state == REDIS_REPL_TRANSFER + server.repl_state == REDIS_REPL_TRANSFER, + server.master ? server.master->reploff : -1 ); if (server.repl_state == REDIS_REPL_TRANSFER) { @@ -2152,6 +2167,17 @@ sds genRedisInfoString(char *section) { slaveid++; } } + info = sdscatprintf(info, + "master_repl_offset:%lld\r\n" + "repl_backlog_active:%d\r\n" + "repl_backlog_size:%lld\r\n" + "repl_backlog_first_byte_offset:%lld\r\n" + "repl_backlog_histlen:%lld\r\n", + server.master_repl_offset, + server.repl_backlog != NULL, + server.repl_backlog_size, + server.repl_backlog_off, + server.repl_backlog_histlen); } /* CPU */ diff --git a/src/redis.h b/src/redis.h index 0134efef746..be27fd72e9f 100644 --- a/src/redis.h +++ b/src/redis.h @@ -93,6 +93,9 @@ #define REDIS_REPL_PING_SLAVE_PERIOD 10 #define REDIS_RUN_ID_SIZE 40 #define REDIS_OPS_SEC_SAMPLES 16 +#define REDIS_DEFAULT_REPL_BACKLOG_SIZE (1024*1024) /* 1mb */ +#define REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT (60*60) /* 1 hour */ +#define REDIS_REPL_BACKLOG_MIN_SIZE (1024*16) /* 16k */ /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ @@ -100,6 +103,7 @@ #define REDIS_REPLY_CHUNK_BYTES (16*1024) /* 16k output buffer */ #define REDIS_INLINE_MAX_SIZE (1024*64) /* Max size of inline reads */ #define REDIS_MBULK_BIG_ARG (1024*32) +#define REDIS_LONGSTR_SIZE 21 /* Bytes needed for long -> str */ /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ @@ -407,7 +411,8 @@ typedef struct redisClient { long bulklen; /* length of bulk argument in multi bulk request */ list *reply; unsigned long reply_bytes; /* Tot bytes of objects in reply list */ - int sentlen; + int sentlen; /* Amount of bytes already sent in the current + buffer or object being sent. */ time_t ctime; /* Client creation time */ time_t lastinteraction; /* time of the last interaction, used for timeout */ time_t obuf_soft_limit_reached_time; @@ -417,6 +422,8 @@ typedef struct redisClient { int repldbfd; /* replication DB file descriptor */ long repldboff; /* replication DB file offset */ off_t repldbsize; /* replication DB file size */ + long long reploff; /* replication offset if this is our master */ + char replrunid[REDIS_RUN_ID_SIZE+1]; /* master run id if this is a master */ int slave_listening_port; /* As configured with: SLAVECONF listening-port */ multiState mstate; /* MULTI/EXEC state */ blockingState bpop; /* blocking state */ @@ -533,7 +540,6 @@ struct redisServer { list *clients; /* List of active clients */ list *clients_to_close; /* Clients to close asynchronously */ list *slaves, *monitors; /* List of slaves and MONITORs */ - int slaveseldb; /* Last SELECTed DB in replication output */ redisClient *current_client; /* Current client, only used on crash report */ char neterr[ANET_ERR_LEN]; /* Error buffer for anet.c */ /* RDB / AOF loading information */ @@ -615,13 +621,27 @@ struct redisServer { int syslog_enabled; /* Is syslog enabled? */ char *syslog_ident; /* Syslog ident */ int syslog_facility; /* Syslog facility */ - /* Slave specific fields */ + /* Replication (master) */ + int slaveseldb; /* Last SELECTed DB in replication output */ + long long master_repl_offset; /* Global replication offset */ + int repl_ping_slave_period; /* Master pings the slave every N seconds */ + char *repl_backlog; /* Replication backlog for partial syncs */ + long long repl_backlog_size; /* Backlog circular buffer size */ + long long repl_backlog_histlen; /* Backlog actual data length */ + long long repl_backlog_idx; /* Backlog circular buffer current offset */ + long long repl_backlog_off; /* Replication offset of first byte in the + backlog buffer. */ + time_t repl_backlog_time_limit; /* Time without slaves after the backlog + gets released. */ + time_t repl_no_slaves_since; /* We have no slaves since that time. + Only valid if server.slaves len is 0. */ + /* Replication (slave) */ char *masterauth; /* AUTH with this password with master */ char *masterhost; /* Hostname of master */ int masterport; /* Port of master */ - int repl_ping_slave_period; /* Master pings the slave every N seconds */ int repl_timeout; /* Timeout after N seconds of master idle */ redisClient *master; /* Client that is master for this slave */ + redisClient *cached_master; /* Cached master to be reused for PSYNC. */ int repl_syncio_timeout; /* Timeout for synchronous I/O calls */ int repl_state; /* Replication status if the instance is a slave */ off_t repl_transfer_size; /* Size of RDB to read from master during sync. */ @@ -636,6 +656,8 @@ struct redisServer { time_t repl_down_since; /* Unix time at which link with master went down */ int repl_disable_tcp_nodelay; /* Disable TCP_NODELAY after SYNC? */ int slave_priority; /* Reported in INFO and used by Sentinel. */ + char repl_master_runid[REDIS_RUN_ID_SIZE+1]; /* Master run id for PSYNC. */ + long long repl_master_initial_offset; /* Master PSYNC offset. */ /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ unsigned long long maxmemory; /* Max number of memory bytes to use */ @@ -795,6 +817,7 @@ void exitFromChild(int retcode); redisClient *createClient(int fd); void closeTimedoutClients(void); void freeClient(redisClient *c); +void freeClientAsync(redisClient *c); void resetClient(redisClient *c); void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask); void addReply(redisClient *c, robj *obj); @@ -918,6 +941,9 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc); void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj **argv, int argc); void updateSlavesWaitingBgsave(int bgsaveerr); void replicationCron(void); +void replicationHandleMasterDisconnection(void); +void replicationCacheMaster(redisClient *c); +void resizeReplicationBacklog(long long newsize); /* Generic persistence functions */ void startLoading(FILE *fp); diff --git a/src/replication.c b/src/replication.c index 872cf473528..0458dc0eef4 100644 --- a/src/replication.c +++ b/src/replication.c @@ -37,13 +37,234 @@ #include #include +void replicationDiscardCachedMaster(void); +void replicationResurrectCachedMaster(int newfd); + /* ---------------------------------- MASTER -------------------------------- */ +void createReplicationBacklog(void) { + redisAssert(server.repl_backlog == NULL); + server.repl_backlog = zmalloc(server.repl_backlog_size); + server.repl_backlog_histlen = 0; + server.repl_backlog_idx = 0; + /* When a new backlog buffer is created, we increment the replication + * offset by one to make sure we'll not be able to PSYNC with any + * previous slave. This is needed because we avoid incrementing the + * master_repl_offset if no backlog exists nor slaves are attached. */ + server.master_repl_offset++; + + /* We don't have any data inside our buffer, but virtually the first + * byte we have is the next byte that will be generated for the + * replication stream. */ + server.repl_backlog_off = server.master_repl_offset+1; +} + +/* This function is called when the user modifies the replication backlog + * size at runtime. It is up to the function to both update the + * server.repl_backlog_size and to resize the buffer and setup it so that + * it contains the same data as the previous one (possibly less data, but + * the most recent bytes, or the same data and more free space in case the + * buffer is enlarged). */ +void resizeReplicationBacklog(long long newsize) { + if (newsize < REDIS_REPL_BACKLOG_MIN_SIZE) + newsize = REDIS_REPL_BACKLOG_MIN_SIZE; + if (server.repl_backlog_size == newsize) return; + + server.repl_backlog_size = newsize; + if (server.repl_backlog != NULL) { + /* What we actually do is to flush the old buffer and realloc a new + * empty one. It will refill with new data incrementally. + * The reason is that copying a few gigabytes adds latency and even + * worse often we need to alloc additional space before freeing the + * old buffer. */ + zfree(server.repl_backlog); + server.repl_backlog = zmalloc(server.repl_backlog_size); + server.repl_backlog_histlen = 0; + server.repl_backlog_idx = 0; + /* Next byte we have is... the next since the buffer is emtpy. */ + server.repl_backlog_off = server.master_repl_offset+1; + } +} + +void freeReplicationBacklog(void) { + redisAssert(server.repl_backlog != NULL); + zfree(server.repl_backlog); + server.repl_backlog = NULL; +} + +/* Add data to the replication backlog. + * This function also increments the global replication offset stored at + * server.master_repl_offset, because there is no case where we want to feed + * the backlog without incrementing the buffer. */ +void feedReplicationBacklog(void *ptr, size_t len) { + unsigned char *p = ptr; + + server.master_repl_offset += len; + + /* This is a circular buffer, so write as much data we can at every + * iteration and rewind the "idx" index if we reach the limit. */ + while(len) { + size_t thislen = server.repl_backlog_size - server.repl_backlog_idx; + if (thislen > len) thislen = len; + memcpy(server.repl_backlog+server.repl_backlog_idx,p,thislen); + server.repl_backlog_idx += thislen; + if (server.repl_backlog_idx == server.repl_backlog_size) + server.repl_backlog_idx = 0; + len -= thislen; + p += thislen; + server.repl_backlog_histlen += thislen; + } + if (server.repl_backlog_histlen > server.repl_backlog_size) + server.repl_backlog_histlen = server.repl_backlog_size; + /* Set the offset of the first byte we have in the backlog. */ + server.repl_backlog_off = server.master_repl_offset - + server.repl_backlog_histlen + 1; +} + +/* Wrapper for feedReplicationBacklog() that takes Redis string objects + * as input. */ +void feedReplicationBacklogWithObject(robj *o) { + char llstr[REDIS_LONGSTR_SIZE]; + void *p; + size_t len; + + if (o->encoding == REDIS_ENCODING_INT) { + len = ll2string(llstr,sizeof(llstr),(long)o->ptr); + p = llstr; + } else { + len = sdslen(o->ptr); + p = o->ptr; + } + feedReplicationBacklog(p,len); +} + +#define FEEDSLAVE_BUF_SIZE (1024*64) void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { listNode *ln; listIter li; - int j; + int j, i, len; + char buf[FEEDSLAVE_BUF_SIZE], *b = buf; + char llstr[REDIS_LONGSTR_SIZE]; + int buf_left = FEEDSLAVE_BUF_SIZE; + robj *o; + + /* If there aren't slaves, and there is no backlog buffer to populate, + * we can return ASAP. */ + if (server.repl_backlog == NULL && listLength(slaves) == 0) return; + + /* We can't have slaves attached and no backlog. */ + redisAssert(!(listLength(slaves) != 0 && server.repl_backlog == NULL)); + + /* What we do here is to try to write as much data as possible in a static + * buffer "buf" that is used to create an object that is later sent to all + * the slaves. This way we do the decoding only one time for most commands + * not containing big payloads. */ + + /* Create the SELECT command into the static buffer if needed. */ + if (server.slaveseldb != dictid) { + char *selectcmd; + size_t sclen; + + if (dictid >= 0 && dictid < REDIS_SHARED_SELECT_CMDS) { + selectcmd = shared.select[dictid]->ptr; + sclen = sdslen(selectcmd); + memcpy(b,selectcmd,sclen); + b += sclen; + buf_left -= sclen; + } else { + int dictid_len; + + dictid_len = ll2string(llstr,sizeof(llstr),dictid); + sclen = snprintf(b,buf_left,"*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + dictid_len, llstr); + b += sclen; + buf_left -= sclen; + } + } + server.slaveseldb = dictid; + + /* Add the multi bulk reply size to the static buffer, that is, the number + * of arguments of the command to send to every slave. */ + b[0] = '*'; + len = ll2string(b+1,REDIS_LONGSTR_SIZE,argc); + b += len+1; + buf_left -= len; + b[0] = '\r'; + b[1] = '\n'; + b += 2; + buf_left -= 2; + + /* Try to use the static buffer for as much arguments is possible. */ + for (j = 0; j < argc; j++) { + int objlen; + char *objptr; + + if (argv[j]->encoding != REDIS_ENCODING_RAW && + argv[j]->encoding != REDIS_ENCODING_INT) { + redisPanic("Unexpected encoding"); + } + if (argv[j]->encoding == REDIS_ENCODING_RAW) { + objlen = sdslen(argv[j]->ptr); + objptr = argv[j]->ptr; + } else { + objlen = ll2string(llstr,REDIS_LONGSTR_SIZE,(long)argv[j]->ptr); + objptr = llstr; + } + /* We need enough space for bulk reply encoding, newlines, and + * the data itself. */ + if (buf_left < objlen+REDIS_LONGSTR_SIZE+32) break; + + /* Write $...CRLF */ + b[0] = '$'; + len = ll2string(b+1,REDIS_LONGSTR_SIZE,objlen); + b += len+1; + buf_left -= len; + b[0] = '\r'; + b[1] = '\n'; + b += 2; + buf_left -= 2; + + /* And data plus CRLF */ + memcpy(b,objptr,objlen); + b += objlen; + buf_left -= objlen; + b[0] = '\r'; + b[1] = '\n'; + b += 2; + buf_left -= 2; + } + + /* Create an object with the static buffer content. */ + redisAssert(buf_left < FEEDSLAVE_BUF_SIZE); + o = createStringObject(buf,b-buf); + + /* If we have a backlog, populate it with data and increment + * the global replication offset. */ + if (server.repl_backlog) { + feedReplicationBacklogWithObject(o); + for (i = j; i < argc; i++) { + char aux[REDIS_LONGSTR_SIZE+3]; + long objlen = stringObjectLen(argv[i]); + + /* We need to feed the buffer with the object as a bulk reply + * not just as a plain string, so create the $..CRLF payload len + * ad add the final CRLF */ + aux[0] = '$'; + len = ll2string(aux+1,objlen,sizeof(aux)-1); + aux[len+1] = '\r'; + aux[len+2] = '\n'; + feedReplicationBacklog(aux,len+3); + feedReplicationBacklogWithObject(argv[j]); + feedReplicationBacklogWithObject(shared.crlf); + } + } + /* Write data to slaves. Here we do two things: + * 1) We write the "o" object that was created using the accumulated + * static buffer. + * 2) We write any additional argument of the command to replicate that + * was not written inside the static buffer for lack of space. + */ listRewind(slaves,&li); while((ln = listNext(&li))) { redisClient *slave = ln->value; @@ -54,29 +275,16 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* Feed slaves that are waiting for the initial SYNC (so these commands * are queued in the output buffer until the initial SYNC completes), * or are already in sync with the master. */ - if (server.slaveseldb != dictid) { - robj *selectcmd; - - if (dictid >= 0 && dictid < REDIS_SHARED_SELECT_CMDS) { - selectcmd = shared.select[dictid]; - incrRefCount(selectcmd); - } else { - char dictid_str[64]; - int dictid_len; - - dictid_len = ll2string(dictid_str,sizeof(dictid_str),dictid); - selectcmd = createObject(REDIS_STRING, - sdscatprintf(sdsempty(), - "*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", - dictid_len, dictid_str)); - } - addReply(slave,selectcmd); - decrRefCount(selectcmd); - } - addReplyMultiBulkLen(slave,argc); - for (j = 0; j < argc; j++) addReplyBulk(slave,argv[j]); + + /* First, trasmit the object created from the static buffer. */ + addReply(slave,o); + + /* Finally any additional argument that was not stored inside the + * static buffer if any (from j to argc). */ + for (i = j; i < argc; i++) + addReplyBulk(slave,argv[i]); } - server.slaveseldb = dictid; + decrRefCount(o); } void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj **argv, int argc) { @@ -120,6 +328,120 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** decrRefCount(cmdobj); } +/* Feed the slave 'c' with the replication backlog starting from the + * specified 'offset' up to the end of the backlog. */ +long long addReplyReplicationBacklog(redisClient *c, long long offset) { + long long j, skip, len; + +// printf("SLAVE REQUEST %lld\n", offset); + + if (server.repl_backlog_histlen == 0) { +// printf("NO HISTORY\n"); + return 0; + } + +// printf("FIRST BYTE WE HAVE %lld\n", server.repl_backlog_off); +// printf("HISTLEN %lld\n", server.repl_backlog_histlen); +// printf("IDX %lld\n", server.repl_backlog_idx); + + /* Compute the amount of bytes we need to discard. */ + skip = offset - server.repl_backlog_off; +// printf("SKIP %lld\n", skip); + + /* Point j to the oldest byte, that is actaully our + * server.repl_backlog_off byte. */ + j = (server.repl_backlog_idx + + (server.repl_backlog_size-server.repl_backlog_histlen)) % + server.repl_backlog_size; +// printf("J %lld\n", j); + + /* Discard the amount of data to seek to the specified 'offset'. */ + j = (j + skip) % server.repl_backlog_size; + + /* Feed slave with data. Since it is a circular buffer we have to + * split the reply in two parts if we are cross-boundary. */ + len = server.repl_backlog_histlen - skip; +// printf("LEN %lld\n", len); + while(len) { + long long thislen = + ((server.repl_backlog_size - j) < len) ? + (server.repl_backlog_size - j) : len; + +// printf("WRITE %lld\n", thislen); + addReplySds(c,sdsnewlen(server.repl_backlog + j, thislen)); + len -= thislen; + j = 0; + } + return server.repl_backlog_histlen - skip; +} + +/* This function handles the PSYNC command from the point of view of a + * master receiving a request for partial resynchronization. + * + * On success return REDIS_OK, otherwise REDIS_ERR is returned and we proceed + * with the usual full resync. */ +int masterTryPartialResynchronization(redisClient *c) { + long long psync_offset, psync_len; + char *master_runid = c->argv[1]->ptr; + + /* Is the runid of this master the same advertised by the wannabe slave + * via PSYNC? If runid changed this master is a different instance and + * there is no way to continue. */ + if (strcasecmp(master_runid, server.runid)) { + /* Run id "?" is used by slaves that want to force a full resync. */ + if (master_runid[0] != '?') { + redisLog(REDIS_NOTICE,"Partial resynchronization not accepted: " + "Runid mismatch (Client asked for '%s', I'm '%s')", + master_runid, server.runid); + } else { + redisLog(REDIS_NOTICE,"Full resync requested by slave."); + } + goto need_full_resync; + } + + /* We still have the data our slave is asking for? */ + if (getLongLongFromObjectOrReply(c,c->argv[2],&psync_offset,NULL) != + REDIS_OK) goto need_full_resync; + if (!server.repl_backlog || + psync_offset < server.repl_backlog_off || + psync_offset >= (server.repl_backlog_off + server.repl_backlog_size)) + { + redisLog(REDIS_NOTICE, + "Unable to partial resync with the slave for lack of backlog (Slave request was: %lld).", psync_offset); + goto need_full_resync; + } + + /* If we reached this point, we are able to perform a partial resync: + * 1) Set client state to make it a slave. + * 2) Inform the client we can continue with +CONTINUE + * 3) Send the backlog data (from the offset to the end) to the slave. */ + c->flags |= REDIS_SLAVE; + c->replstate = REDIS_REPL_ONLINE; + listAddNodeTail(server.slaves,c); + addReplySds(c,sdsnew("+CONTINUE\r\n")); + psync_len = addReplyReplicationBacklog(c,psync_offset); + redisLog(REDIS_NOTICE, + "Partial resynchronization request accepted. Sending %lld bytes of backlog starting from offset %lld.", psync_len, psync_offset); + /* Note that we don't need to set the selected DB at server.slaveseldb + * to -1 to force the master to emit SELECT, since the slave already + * has this state from the previous connection with the master. */ + + return REDIS_OK; /* The caller can return, no full resync needed. */ + +need_full_resync: + /* We need a full resync for some reason... notify the client. */ + psync_offset = server.master_repl_offset; + /* Add 1 to psync_offset if it the replication backlog does not exists + * as when it will be created later we'll increment the offset by one. */ + if (server.repl_backlog == NULL) psync_offset++; + addReplySds(c, + sdscatprintf(sdsempty(),"+FULLRESYNC %s %lld\r\n", + server.runid, + psync_offset)); + return REDIS_ERR; +} + +/* SYNC ad PSYNC command implemenation. */ void syncCommand(redisClient *c) { /* ignore SYNC if already slave or in monitor mode */ if (c->flags & REDIS_SLAVE) return; @@ -136,11 +458,24 @@ void syncCommand(redisClient *c) { * buffer registering the differences between the BGSAVE and the current * dataset, so that we can copy to other slaves if needed. */ if (listLength(c->reply) != 0) { - addReplyError(c,"SYNC is invalid with pending input"); + addReplyError(c,"SYNC and PSYNC are invalid with pending input"); return; } - redisLog(REDIS_NOTICE,"Slave ask for synchronization"); + redisLog(REDIS_NOTICE,"Slave asks for synchronization"); + + /* Try a partial resynchronization if this is a PSYNC command. + * If it fails, we continue with usual full resynchronization, however + * when this happens masterTryPartialResynchronization() already + * replied with: + * + * +FULLRESYNC + * + * So the slave knows the new runid and offset to try a PSYNC later + * if the connection with the master is lost. */ + if (!strcasecmp(c->argv[0]->ptr,"psync") && + masterTryPartialResynchronization(c) == REDIS_OK) return; + /* Here we need to check if there is a background saving operation * in progress, or if it is required to start one */ if (server.rdb_child_pid != -1) { @@ -185,6 +520,8 @@ void syncCommand(redisClient *c) { c->flags |= REDIS_SLAVE; server.slaveseldb = -1; /* Force to re-emit the SELECT command. */ listAddNodeTail(server.slaves,c); + if (listLength(server.slaves) == 1 && server.repl_backlog == NULL) + createReplicationBacklog(); return; } @@ -452,6 +789,9 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { server.master->flags |= REDIS_MASTER; server.master->authenticated = 1; server.repl_state = REDIS_REPL_CONNECTED; + server.master->reploff = server.repl_master_initial_offset; + memcpy(server.master->replrunid, server.repl_master_runid, + sizeof(server.repl_master_runid)); redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Finished with success"); /* Restart the AOF subsystem now that we finished the sync. This * will trigger an AOF rewrite, and when done will start appending @@ -481,8 +821,8 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { /* Send a synchronous command to the master. Used to send AUTH and * REPLCONF commands before starting the replication with SYNC. * - * On success NULL is returned. - * On error an sds string describing the error is returned. + * The command returns an sds string representing the result of the + * operation. On error the first byte is a "-". */ char *sendSynchronousCommand(int fd, ...) { va_list ap; @@ -504,7 +844,7 @@ char *sendSynchronousCommand(int fd, ...) { /* Transfer command to the server. */ if (syncWrite(fd,cmd,sdslen(cmd),server.repl_syncio_timeout*1000) == -1) { sdsfree(cmd); - return sdscatprintf(sdsempty(),"Writing to master: %s", + return sdscatprintf(sdsempty(),"-Writing to master: %s", strerror(errno)); } sdsfree(cmd); @@ -512,22 +852,123 @@ char *sendSynchronousCommand(int fd, ...) { /* Read the reply from the server. */ if (syncReadLine(fd,buf,sizeof(buf),server.repl_syncio_timeout*1000) == -1) { - return sdscatprintf(sdsempty(),"Reading from master: %s", + return sdscatprintf(sdsempty(),"-Reading from master: %s", strerror(errno)); } + return sdsnew(buf); +} + +/* Try a partial resynchronization with the master if we are about to reconnect. + * If there is no cached master structure, at least try to issue a + * "PSYNC ? -1" command in order to trigger a full resync using the PSYNC + * command in order to obtain the master run id and the master replication + * global offset. + * + * This function is designed to be called from syncWithMaster(), so the + * following assumptions are made: + * + * 1) We pass the function an already connected socket "fd". + * 2) This function does not close the file descriptor "fd". However in case + * of successful partial resynchronization, the function will reuse + * 'fd' as file descriptor of the server.master client structure. + * + * The function returns: + * + * PSYNC_CONTINUE: If the PSYNC command succeded and we can continue. + * PSYNC_FULLRESYNC: If PSYNC is supported but a full resync is needed. + * In this case the master run_id and global replication + * offset is saved. + * PSYNC_NOT_SUPPORTED: If the server does not understand PSYNC at all and + * the caller should fall back to SYNC. + */ + +#define PSYNC_CONTINUE 0 +#define PSYNC_FULLRESYNC 1 +#define PSYNC_NOT_SUPPORTED 2 +int slaveTryPartialResynchronization(int fd) { + char *psync_runid; + char psync_offset[32]; + sds reply; + + /* Initially set repl_master_initial_offset to -1 to mark the current + * master run_id and offset as not valid. Later if we'll be able to do + * a FULL resync using the PSYNC command we'll set the offset at the + * right value, so that this information will be propagated to the + * client structure representing the master into server.master. */ + server.repl_master_initial_offset = -1; + + if (server.cached_master) { + psync_runid = server.cached_master->replrunid; + snprintf(psync_offset,sizeof(psync_offset),"%lld", server.cached_master->reploff+1); + redisLog(REDIS_NOTICE,"Trying a partial resynchronization (request %s:%s).", psync_runid, psync_offset); + } else { + redisLog(REDIS_NOTICE,"Partial resynchronization not possible (no cached master)"); + psync_runid = "?"; + memcpy(psync_offset,"-1",3); + } + + /* Issue the PSYNC command */ + reply = sendSynchronousCommand(fd,"PSYNC",psync_runid,psync_offset,NULL); + + if (!strncmp(reply,"+FULLRESYNC",11)) { + char *runid, *offset; + + /* FULL RESYNC, parse the reply in order to extract the run id + * and the replication offset. */ + runid = strchr(reply,' '); + if (runid) { + runid++; + offset = strchr(runid,' '); + if (offset) offset++; + } + if (!runid || !offset || (offset-runid-1) != REDIS_RUN_ID_SIZE) { + redisLog(REDIS_WARNING, + "Master replied with wrong +FULLRESYNC syntax."); + } else { + memcpy(server.repl_master_runid, runid, offset-runid-1); + server.repl_master_runid[REDIS_RUN_ID_SIZE] = '\0'; + server.repl_master_initial_offset = strtoll(offset,NULL,10); + redisLog(REDIS_NOTICE,"Full resync from master: %s:%lld", + server.repl_master_runid, + server.repl_master_initial_offset); + } + /* We are going to full resync, discard the cached master structure. */ + replicationDiscardCachedMaster(); + sdsfree(reply); + return PSYNC_FULLRESYNC; + } - /* Check for errors from the server. */ - if (buf[0] != '+') { - return sdscatprintf(sdsempty(),"Error from master: %s", buf); + if (!strncmp(reply,"+CONTINUE",9)) { + /* Partial resync was accepted, set the replication state accordingly */ + redisLog(REDIS_NOTICE, + "Successful partial resynchronization with master."); + sdsfree(reply); + replicationResurrectCachedMaster(fd); + return PSYNC_CONTINUE; } - return NULL; /* No errors. */ + /* If we reach this point we receied either an error since the master does + * not understand PSYNC, or an unexpected reply from the master. + * Reply with PSYNC_NOT_SUPPORTED in both cases. */ + + if (strncmp(reply,"-ERR",4)) { + /* If it's not an error, log the unexpected event. */ + redisLog(REDIS_WARNING, + "Unexpected reply to PSYNC from master: %s", reply); + } else { + redisLog(REDIS_NOTICE, + "Master does not support PSYNC or is in " + "error state (reply: %s)", reply); + } + sdsfree(reply); + replicationDiscardCachedMaster(); + return PSYNC_NOT_SUPPORTED; } void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { char tmpfile[256], *err; int dfd, maxtries = 5; - int sockerr = 0; + int sockerr = 0, psync_result; socklen_t errlen = sizeof(sockerr); REDIS_NOTUSED(el); REDIS_NOTUSED(privdata); @@ -600,11 +1041,12 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { /* AUTH with the master if required. */ if(server.masterauth) { err = sendSynchronousCommand(fd,"AUTH",server.masterauth,NULL); - if (err) { + if (err[0] == '-') { redisLog(REDIS_WARNING,"Unable to AUTH to MASTER: %s",err); sdsfree(err); goto error; } + sdsfree(err); } /* Set the slave port, so that Master's INFO command can list the @@ -616,17 +1058,33 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { sdsfree(port); /* Ignore the error if any, not all the Redis versions support * REPLCONF listening-port. */ - if (err) { - redisLog(REDIS_NOTICE,"(non critical): Master does not understand REPLCONF listening-port: %s", err); - sdsfree(err); + if (err[0] == '-') { + redisLog(REDIS_NOTICE,"(Non critical) Master does not understand REPLCONF listening-port: %s", err); } + sdsfree(err); } - /* Issue the SYNC command */ - if (syncWrite(fd,"SYNC\r\n",6,server.repl_syncio_timeout*1000) == -1) { - redisLog(REDIS_WARNING,"I/O error writing to MASTER: %s", - strerror(errno)); - goto error; + /* Try a partial resynchonization. If we don't have a cached master + * slaveTryPartialResynchronization() will at least try to use PSYNC + * to start a full resynchronization so that we get the master run id + * and the global offset, to try a partial resync at the next + * reconnection attempt. */ + psync_result = slaveTryPartialResynchronization(fd); + if (psync_result == PSYNC_CONTINUE) { + redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Master accepted a Partial Resynchronization."); + return; + } + + /* Fall back to SYNC if needed. Otherwise psync_result == PSYNC_FULLRESYNC + * and the server.repl_master_runid and repl_master_initial_offset are + * already populated. */ + if (psync_result == PSYNC_NOT_SUPPORTED) { + redisLog(REDIS_NOTICE,"Retrying with SYNC..."); + if (syncWrite(fd,"SYNC\r\n",6,server.repl_syncio_timeout*1000) == -1) { + redisLog(REDIS_WARNING,"I/O error writing to MASTER: %s", + strerror(errno)); + goto error; + } } /* Prepare a suitable temp file for bulk transfer */ @@ -733,6 +1191,7 @@ void slaveofCommand(redisClient *c) { sdsfree(server.masterhost); server.masterhost = NULL; if (server.master) freeClient(server.master); + replicationDiscardCachedMaster(); cancelReplicationHandshake(); server.repl_state = REDIS_REPL_NONE; redisLog(REDIS_NOTICE,"MASTER MODE enabled (user request)"); @@ -757,6 +1216,7 @@ void slaveofCommand(redisClient *c) { server.masterport = port; if (server.master) freeClient(server.master); disconnectSlaves(); /* Force our slaves to resync with us as well. */ + replicationDiscardCachedMaster(); /* Don't try a PSYNC. */ cancelReplicationHandshake(); server.repl_state = REDIS_REPL_CONNECT; redisLog(REDIS_NOTICE,"SLAVE OF %s:%d enabled (user request)", @@ -765,6 +1225,92 @@ void slaveofCommand(redisClient *c) { addReply(c,shared.ok); } +/* ---------------------- MASTER CACHING FOR PSYNC -------------------------- */ + +/* In order to implement partial synchronization we need to be able to cache + * our master's client structure after a transient disconnection. + * It is cached into server.cached_master and flushed away using the following + * functions. */ + +/* This function is called by freeClient() in order to cache the master + * client structure instead of destryoing it. freeClient() will return + * ASAP after this function returns, so every action needed to avoid problems + * with a client that is really "suspended" has to be done by this function. + * + * The other functions that will deal with the cached master are: + * + * replicationDiscardCachedMaster() that will make sure to kill the client + * as for some reason we don't want to use it in the future. + * + * replicationResurrectCachedMaster() that is used after a successful PSYNC + * handshake in order to reactivate the cached master. + */ +void replicationCacheMaster(redisClient *c) { + listNode *ln; + + redisAssert(server.master != NULL && server.cached_master == NULL); + redisLog(REDIS_NOTICE,"Caching the disconnected master state."); + + /* Remove from the list of clients, we don't want this client to be + * listed by CLIENT LIST or processed in any way by batch operations. */ + ln = listSearchKey(server.clients,c); + redisAssert(ln != NULL); + listDelNode(server.clients,ln); + + /* Save the master. Server.master will be set to null later by + * replicationHandleMasterDisconnection(). */ + server.cached_master = server.master; + + /* Remove the event handlers and close the socket. We'll later reuse + * the socket of the new connection with the master during PSYNC. */ + aeDeleteFileEvent(server.el,c->fd,AE_READABLE); + aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); + close(c->fd); + + /* Set fd to -1 so that we can safely call freeClient(c) later. */ + c->fd = -1; + + /* Caching the master happens instead of the actual freeClient() call, + * so make sure to adjust the replication state. This function will + * also set server.master to NULL. */ + replicationHandleMasterDisconnection(); +} + +/* Free a cached master, called when there are no longer the conditions for + * a partial resync on reconnection. */ +void replicationDiscardCachedMaster(void) { + if (server.cached_master == NULL) return; + + redisLog(REDIS_NOTICE,"Discarding previously cached master state."); + server.cached_master->flags &= ~REDIS_MASTER; + freeClient(server.cached_master); + server.cached_master = NULL; +} + +/* Turn the cached master into the current master, using the file descriptor + * passed as argument as the socket for the new master. + * + * This funciton is called when successfully setup a partial resynchronization + * so the stream of data that we'll receive will start from were this + * master left. */ +void replicationResurrectCachedMaster(int newfd) { + server.master = server.cached_master; + server.cached_master = NULL; + server.master->fd = newfd; + server.master->flags &= ~(REDIS_CLOSE_AFTER_REPLY|REDIS_CLOSE_ASAP); + server.master->authenticated = 1; + server.master->lastinteraction = server.unixtime; + server.repl_state = REDIS_REPL_CONNECTED; + + /* Re-add to the list of clients. */ + listAddNodeTail(server.clients,server.master); + if (aeCreateFileEvent(server.el, newfd, AE_READABLE, + readQueryFromClient, server.master)) { + redisLog(REDIS_WARNING,"Error resurrecting the cached master, impossible to add the readable handler: %s", strerror(errno)); + freeClientAsync(server.master); /* Close ASAP. */ + } +} + /* --------------------------- REPLICATION CRON ---------------------------- */ void replicationCron(void) { @@ -816,8 +1362,8 @@ void replicationCron(void) { replicationFeedSlaves(server.slaves, server.slaveseldb, ping_argv, 1); decrRefCount(ping_argv[0]); - /* Second, send a newline to all the slaves in pre-synchronization stage, - * that is, slaves waiting for the master to create the RDB file. + /* Second, send a newline to all the slaves in pre-synchronization + * stage, that is, slaves waiting for the master to create the RDB file. * The newline will be ignored by the slave but will refresh the * last-io timer preventing a timeout. */ listRewind(server.slaves,&li); @@ -832,4 +1378,19 @@ void replicationCron(void) { } } } + + /* If we have no attached slaves and there is a replication backlog + * using memory, free it after some (configured) time. */ + if (listLength(server.slaves) == 0 && server.repl_backlog_time_limit && + server.repl_backlog) + { + time_t idle = server.unixtime - server.repl_no_slaves_since; + + if (idle > server.repl_backlog_time_limit) { + freeReplicationBacklog(); + redisLog(REDIS_NOTICE, + "Replication backlog freed after %d seconds " + "without connected slaves.", server.repl_backlog_time_limit); + } + } } From f5edd535d18dd13c02ef44ec9977ae0803957824 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Jan 2013 16:33:26 +0100 Subject: [PATCH 0489/1752] After SLAVEOF don't allow chained slaves to PSYNC. --- src/replication.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 0458dc0eef4..157cbcfe92b 100644 --- a/src/replication.c +++ b/src/replication.c @@ -87,7 +87,7 @@ void resizeReplicationBacklog(long long newsize) { } void freeReplicationBacklog(void) { - redisAssert(server.repl_backlog != NULL); + redisAssert(listLength(server.slaves) == 0); zfree(server.repl_backlog); server.repl_backlog = NULL; } @@ -1217,6 +1217,7 @@ void slaveofCommand(redisClient *c) { if (server.master) freeClient(server.master); disconnectSlaves(); /* Force our slaves to resync with us as well. */ replicationDiscardCachedMaster(); /* Don't try a PSYNC. */ + freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */ cancelReplicationHandshake(); server.repl_state = REDIS_REPL_CONNECT; redisLog(REDIS_NOTICE,"SLAVE OF %s:%d enabled (user request)", From 6726ea5b535dd294b6c2c7c37c3ae735322fa7f2 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 1 Feb 2013 13:01:01 +0100 Subject: [PATCH 0490/1752] Log the unexpected string received in place of the SYNC payload length. --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 157cbcfe92b..ec36000b2bc 100644 --- a/src/replication.c +++ b/src/replication.c @@ -722,7 +722,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { server.repl_transfer_lastio = server.unixtime; return; } else if (buf[0] != '$') { - redisLog(REDIS_WARNING,"Bad protocol from MASTER, the first byte is not '$', are you sure the host and port are right?"); + redisLog(REDIS_WARNING,"Bad protocol from MASTER, the first byte is not '$' (we received '%s'), are you sure the host and port are right?", buf); goto error; } server.repl_transfer_size = strtol(buf+1,NULL,10); From 8529dd218f05eee158771d654a0d72a446cc2530 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 1 Feb 2013 15:05:43 +0100 Subject: [PATCH 0491/1752] SYNC not allowed with pending data on the static output buffer. --- src/replication.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/replication.c b/src/replication.c index ec36000b2bc..4348dcfa79e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -457,8 +457,8 @@ void syncCommand(redisClient *c) { * the client about already issued commands. We need a fresh reply * buffer registering the differences between the BGSAVE and the current * dataset, so that we can copy to other slaves if needed. */ - if (listLength(c->reply) != 0) { - addReplyError(c,"SYNC and PSYNC are invalid with pending input"); + if (listLength(c->reply) != 0 || c->bufpos != 0) { + addReplyError(c,"SYNC and PSYNC are invalid with pending output"); return; } From 4fe7672dfc7b8f2f4cdf3e7412870f5f25cf9793 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 1 Feb 2013 15:16:56 +0100 Subject: [PATCH 0492/1752] PSYNC: don't use the client buffer to send +CONTINUE and +FULLRESYNC. When we are preparing an handshake with the slave we can't touch the connection buffer as it'll be used to accumulate differences between the sent RDB file and what arrives next from clients. So in short we can't use addReply() family functions. However we just use write(2) because we know that the socket buffer is empty, since a prerequisite for SYNC to work is that the static buffer and the output list are empty, and in general it is not expected that a client SYNCs after doing some heavy I/O with the master. However a short write connection is explicitly handled to avoid fragility (we simply close the connection and the slave will retry). --- src/replication.c | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/src/replication.c b/src/replication.c index 4348dcfa79e..1a96d5a6e3c 100644 --- a/src/replication.c +++ b/src/replication.c @@ -383,6 +383,8 @@ long long addReplyReplicationBacklog(redisClient *c, long long offset) { int masterTryPartialResynchronization(redisClient *c) { long long psync_offset, psync_len; char *master_runid = c->argv[1]->ptr; + char buf[128]; + int buflen; /* Is the runid of this master the same advertised by the wannabe slave * via PSYNC? If runid changed this master is a different instance and @@ -418,7 +420,14 @@ int masterTryPartialResynchronization(redisClient *c) { c->flags |= REDIS_SLAVE; c->replstate = REDIS_REPL_ONLINE; listAddNodeTail(server.slaves,c); - addReplySds(c,sdsnew("+CONTINUE\r\n")); + /* We can't use the connection buffers since they are used to accumulate + * new commands at this stage. But we are sure the socket send buffer is + * emtpy so this write will never fail actually. */ + buflen = snprintf(buf,sizeof(buf),"+CONTINUE\r\n"); + if (write(c->fd,buf,buflen) != buflen) { + freeClientAsync(c); + return REDIS_OK; + } psync_len = addReplyReplicationBacklog(c,psync_offset); redisLog(REDIS_NOTICE, "Partial resynchronization request accepted. Sending %lld bytes of backlog starting from offset %lld.", psync_len, psync_offset); @@ -434,10 +443,13 @@ int masterTryPartialResynchronization(redisClient *c) { /* Add 1 to psync_offset if it the replication backlog does not exists * as when it will be created later we'll increment the offset by one. */ if (server.repl_backlog == NULL) psync_offset++; - addReplySds(c, - sdscatprintf(sdsempty(),"+FULLRESYNC %s %lld\r\n", - server.runid, - psync_offset)); + /* Again, we can't use the connection buffers (see above). */ + buflen = snprintf(buf,sizeof(buf),"+FULLRESYNC %s %lld\r\n", + server.runid,psync_offset); + if (write(c->fd,buf,buflen) != buflen) { + freeClientAsync(c); + return REDIS_OK; + } return REDIS_ERR; } From 5a4ef3e5b6f3708fae9fd7542856fe06d8c9e9f4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 1 Feb 2013 15:28:02 +0100 Subject: [PATCH 0493/1752] Remove harmless warning in slaveTryPartialResynchronization(). --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 1a96d5a6e3c..40bc91c1a15 100644 --- a/src/replication.c +++ b/src/replication.c @@ -923,7 +923,7 @@ int slaveTryPartialResynchronization(int fd) { reply = sendSynchronousCommand(fd,"PSYNC",psync_runid,psync_offset,NULL); if (!strncmp(reply,"+FULLRESYNC",11)) { - char *runid, *offset; + char *runid = NULL, *offset = NULL; /* FULL RESYNC, parse the reply in order to extract the run id * and the replication offset. */ From 2b1398e87003394f9c02e73bed69a92bc7d57776 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 9 Feb 2013 16:33:57 +0100 Subject: [PATCH 0494/1752] PSYNC: debugging printf() calls are now logs at DEBUG level. --- src/replication.c | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/replication.c b/src/replication.c index 40bc91c1a15..c17e5aa79ed 100644 --- a/src/replication.c +++ b/src/replication.c @@ -333,27 +333,32 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** long long addReplyReplicationBacklog(redisClient *c, long long offset) { long long j, skip, len; -// printf("SLAVE REQUEST %lld\n", offset); + redisLog(REDIS_DEBUG, "[PSYNC] Slave request offset: %lld", offset); if (server.repl_backlog_histlen == 0) { -// printf("NO HISTORY\n"); + redisLog(REDIS_DEBUG, "[PSYNC] Backlog history len is zero"); return 0; } -// printf("FIRST BYTE WE HAVE %lld\n", server.repl_backlog_off); -// printf("HISTLEN %lld\n", server.repl_backlog_histlen); -// printf("IDX %lld\n", server.repl_backlog_idx); + redisLog(REDIS_DEBUG, "[PSYNC] Backlog size: %lld", + server.repl_backlog_size); + redisLog(REDIS_DEBUG, "[PSYNC] First byte: %lld", + server.repl_backlog_off); + redisLog(REDIS_DEBUG, "[PSYNC] History len: %lld", + server.repl_backlog_histlen); + redisLog(REDIS_DEBUG, "[PSYNC] Current index: %lld", + server.repl_backlog_idx); /* Compute the amount of bytes we need to discard. */ skip = offset - server.repl_backlog_off; -// printf("SKIP %lld\n", skip); + redisLog(REDIS_DEBUG, "[PSYNC] Skipping: %lld", skip); /* Point j to the oldest byte, that is actaully our * server.repl_backlog_off byte. */ j = (server.repl_backlog_idx + (server.repl_backlog_size-server.repl_backlog_histlen)) % server.repl_backlog_size; -// printf("J %lld\n", j); + redisLog(REDIS_DEBUG, "[PSYNC] Index of first byte: %lld", j); /* Discard the amount of data to seek to the specified 'offset'. */ j = (j + skip) % server.repl_backlog_size; @@ -361,13 +366,13 @@ long long addReplyReplicationBacklog(redisClient *c, long long offset) { /* Feed slave with data. Since it is a circular buffer we have to * split the reply in two parts if we are cross-boundary. */ len = server.repl_backlog_histlen - skip; -// printf("LEN %lld\n", len); + redisLog(REDIS_DEBUG, "[PSYNC] Reply total length: %lld", len); while(len) { long long thislen = ((server.repl_backlog_size - j) < len) ? (server.repl_backlog_size - j) : len; -// printf("WRITE %lld\n", thislen); + redisLog(REDIS_DEBUG, "[PSYNC] addReply() length: %lld", thislen); addReplySds(c,sdsnewlen(server.repl_backlog + j, thislen)); len -= thislen; j = 0; From 0ae1a6b1c3c99aee67395221b1258db9a3fc15aa Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Feb 2013 12:56:32 +0100 Subject: [PATCH 0495/1752] Add missing bracket removed for error after rebase of PSYNC. --- src/config.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/config.c b/src/config.c index f84ce191aeb..30e844313c5 100644 --- a/src/config.c +++ b/src/config.c @@ -238,6 +238,7 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"repl-disable-tcp-nodelay") && argc==2) { if ((server.repl_disable_tcp_nodelay = yesnotoi(argv[1])) == -1) { err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { long long size = strtoll(argv[0],NULL,10); if (size <= 0) { From 5a8cf340a4f610909dad940dd48a8c7504665c89 Mon Sep 17 00:00:00 2001 From: Steven Penny Date: Fri, 8 Feb 2013 12:11:06 -0600 Subject: [PATCH 0496/1752] Format to fit 80 columns This makes it readable on GitHub and editors without auto wrapping. --- MANIFESTO | 63 ++++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/MANIFESTO b/MANIFESTO index ad94fd8ea71..2b719057ee7 100644 --- a/MANIFESTO +++ b/MANIFESTO @@ -4,17 +4,64 @@ Redis Manifesto =============== -1 - A DSL for Abstract Data Types. Redis is a DSL (Domain Specific Language) that manipulates abstract data types and implemented as a TCP daemon. Commands manipulate a key space where keys are binary-safe strings and values are different kinds of abstract data types. Every data type represents an abstract version of a fundamental data structure. For instance Redis Lists are an abstract representation of linked lists. In Redis, the essence of a data type isn't just the kind of operations that the data types support, but also the space and time complexity of the data type and the operations performed upon it. +1 - A DSL for Abstract Data Types. Redis is a DSL (Domain Specific Language) + that manipulates abstract data types and implemented as a TCP daemon. + Commands manipulate a key space where keys are binary-safe strings and + values are different kinds of abstract data types. Every data type + represents an abstract version of a fundamental data structure. For instance + Redis Lists are an abstract representation of linked lists. In Redis, the + essence of a data type isn't just the kind of operations that the data types + support, but also the space and time complexity of the data type and the + operations performed upon it. -2 - Memory storage is #1. The Redis data set, composed of defined key-value pairs, is primarily stored in the computer's memory. The amount of memory in all kinds of computers, including entry-level servers, is increasing significantly each year. Memory is fast, and allows Redis to have very predictable performance. Datasets composed of 10k or 40 millions keys will perform similarly. Complex data types like Redis Sorted Sets are easy to implement and manipulate in memory with good performance, making Redis very simple. Redis will continue to explore alternative options (where data can be optionally stored on disk, say) but the main goal of the project remains the development of an in-memory database. +2 - Memory storage is #1. The Redis data set, composed of defined key-value + pairs, is primarily stored in the computer's memory. The amount of memory in + all kinds of computers, including entry-level servers, is increasing + significantly each year. Memory is fast, and allows Redis to have very + predictable performance. Datasets composed of 10k or 40 millions keys will + perform similarly. Complex data types like Redis Sorted Sets are easy to + implement and manipulate in memory with good performance, making Redis very + simple. Redis will continue to explore alternative options (where data can + be optionally stored on disk, say) but the main goal of the project remains + the development of an in-memory database. -3 - Fundamental data structures for a fundamental API. The Redis API is a direct consequence of fundamental data structures. APIs can often be arbitrary but not an API that resembles the nature of fundamental data structures. If we ever meet intelligent life forms from another part of the universe, they'll likely know, understand and recognize the same basic data structures we have in our computer science books. Redis will avoid intermediate layers in API, so that the complexity is obvious and more complex operations can be performed as the sum of the basic operations. +3 - Fundamental data structures for a fundamental API. The Redis API is a direct + consequence of fundamental data structures. APIs can often be arbitrary but + not an API that resembles the nature of fundamental data structures. If we + ever meet intelligent life forms from another part of the universe, they'll + likely know, understand and recognize the same basic data structures we have + in our computer science books. Redis will avoid intermediate layers in API, + so that the complexity is obvious and more complex operations can be + performed as the sum of the basic operations. -4 - Code is like a poem; it's not just something we write to reach some practical result. Sometimes people that are far from the Redis philosophy suggest using other code written by other authors (frequently in other languages) in order to implement something Redis currently lacks. But to us this is like if Shakespeare decided to end Enrico IV using the Paradiso from the Divina Commedia. Is using any external code a bad idea? Not at all. Like in "One Thousand and One Nights" smaller self contained stories are embedded in a bigger story, we'll be happy to use beautiful self contained libraries when needed. At the same time, when writing the Redis story we're trying to write smaller stories that will fit in to other code. +4 - Code is like a poem; it's not just something we write to reach some + practical result. Sometimes people that are far from the Redis philosophy + suggest using other code written by other authors (frequently in other + languages) in order to implement something Redis currently lacks. But to us + this is like if Shakespeare decided to end Enrico IV using the Paradiso from + the Divina Commedia. Is using any external code a bad idea? Not at all. Like + in "One Thousand and One Nights" smaller self contained stories are embedded + in a bigger story, we'll be happy to use beautiful self contained libraries + when needed. At the same time, when writing the Redis story we're trying to + write smaller stories that will fit in to other code. -5 - We're against complexity. We believe designing systems is a fight against complexity. We'll accept to fight the complexity when it's worthwhile but we'll try hard to recognize when a small feature is not worth 1000s of lines of code. Most of the time the best way to fight complexity is by not creating it at all. +5 - We're against complexity. We believe designing systems is a fight against + complexity. We'll accept to fight the complexity when it's worthwhile but + we'll try hard to recognize when a small feature is not worth 1000s of lines + of code. Most of the time the best way to fight complexity is by not + creating it at all. -6 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits naturally into a distributed version of Redis and 2) a more complex API that supports multi-key operations. Both are useful if used judiciously but there's no way to make the more complex multi-keys API distributed in an opaque way without violating our other principles. We don't want to provide the illusion of something that will work magically when actually it can't in all cases. Instead we'll provide commands to quickly migrate keys from one instance to another to perform multi-key operations and expose the tradeoffs to the user. - -7 - We optimize for joy. We believe writing code is a lot of hard work, and the only way it can be worth is by enjoying it. When there is no longer joy in writing code, the best thing to do is stop. To prevent this, we'll avoid taking paths that will make Redis less of a joy to develop. +6 - Two levels of API. The Redis API has two levels: 1) a subset of the API fits + naturally into a distributed version of Redis and 2) a more complex API that + supports multi-key operations. Both are useful if used judiciously but + there's no way to make the more complex multi-keys API distributed in an + opaque way without violating our other principles. We don't want to provide + the illusion of something that will work magically when actually it can't in + all cases. Instead we'll provide commands to quickly migrate keys from one + instance to another to perform multi-key operations and expose the tradeoffs + to the user. +7 - We optimize for joy. We believe writing code is a lot of hard work, and the + only way it can be worth is by enjoying it. When there is no longer joy in + writing code, the best thing to do is stop. To prevent this, we'll avoid + taking paths that will make Redis less of a joy to develop. From 985425279820ca87305e45b9b8216e993e27fa36 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Feb 2013 13:27:24 +0100 Subject: [PATCH 0497/1752] Test: avoid false positives in CLIENT SETNAME closed connection test. --- tests/unit/introspection.tcl | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/unit/introspection.tcl b/tests/unit/introspection.tcl index a6c2130f53e..54742bb0279 100644 --- a/tests/unit/introspection.tcl +++ b/tests/unit/introspection.tcl @@ -50,6 +50,10 @@ start_server {tags {"introspection"}} { assert_match {*foobar*} [r client list] $rd close # Now the client should no longer be listed - string match {*foobar*} [r client list] - } {0} + wait_for_condition 50 100 { + [string match {*foobar*} [r client list]] == 0 + } else { + fail "Client still listed in CLIENT LIST after SETNAME." + } + } } From 5fe2577a19e1ef5ddad08e3a7013685de6ad9950 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Feb 2013 16:25:41 +0100 Subject: [PATCH 0498/1752] Return a specific NOAUTH error if authentication is required. --- src/redis.c | 4 +++- src/redis.h | 2 +- tests/support/server.tcl | 4 ++-- tests/unit/auth.tcl | 2 +- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/redis.c b/src/redis.c index d2bfe1b3b17..e5e6405e495 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1050,6 +1050,8 @@ void createSharedObjects(void) { "-MISCONF Redis is configured to save RDB snapshots, but is currently not able to persist on disk. Commands that may modify the data set are disabled. Please check Redis logs for details about the error.\r\n")); shared.roslaveerr = createObject(REDIS_STRING,sdsnew( "-READONLY You can't write against a read only slave.\r\n")); + shared.noautherr = createObject(REDIS_STRING,sdsnew( + "-NOAUTH Authentication required.\r\n")); shared.oomerr = createObject(REDIS_STRING,sdsnew( "-OOM command not allowed when used memory > 'maxmemory'.\r\n")); shared.execaborterr = createObject(REDIS_STRING,sdsnew( @@ -1610,7 +1612,7 @@ int processCommand(redisClient *c) { if (server.requirepass && !c->authenticated && c->cmd->proc != authCommand) { flagTransaction(c); - addReplyError(c,"operation not permitted"); + addReply(c,shared.noautherr); return REDIS_OK; } diff --git a/src/redis.h b/src/redis.h index be27fd72e9f..100896b5090 100644 --- a/src/redis.h +++ b/src/redis.h @@ -448,7 +448,7 @@ struct sharedObjectsStruct { *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, - *masterdownerr, *roslaveerr, *execaborterr, + *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, *lpush, diff --git a/tests/support/server.tcl b/tests/support/server.tcl index acbfe73cff4..e10c350ff69 100644 --- a/tests/support/server.tcl +++ b/tests/support/server.tcl @@ -84,8 +84,8 @@ proc ping_server {host port} { puts $fd "PING\r\n" flush $fd set reply [gets $fd] - if {[string range $reply 0 4] eq {+PONG} || - [string range $reply 0 3] eq {-ERR}} { + if {[string range $reply 0 0] eq {+} || + [string range $reply 0 0] eq {-}} { set retval 1 } close $fd diff --git a/tests/unit/auth.tcl b/tests/unit/auth.tcl index bd4b8dca06f..15753e9e7a2 100644 --- a/tests/unit/auth.tcl +++ b/tests/unit/auth.tcl @@ -14,7 +14,7 @@ start_server {tags {"auth"} overrides {requirepass foobar}} { test {Arbitrary command gives an error when AUTH is required} { catch {r set foo bar} err set _ $err - } {ERR*operation not permitted} + } {NOAUTH*} test {AUTH succeeds when the right password is given} { r auth foobar From 31f0a6ec50a43f410d03facdbde847e035adeb28 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Feb 2013 15:24:25 +0100 Subject: [PATCH 0499/1752] Replication: added new stats counting full and partial resynchronizations. --- src/redis.c | 9 +++++++++ src/redis.h | 3 +++ src/replication.c | 19 +++++++++++++++++-- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index e5e6405e495..4575e271bdb 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1350,6 +1350,9 @@ void initServer() { server.stat_peak_memory = 0; server.stat_fork_time = 0; server.stat_rejected_conn = 0; + server.stat_sync_full = 0; + server.stat_sync_partial_ok = 0; + server.stat_sync_partial_err = 0; memset(server.ops_sec_samples,0,sizeof(server.ops_sec_samples)); server.ops_sec_idx = 0; server.ops_sec_last_sample_time = mstime(); @@ -2069,6 +2072,9 @@ sds genRedisInfoString(char *section) { "total_commands_processed:%lld\r\n" "instantaneous_ops_per_sec:%lld\r\n" "rejected_connections:%lld\r\n" + "sync_full:%lld\r\n" + "sync_partial_ok:%lld\r\n" + "sync_partial_err:%lld\r\n" "expired_keys:%lld\r\n" "evicted_keys:%lld\r\n" "keyspace_hits:%lld\r\n" @@ -2080,6 +2086,9 @@ sds genRedisInfoString(char *section) { server.stat_numcommands, getOperationsPerSecond(), server.stat_rejected_conn, + server.stat_sync_full, + server.stat_sync_partial_ok, + server.stat_sync_partial_err, server.stat_expiredkeys, server.stat_evictedkeys, server.stat_keyspace_hits, diff --git a/src/redis.h b/src/redis.h index 100896b5090..9a5e05a89b4 100644 --- a/src/redis.h +++ b/src/redis.h @@ -561,6 +561,9 @@ struct redisServer { size_t stat_peak_memory; /* Max used memory record */ long long stat_fork_time; /* Time needed to perform latest fork() */ long long stat_rejected_conn; /* Clients rejected because of maxclients */ + long long stat_sync_full; /* Number of full resyncs with slaves. */ + long long stat_sync_partial_ok; /* Number of accepted PSYNC requests. */ + long long stat_sync_partial_err;/* Number of unaccepted PSYNC requests. */ list *slowlog; /* SLOWLOG list of commands */ long long slowlog_entry_id; /* SLOWLOG current entry ID */ long long slowlog_log_slower_than; /* SLOWLOG time limit (to get logged) */ diff --git a/src/replication.c b/src/replication.c index c17e5aa79ed..36ee9b9126c 100644 --- a/src/replication.c +++ b/src/replication.c @@ -490,8 +490,23 @@ void syncCommand(redisClient *c) { * * So the slave knows the new runid and offset to try a PSYNC later * if the connection with the master is lost. */ - if (!strcasecmp(c->argv[0]->ptr,"psync") && - masterTryPartialResynchronization(c) == REDIS_OK) return; + if (!strcasecmp(c->argv[0]->ptr,"psync")) { + if (masterTryPartialResynchronization(c) == REDIS_OK) { + server.stat_sync_partial_ok++; + return; /* No full resync needed, return. */ + } else { + char *master_runid = c->argv[1]->ptr; + + /* Increment stats for failed PSYNCs, but only if the + * runid is not "?", as this is used by slaves to force a full + * resync on purpose when they are not albe to partially + * resync. */ + if (master_runid[0] != '?') server.stat_sync_partial_err++; + } + } + + /* Full resynchronization. */ + server.stat_sync_full++; /* Here we need to check if there is a background saving operation * in progress, or if it is required to start one */ From d96a66f5313b704409433649375e27609f30bacc Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Feb 2013 16:53:27 +0100 Subject: [PATCH 0500/1752] Replication: more strict error checking for master PING reply. --- src/replication.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/replication.c b/src/replication.c index 36ee9b9126c..409d5a71963 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1058,11 +1058,16 @@ void syncWithMaster(aeEventLoop *el, int fd, void *privdata, int mask) { goto error; } - /* We don't care about the reply, it can be +PONG or an error since - * the server requires AUTH. As long as it replies correctly, it's - * fine from our point of view. */ - if (buf[0] != '-' && buf[0] != '+') { - redisLog(REDIS_WARNING,"Unexpected reply to PING from master."); + /* We accept only two replies as valid, a positive +PONG reply + * (we just check for "+") or an authentication error. + * Note that older versions of Redis replied with "operation not + * permitted" instead of using a proper error code, so we test + * both. */ + if (buf[0] != '+' && + strncmp(buf,"-NOAUTH",7) != 0 && + strncmp(buf,"-ERR operation not permitted",28) != 0) + { + redisLog(REDIS_WARNING,"Error reply to PING from master: '%s'",buf); goto error; } else { redisLog(REDIS_NOTICE, From 3b4c0d80d559f4afae702fee3572553433b1953a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Feb 2013 13:38:20 +0100 Subject: [PATCH 0501/1752] Avoid compiler warning by casting to match printf() specifier. --- src/debug.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index d6df93c62e9..65174af9daa 100644 --- a/src/debug.c +++ b/src/debug.c @@ -704,7 +704,8 @@ int memtest_test_linux_anonymous_maps(void) { start_vect[regions] = start_addr; size_vect[regions] = size; - printf("Testing %lx %lu\n", start_vect[regions], size_vect[regions]); + printf("Testing %lx %lu\n", (unsigned long) start_vect[regions], + (unsigned long) size_vect[regions]); regions++; } From 964ee00de01e7fa80914f127f0aa78179aed75a5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Feb 2013 18:33:33 +0100 Subject: [PATCH 0502/1752] PSYNC: More robust handling of unexpected reply to PSYNC. --- src/replication.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/replication.c b/src/replication.c index 409d5a71963..52ff1a74779 100644 --- a/src/replication.c +++ b/src/replication.c @@ -956,6 +956,14 @@ int slaveTryPartialResynchronization(int fd) { if (!runid || !offset || (offset-runid-1) != REDIS_RUN_ID_SIZE) { redisLog(REDIS_WARNING, "Master replied with wrong +FULLRESYNC syntax."); + sdsfree(reply); + /* This is an unexpected condition, actually the +FULLRESYNC + * reply means that the master supports PSYNC, but the reply + * format seems wrong. To stay safe we blank the master + * runid to make sure next PSYNCs will fail, and return + * NOT_SUPPORTED to the caller to use SYNC instead. */ + memset(server.repl_master_runid,0,REDIS_RUN_ID_SIZE+1); + return PSYNC_NOT_SUPPORTED; } else { memcpy(server.repl_master_runid, runid, offset-runid-1); server.repl_master_runid[REDIS_RUN_ID_SIZE] = '\0'; From ff56772115da78860ff6477d0511e4aa54f223e1 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Feb 2013 18:43:40 +0100 Subject: [PATCH 0503/1752] PSYNC: another change to unexpected reply from PSYNC. --- src/replication.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index 52ff1a74779..3015886a511 100644 --- a/src/replication.c +++ b/src/replication.c @@ -956,14 +956,11 @@ int slaveTryPartialResynchronization(int fd) { if (!runid || !offset || (offset-runid-1) != REDIS_RUN_ID_SIZE) { redisLog(REDIS_WARNING, "Master replied with wrong +FULLRESYNC syntax."); - sdsfree(reply); /* This is an unexpected condition, actually the +FULLRESYNC * reply means that the master supports PSYNC, but the reply * format seems wrong. To stay safe we blank the master - * runid to make sure next PSYNCs will fail, and return - * NOT_SUPPORTED to the caller to use SYNC instead. */ + * runid to make sure next PSYNCs will fail. */ memset(server.repl_master_runid,0,REDIS_RUN_ID_SIZE+1); - return PSYNC_NOT_SUPPORTED; } else { memcpy(server.repl_master_runid, runid, offset-runid-1); server.repl_master_runid[REDIS_RUN_ID_SIZE] = '\0'; From fedcd51185aaebc755139db31adedd8e1b28f340 Mon Sep 17 00:00:00 2001 From: Arnaud Granal Date: Mon, 25 Feb 2013 19:25:48 +0200 Subject: [PATCH 0504/1752] Fix error "repl-backlog-size must be 1 or greater" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parameter repl-backlog-size is not parsed correctly in the configuration file. argv[0] is parsed instead of argv[1]. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 30e844313c5..20f3c413235 100644 --- a/src/config.c +++ b/src/config.c @@ -240,7 +240,7 @@ void loadServerConfigFromString(char *config) { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { - long long size = strtoll(argv[0],NULL,10); + long long size = strtoll(argv[1],NULL,10); if (size <= 0) { err = "repl-backlog-size must be 1 or greater."; goto loaderr; From ac3100bc3bd6ec50d44c0ae11d22eab4e9bb4152 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Feb 2013 11:52:12 +0100 Subject: [PATCH 0505/1752] Set process name in ps output to make operations safer. This commit allows Redis to set a process name that includes the binding address and the port number in order to make operations simpler. Redis children processes doing AOF rewrites or RDB saving change the name into redis-aof-rewrite and redis-rdb-bgsave respectively. This in general makes harder to kill the wrong process because of an error and makes simpler to identify saving children. This feature was suggested by Arnaud GRANAL in the Redis Google Group, Arnaud also pointed me to the setproctitle.c implementation includeed in this commit. This feature should work on all the Linux, OSX, and all the three major BSD systems. --- src/Makefile | 2 +- src/Makefile.dep | 1 + src/aof.c | 1 + src/config.h | 12 +++ src/rdb.c | 1 + src/redis.c | 8 ++ src/redis.h | 1 + src/setproctitle.c | 259 +++++++++++++++++++++++++++++++++++++++++++++ 8 files changed, 284 insertions(+), 1 deletion(-) create mode 100644 src/setproctitle.c diff --git a/src/Makefile b/src/Makefile index 0aa409de53b..9202fb4bd97 100644 --- a/src/Makefile +++ b/src/Makefile @@ -99,7 +99,7 @@ endif REDIS_SERVER_NAME= redis-server REDIS_SENTINEL_NAME= redis-sentinel -REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o +REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o REDIS_CLI_NAME= redis-cli REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o REDIS_BENCHMARK_NAME= redis-benchmark diff --git a/src/Makefile.dep b/src/Makefile.dep index 35025070c7e..784b272af38 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -82,6 +82,7 @@ sentinel.o: sentinel.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ziplist.h intset.h version.h util.h rdb.h rio.h \ ../deps/hiredis/hiredis.h ../deps/hiredis/async.h \ ../deps/hiredis/hiredis.h +setproctitle.o: setproctitle.c sha1.o: sha1.c sha1.h config.h slowlog.o: slowlog.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ diff --git a/src/aof.c b/src/aof.c index 2265222cc6d..c45265010e4 100644 --- a/src/aof.c +++ b/src/aof.c @@ -961,6 +961,7 @@ int rewriteAppendOnlyFileBackground(void) { /* Child */ if (server.ipfd > 0) close(server.ipfd); if (server.sofd > 0) close(server.sofd); + redisSetProcTitle("redis-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) { size_t private_dirty = zmalloc_get_private_dirty(); diff --git a/src/config.h b/src/config.h index 68e99b427d2..961503c6325 100644 --- a/src/config.h +++ b/src/config.h @@ -105,6 +105,18 @@ #define rdb_fsync_range(fd,off,size) fsync(fd) #endif +/* Check if we can use setproctitle(). + * BSD systems have support for it, we provide an implementation for + * Linux and osx. */ +#if (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__) +#define USE_SETPROCTITLE +#endif + +#if (defined __linux || defined __APPLE__) +#define USE_SETPROCTITLE +void setproctitle(const char *fmt, ...); +#endif + /* Byte ordering detection */ #include /* This will likely define BYTE_ORDER */ diff --git a/src/rdb.c b/src/rdb.c index a5e4c47af42..e327cd8cb12 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -730,6 +730,7 @@ int rdbSaveBackground(char *filename) { /* Child */ if (server.ipfd > 0) close(server.ipfd); if (server.sofd > 0) close(server.sofd); + redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename); if (retval == REDIS_OK) { size_t private_dirty = zmalloc_get_private_dirty(); diff --git a/src/redis.c b/src/redis.c index 4575e271bdb..4f12ecd7634 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2585,6 +2585,13 @@ void redisOutOfMemoryHandler(size_t allocation_size) { redisPanic("OOM"); } +void redisSetProcTitle(char *title) { + setproctitle("%s %s:%d", + title, + server.bindaddr ? server.bindaddr : "*", + server.port); +} + int main(int argc, char **argv) { struct timeval tv; @@ -2655,6 +2662,7 @@ int main(int argc, char **argv) { if (server.daemonize) daemonize(); initServer(); if (server.daemonize) createPidFile(); + redisSetProcTitle(argv[0]); redisAsciiArt(); if (!server.sentinel_mode) { diff --git a/src/redis.h b/src/redis.h index 9a5e05a89b4..d2a6a4fb90a 100644 --- a/src/redis.h +++ b/src/redis.h @@ -815,6 +815,7 @@ long long mstime(void); void getRandomHexChars(char *p, unsigned int len); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); void exitFromChild(int retcode); +void redisSetProcTitle(char *title); /* networking.c -- Networking and Client related operations */ redisClient *createClient(int fd); diff --git a/src/setproctitle.c b/src/setproctitle.c new file mode 100644 index 00000000000..399205c064d --- /dev/null +++ b/src/setproctitle.c @@ -0,0 +1,259 @@ +/* ========================================================================== + * setproctitle.c - Linux/Darwin setproctitle. + * -------------------------------------------------------------------------- + * Copyright (C) 2010 William Ahern + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sublicense, and/or sell copies of the Software, and to permit + * persons to whom the Software is furnished to do so, subject to the + * following conditions: + * + * The above copyright notice and this permission notice shall be included + * in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + * NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + * USE OR OTHER DEALINGS IN THE SOFTWARE. + * ========================================================================== + */ +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include /* NULL size_t */ +#include /* va_list va_start va_end */ +#include /* malloc(3) setenv(3) clearenv(3) setproctitle(3) getprogname(3) */ +#include /* vsnprintf(3) snprintf(3) */ + +#include /* strlen(3) strchr(3) strdup(3) memset(3) memcpy(3) */ + +#include /* errno program_invocation_name program_invocation_short_name */ + +#if !defined(HAVE_SETPROCTITLE) +#define HAVE_SETPROCTITLE (defined __NetBSD__ || defined __FreeBSD__ || defined __OpenBSD__) +#endif + + +#if !HAVE_SETPROCTITLE +#if (defined __linux || defined __APPLE__) && defined __GNUC__ + + +static struct { + /* original value */ + const char *arg0; + + /* title space available */ + char *base, *end; + + /* pointer to original nul character within base */ + char *nul; + + _Bool reset; + int error; +} SPT; + + +#ifndef SPT_MIN +#define SPT_MIN(a, b) (((a) < (b))? (a) : (b)) +#endif + +static inline size_t spt_min(size_t a, size_t b) { + return SPT_MIN(a, b); +} /* spt_min() */ + + +/* + * For discussion on the portability of the various methods, see + * http://lists.freebsd.org/pipermail/freebsd-stable/2008-June/043136.html + */ +static int spt_clearenv(void) { +#if __GLIBC__ + clearenv(); + + return 0; +#else + extern char **environ; + char **tmp; + + if (!(tmp = malloc(sizeof *tmp))) + return errno; + + tmp[0] = NULL; + environ = tmp; + + return 0; +#endif +} /* spt_clearenv() */ + + +static int spt_copyenv(char *oldenv[]) { + extern char **environ; + char *eq; + int i, error; + + if (environ != oldenv) + return 0; + + if ((error = spt_clearenv())) + goto error; + + for (i = 0; oldenv[i]; i++) { + if (!(eq = strchr(oldenv[i], '='))) + continue; + + *eq = '\0'; + error = (0 != setenv(oldenv[i], eq + 1, 1))? errno : 0; + *eq = '='; + + if (error) + goto error; + } + + return 0; +error: + environ = oldenv; + + return error; +} /* spt_copyenv() */ + + +static int spt_copyargs(int argc, char *argv[]) { + char *tmp; + int i; + + for (i = 1; i < argc || (i >= argc && argv[i]); i++) { + if (!argv[i]) + continue; + + if (!(tmp = strdup(argv[i]))) + return errno; + + argv[i] = tmp; + } + + return 0; +} /* spt_copyargs() */ + + +void spt_init(int argc, char *argv[], char *envp[]) __attribute__((constructor)); + +void spt_init(int argc, char *argv[], char *envp[]) { + char *base, *end, *nul, *tmp; + int i, error; + + if (!(base = argv[0])) + return; + + nul = &base[strlen(base)]; + end = nul + 1; + + for (i = 0; i < argc || (i >= argc && argv[i]); i++) { + if (!argv[i] || argv[i] < end) + continue; + + end = argv[i] + strlen(argv[i]) + 1; + } + + for (i = 0; envp[i]; i++) { + if (envp[i] < end) + continue; + + end = envp[i] + strlen(envp[i]) + 1; + } + + if (!(SPT.arg0 = strdup(argv[0]))) + goto syerr; + +#if __GLIBC__ + if (!(tmp = strdup(program_invocation_name))) + goto syerr; + + program_invocation_name = tmp; + + if (!(tmp = strdup(program_invocation_short_name))) + goto syerr; + + program_invocation_short_name = tmp; +#elif __APPLE__ + if (!(tmp = strdup(getprogname()))) + goto syerr; + + setprogname(tmp); +#endif + + + if ((error = spt_copyenv(envp))) + goto error; + + if ((error = spt_copyargs(argc, argv))) + goto error; + + SPT.nul = nul; + SPT.base = base; + SPT.end = end; + + return; +syerr: + error = errno; +error: + SPT.error = error; +} /* spt_init() */ + + +#ifndef SPT_MAXTITLE +#define SPT_MAXTITLE 255 +#endif + +void setproctitle(const char *fmt, ...) { + char buf[SPT_MAXTITLE + 1]; /* use buffer in case argv[0] is passed */ + va_list ap; + char *nul; + int len, error; + + if (!SPT.base) + return; + + if (fmt) { + va_start(ap, fmt); + len = vsnprintf(buf, sizeof buf, fmt, ap); + va_end(ap); + } else { + len = snprintf(buf, sizeof buf, "%s", SPT.arg0); + } + + if (len <= 0) + { error = errno; goto error; } + + if (!SPT.reset) { + memset(SPT.base, 0, SPT.end - SPT.base); + SPT.reset = 1; + } else { + memset(SPT.base, 0, spt_min(sizeof buf, SPT.end - SPT.base)); + } + + len = spt_min(len, spt_min(sizeof buf, SPT.end - SPT.base) - 1); + memcpy(SPT.base, buf, len); + nul = &SPT.base[len]; + + if (nul < SPT.nul) { + *SPT.nul = '.'; + } else if (nul == SPT.nul && &nul[1] < SPT.end) { + *SPT.nul = ' '; + *++nul = '\0'; + } + + return; +error: + SPT.error = error; +} /* setproctitle() */ + + +#endif /* __linux || __APPLE__ */ +#endif /* !HAVE_SETPROCTITLE */ From f69b0a0db8bebb27396805dcd7326b59ef44ca5a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Feb 2013 15:51:15 +0100 Subject: [PATCH 0506/1752] setproctitle.c: declar tmp as static so valgrind will not detect a leak. --- src/setproctitle.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/setproctitle.c b/src/setproctitle.c index 399205c064d..99bccf2c0c3 100644 --- a/src/setproctitle.c +++ b/src/setproctitle.c @@ -80,7 +80,7 @@ static int spt_clearenv(void) { return 0; #else extern char **environ; - char **tmp; + static char **tmp; if (!(tmp = malloc(sizeof *tmp))) return errno; From 6f96ac1c1ca040043f4249e5b230d080aec82ac7 Mon Sep 17 00:00:00 2001 From: Stam He Date: Wed, 27 Feb 2013 11:53:11 +0800 Subject: [PATCH 0507/1752] Set proctitle: avoid the use of __attribute__((constructor)). This cased a segfault in some Linux system and was GCC-specific. Commit modified by @antirez: 1) Stripped away the part to set the proc title via config for now. 2) Handle initialization of setproctitle only when the replacement is used. 3) Don't require GCC now that the attribute constructor is no longer used. --- src/config.h | 2 ++ src/redis.c | 7 +++++++ src/setproctitle.c | 10 ++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/config.h b/src/config.h index 961503c6325..9f2baaa1f0a 100644 --- a/src/config.h +++ b/src/config.h @@ -114,6 +114,8 @@ #if (defined __linux || defined __APPLE__) #define USE_SETPROCTITLE +#define INIT_SETPROCTITLE_REPLACEMENT +void spt_init(int argc, char *argv[]); void setproctitle(const char *fmt, ...); #endif diff --git a/src/redis.c b/src/redis.c index 4f12ecd7634..cc3b0841c79 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2586,16 +2586,23 @@ void redisOutOfMemoryHandler(size_t allocation_size) { } void redisSetProcTitle(char *title) { +#ifdef USE_SETPROCTITLE setproctitle("%s %s:%d", title, server.bindaddr ? server.bindaddr : "*", server.port); +#else + REDIS_NOTUSED(title); +#endif } int main(int argc, char **argv) { struct timeval tv; /* We need to initialize our libraries, and the server configuration. */ +#ifdef INIT_SETPROCTITLE_REPLACEMENT + spt_init(argc, argv); +#endif zmalloc_enable_thread_safeness(); zmalloc_set_oom_handler(redisOutOfMemoryHandler); srand(time(NULL)^getpid()); diff --git a/src/setproctitle.c b/src/setproctitle.c index 99bccf2c0c3..f44253e1697 100644 --- a/src/setproctitle.c +++ b/src/setproctitle.c @@ -2,6 +2,8 @@ * setproctitle.c - Linux/Darwin setproctitle. * -------------------------------------------------------------------------- * Copyright (C) 2010 William Ahern + * Copyright (C) 2013 Salvatore Sanfilippo + * Copyright (C) 2013 Stam He * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the @@ -42,8 +44,9 @@ #if !HAVE_SETPROCTITLE -#if (defined __linux || defined __APPLE__) && defined __GNUC__ +#if (defined __linux || defined __APPLE__) +extern char **environ; static struct { /* original value */ @@ -142,9 +145,8 @@ static int spt_copyargs(int argc, char *argv[]) { } /* spt_copyargs() */ -void spt_init(int argc, char *argv[], char *envp[]) __attribute__((constructor)); - -void spt_init(int argc, char *argv[], char *envp[]) { +void spt_init(int argc, char *argv[]) { + char **envp = environ; char *base, *end, *nul, *tmp; int i, error; From 438dfdd13dbfc3712be7df20e4a9c621919222ee Mon Sep 17 00:00:00 2001 From: Stam He Date: Wed, 27 Feb 2013 18:00:47 +0800 Subject: [PATCH 0508/1752] add a check for aeCreateTimeEvent 1) Add a check for aeCreateTimeEvent in function initServer. --- src/redis.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index cc3b0841c79..0bdb7102992 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1360,7 +1360,10 @@ void initServer() { server.unixtime = time(NULL); server.lastbgsave_status = REDIS_OK; server.stop_writes_on_bgsave_err = 1; - aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL); + if(aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR) { + redisPanic("create time event failed"); + exit(1); + } if (server.ipfd > 0 && aeCreateFileEvent(server.el,server.ipfd,AE_READABLE, acceptTcpHandler,NULL) == AE_ERR) redisPanic("Unrecoverable error creating server.ipfd file event."); if (server.sofd > 0 && aeCreateFileEvent(server.el,server.sofd,AE_READABLE, From 0d8d7f106ac092f67e1f4e51088301dfc9df19ce Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Feb 2013 12:00:11 +0100 Subject: [PATCH 0509/1752] Better panic message for failed time event creation. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 0bdb7102992..501d04a9ca5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1361,7 +1361,7 @@ void initServer() { server.lastbgsave_status = REDIS_OK; server.stop_writes_on_bgsave_err = 1; if(aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR) { - redisPanic("create time event failed"); + redisPanic("Can't create the serverCron time event."); exit(1); } if (server.ipfd > 0 && aeCreateFileEvent(server.el,server.ipfd,AE_READABLE, From d2a37badc2660bfe449a55d62b98a00532d4e299 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Feb 2013 12:27:15 +0100 Subject: [PATCH 0510/1752] Use GCC printf format attribute for redisLog(). This commit also fixes redisLog() statements producing warnings. --- src/aof.c | 2 +- src/debug.c | 155 +++++++++++++++++++++++----------------------- src/redis.c | 4 +- src/redis.h | 5 ++ src/replication.c | 7 ++- 5 files changed, 91 insertions(+), 82 deletions(-) diff --git a/src/aof.c b/src/aof.c index c45265010e4..c629bed4744 100644 --- a/src/aof.c +++ b/src/aof.c @@ -536,7 +536,7 @@ int loadAppendOnlyFile(char *filename) { /* Command lookup */ cmd = lookupCommand(argv[0]->ptr); if (!cmd) { - redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", argv[0]->ptr); + redisLog(REDIS_WARNING,"Unknown command '%s' reading the append only file", (char*)argv[0]->ptr); exit(1); } /* Run the command in the context of a fake client */ diff --git a/src/debug.c b/src/debug.c index 65174af9daa..b71279795c6 100644 --- a/src/debug.c +++ b/src/debug.c @@ -384,7 +384,7 @@ void redisLogObjectDebugInfo(robj *o) { redisLog(REDIS_WARNING,"Object encoding: %d", o->encoding); redisLog(REDIS_WARNING,"Object refcount: %d", o->refcount); if (o->type == REDIS_STRING && o->encoding == REDIS_ENCODING_RAW) { - redisLog(REDIS_WARNING,"Object raw string len: %d", sdslen(o->ptr)); + redisLog(REDIS_WARNING,"Object raw string len: %zu", sdslen(o->ptr)); if (sdslen(o->ptr) < 4096) redisLog(REDIS_WARNING,"Object raw string content: \"%s\"", (char*)o->ptr); } else if (o->type == REDIS_LIST) { @@ -467,10 +467,13 @@ static void *getMcontextEip(ucontext_t *uc) { void logStackContent(void **sp) { int i; for (i = 15; i >= 0; i--) { + unsigned long addr = (unsigned long) sp+i; + unsigned long val = (unsigned long) sp[i]; + if (sizeof(long) == 4) - redisLog(REDIS_WARNING, "(%08lx) -> %08lx", sp+i, sp[i]); + redisLog(REDIS_WARNING, "(%08lx) -> %08lx", addr, val); else - redisLog(REDIS_WARNING, "(%016lx) -> %016lx", sp+i, sp[i]); + redisLog(REDIS_WARNING, "(%016lx) -> %016lx", addr, val); } } @@ -488,27 +491,27 @@ void logRegisters(ucontext_t *uc) { "R8 :%016lx R9 :%016lx\nR10:%016lx R11:%016lx\n" "R12:%016lx R13:%016lx\nR14:%016lx R15:%016lx\n" "RIP:%016lx EFL:%016lx\nCS :%016lx FS:%016lx GS:%016lx", - uc->uc_mcontext->__ss.__rax, - uc->uc_mcontext->__ss.__rbx, - uc->uc_mcontext->__ss.__rcx, - uc->uc_mcontext->__ss.__rdx, - uc->uc_mcontext->__ss.__rdi, - uc->uc_mcontext->__ss.__rsi, - uc->uc_mcontext->__ss.__rbp, - uc->uc_mcontext->__ss.__rsp, - uc->uc_mcontext->__ss.__r8, - uc->uc_mcontext->__ss.__r9, - uc->uc_mcontext->__ss.__r10, - uc->uc_mcontext->__ss.__r11, - uc->uc_mcontext->__ss.__r12, - uc->uc_mcontext->__ss.__r13, - uc->uc_mcontext->__ss.__r14, - uc->uc_mcontext->__ss.__r15, - uc->uc_mcontext->__ss.__rip, - uc->uc_mcontext->__ss.__rflags, - uc->uc_mcontext->__ss.__cs, - uc->uc_mcontext->__ss.__fs, - uc->uc_mcontext->__ss.__gs + (unsigned long) uc->uc_mcontext->__ss.__rax, + (unsigned long) uc->uc_mcontext->__ss.__rbx, + (unsigned long) uc->uc_mcontext->__ss.__rcx, + (unsigned long) uc->uc_mcontext->__ss.__rdx, + (unsigned long) uc->uc_mcontext->__ss.__rdi, + (unsigned long) uc->uc_mcontext->__ss.__rsi, + (unsigned long) uc->uc_mcontext->__ss.__rbp, + (unsigned long) uc->uc_mcontext->__ss.__rsp, + (unsigned long) uc->uc_mcontext->__ss.__r8, + (unsigned long) uc->uc_mcontext->__ss.__r9, + (unsigned long) uc->uc_mcontext->__ss.__r10, + (unsigned long) uc->uc_mcontext->__ss.__r11, + (unsigned long) uc->uc_mcontext->__ss.__r12, + (unsigned long) uc->uc_mcontext->__ss.__r13, + (unsigned long) uc->uc_mcontext->__ss.__r14, + (unsigned long) uc->uc_mcontext->__ss.__r15, + (unsigned long) uc->uc_mcontext->__ss.__rip, + (unsigned long) uc->uc_mcontext->__ss.__rflags, + (unsigned long) uc->uc_mcontext->__ss.__cs, + (unsigned long) uc->uc_mcontext->__ss.__fs, + (unsigned long) uc->uc_mcontext->__ss.__gs ); logStackContent((void**)uc->uc_mcontext->__ss.__rsp); #else @@ -519,22 +522,22 @@ void logRegisters(ucontext_t *uc) { "EDI:%08lx ESI:%08lx EBP:%08lx ESP:%08lx\n" "SS:%08lx EFL:%08lx EIP:%08lx CS :%08lx\n" "DS:%08lx ES:%08lx FS :%08lx GS :%08lx", - uc->uc_mcontext->__ss.__eax, - uc->uc_mcontext->__ss.__ebx, - uc->uc_mcontext->__ss.__ecx, - uc->uc_mcontext->__ss.__edx, - uc->uc_mcontext->__ss.__edi, - uc->uc_mcontext->__ss.__esi, - uc->uc_mcontext->__ss.__ebp, - uc->uc_mcontext->__ss.__esp, - uc->uc_mcontext->__ss.__ss, - uc->uc_mcontext->__ss.__eflags, - uc->uc_mcontext->__ss.__eip, - uc->uc_mcontext->__ss.__cs, - uc->uc_mcontext->__ss.__ds, - uc->uc_mcontext->__ss.__es, - uc->uc_mcontext->__ss.__fs, - uc->uc_mcontext->__ss.__gs + (unsigned long) uc->uc_mcontext->__ss.__eax, + (unsigned long) uc->uc_mcontext->__ss.__ebx, + (unsigned long) uc->uc_mcontext->__ss.__ecx, + (unsigned long) uc->uc_mcontext->__ss.__edx, + (unsigned long) uc->uc_mcontext->__ss.__edi, + (unsigned long) uc->uc_mcontext->__ss.__esi, + (unsigned long) uc->uc_mcontext->__ss.__ebp, + (unsigned long) uc->uc_mcontext->__ss.__esp, + (unsigned long) uc->uc_mcontext->__ss.__ss, + (unsigned long) uc->uc_mcontext->__ss.__eflags, + (unsigned long) uc->uc_mcontext->__ss.__eip, + (unsigned long) uc->uc_mcontext->__ss.__cs, + (unsigned long) uc->uc_mcontext->__ss.__ds, + (unsigned long) uc->uc_mcontext->__ss.__es, + (unsigned long) uc->uc_mcontext->__ss.__fs, + (unsigned long) uc->uc_mcontext->__ss.__gs ); logStackContent((void**)uc->uc_mcontext->__ss.__esp); #endif @@ -548,22 +551,22 @@ void logRegisters(ucontext_t *uc) { "EDI:%08lx ESI:%08lx EBP:%08lx ESP:%08lx\n" "SS :%08lx EFL:%08lx EIP:%08lx CS:%08lx\n" "DS :%08lx ES :%08lx FS :%08lx GS:%08lx", - uc->uc_mcontext.gregs[11], - uc->uc_mcontext.gregs[8], - uc->uc_mcontext.gregs[10], - uc->uc_mcontext.gregs[9], - uc->uc_mcontext.gregs[4], - uc->uc_mcontext.gregs[5], - uc->uc_mcontext.gregs[6], - uc->uc_mcontext.gregs[7], - uc->uc_mcontext.gregs[18], - uc->uc_mcontext.gregs[17], - uc->uc_mcontext.gregs[14], - uc->uc_mcontext.gregs[15], - uc->uc_mcontext.gregs[3], - uc->uc_mcontext.gregs[2], - uc->uc_mcontext.gregs[1], - uc->uc_mcontext.gregs[0] + (unsigned long) uc->uc_mcontext.gregs[11], + (unsigned long) uc->uc_mcontext.gregs[8], + (unsigned long) uc->uc_mcontext.gregs[10], + (unsigned long) uc->uc_mcontext.gregs[9], + (unsigned long) uc->uc_mcontext.gregs[4], + (unsigned long) uc->uc_mcontext.gregs[5], + (unsigned long) uc->uc_mcontext.gregs[6], + (unsigned long) uc->uc_mcontext.gregs[7], + (unsigned long) uc->uc_mcontext.gregs[18], + (unsigned long) uc->uc_mcontext.gregs[17], + (unsigned long) uc->uc_mcontext.gregs[14], + (unsigned long) uc->uc_mcontext.gregs[15], + (unsigned long) uc->uc_mcontext.gregs[3], + (unsigned long) uc->uc_mcontext.gregs[2], + (unsigned long) uc->uc_mcontext.gregs[1], + (unsigned long) uc->uc_mcontext.gregs[0] ); logStackContent((void**)uc->uc_mcontext.gregs[7]); #elif defined(__X86_64__) || defined(__x86_64__) @@ -575,25 +578,25 @@ void logRegisters(ucontext_t *uc) { "R8 :%016lx R9 :%016lx\nR10:%016lx R11:%016lx\n" "R12:%016lx R13:%016lx\nR14:%016lx R15:%016lx\n" "RIP:%016lx EFL:%016lx\nCSGSFS:%016lx", - uc->uc_mcontext.gregs[13], - uc->uc_mcontext.gregs[11], - uc->uc_mcontext.gregs[14], - uc->uc_mcontext.gregs[12], - uc->uc_mcontext.gregs[8], - uc->uc_mcontext.gregs[9], - uc->uc_mcontext.gregs[10], - uc->uc_mcontext.gregs[15], - uc->uc_mcontext.gregs[0], - uc->uc_mcontext.gregs[1], - uc->uc_mcontext.gregs[2], - uc->uc_mcontext.gregs[3], - uc->uc_mcontext.gregs[4], - uc->uc_mcontext.gregs[5], - uc->uc_mcontext.gregs[6], - uc->uc_mcontext.gregs[7], - uc->uc_mcontext.gregs[16], - uc->uc_mcontext.gregs[17], - uc->uc_mcontext.gregs[18] + (unsigned long) uc->uc_mcontext.gregs[13], + (unsigned long) uc->uc_mcontext.gregs[11], + (unsigned long) uc->uc_mcontext.gregs[14], + (unsigned long) uc->uc_mcontext.gregs[12], + (unsigned long) uc->uc_mcontext.gregs[8], + (unsigned long) uc->uc_mcontext.gregs[9], + (unsigned long) uc->uc_mcontext.gregs[10], + (unsigned long) uc->uc_mcontext.gregs[15], + (unsigned long) uc->uc_mcontext.gregs[0], + (unsigned long) uc->uc_mcontext.gregs[1], + (unsigned long) uc->uc_mcontext.gregs[2], + (unsigned long) uc->uc_mcontext.gregs[3], + (unsigned long) uc->uc_mcontext.gregs[4], + (unsigned long) uc->uc_mcontext.gregs[5], + (unsigned long) uc->uc_mcontext.gregs[6], + (unsigned long) uc->uc_mcontext.gregs[7], + (unsigned long) uc->uc_mcontext.gregs[16], + (unsigned long) uc->uc_mcontext.gregs[17], + (unsigned long) uc->uc_mcontext.gregs[18] ); logStackContent((void**)uc->uc_mcontext.gregs[15]); #endif @@ -660,7 +663,7 @@ void logCurrentClient(void) { de = dictFind(cc->db->dict, key->ptr); if (de) { val = dictGetVal(de); - redisLog(REDIS_WARNING,"key '%s' found in DB containing the following object:", key->ptr); + redisLog(REDIS_WARNING,"key '%s' found in DB containing the following object:", (char*)key->ptr); redisLogObjectDebugInfo(val); } decrRefCount(key); diff --git a/src/redis.c b/src/redis.c index 501d04a9ca5..3ad35117cc5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -886,7 +886,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (!server.sentinel_mode) { run_with_period(5000) { redisLog(REDIS_VERBOSE, - "%d clients connected (%d slaves), %zu bytes in use", + "%lu clients connected (%lu slaves), %zu bytes in use", listLength(server.clients)-listLength(server.slaves), listLength(server.slaves), zmalloc_used_memory()); @@ -935,7 +935,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { if (server.dirty >= sp->changes && server.unixtime-server.lastsave > sp->seconds) { redisLog(REDIS_NOTICE,"%d changes in %d seconds. Saving...", - sp->changes, sp->seconds); + sp->changes, (int)sp->seconds); rdbSaveBackground(server.rdb_filename); break; } diff --git a/src/redis.h b/src/redis.h index d2a6a4fb90a..90ff59abed7 100644 --- a/src/redis.h +++ b/src/redis.h @@ -999,7 +999,12 @@ void call(redisClient *c, int flags); void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags); void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int target); int prepareForShutdown(); +#ifdef __GNUC__ +void redisLog(int level, const char *fmt, ...) + __attribute__((format(printf, 2, 3))); +#else void redisLog(int level, const char *fmt, ...); +#endif void redisLogRaw(int level, const char *msg); void redisLogFromHandler(int level, const char *msg); void usage(); diff --git a/src/replication.c b/src/replication.c index 3015886a511..6583f322a86 100644 --- a/src/replication.c +++ b/src/replication.c @@ -759,8 +759,8 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { } server.repl_transfer_size = strtol(buf+1,NULL,10); redisLog(REDIS_NOTICE, - "MASTER <-> SLAVE sync: receiving %ld bytes from master", - server.repl_transfer_size); + "MASTER <-> SLAVE sync: receiving %lld bytes from master", + (long long) server.repl_transfer_size); return; } @@ -1433,7 +1433,8 @@ void replicationCron(void) { freeReplicationBacklog(); redisLog(REDIS_NOTICE, "Replication backlog freed after %d seconds " - "without connected slaves.", server.repl_backlog_time_limit); + "without connected slaves.", + (int) server.repl_backlog_time_limit); } } } From 5e9e96999ffc38be8f54cb7ae767edb3a3887855 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Feb 2013 12:29:20 +0100 Subject: [PATCH 0511/1752] Remove too agressive/spamming log in rdb.c. --- src/rdb.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index e327cd8cb12..fbf9618dc16 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -773,7 +773,6 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { size_t len; unsigned int i; - redisLog(REDIS_DEBUG,"LOADING OBJECT %d (at %d)\n",rdbtype,rioTell(rdb)); if (rdbtype == REDIS_RDB_TYPE_STRING) { /* Read string value */ if ((o = rdbLoadEncodedStringObject(rdb)) == NULL) return NULL; From b148513fec583c23c7f7f064bdf9aaff32fbd60f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Feb 2013 12:33:27 +0100 Subject: [PATCH 0512/1752] Remove warning when printing redisBuildId(). --- src/redis.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 3ad35117cc5..c4b6b4c7cf1 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1920,7 +1920,7 @@ sds genRedisInfoString(char *section) { REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, - redisBuildId(), + (unsigned long long) redisBuildId(), mode, name.sysname, name.release, name.machine, server.arch_bits, @@ -2483,7 +2483,7 @@ void version() { atoi(redisGitDirty()) > 0, ZMALLOC_LIB, sizeof(long) == 4 ? 32 : 64, - redisBuildId()); + (unsigned long long) redisBuildId()); exit(0); } From 02cd545f7d77c03171955e3d6cd84a623371a301 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 25 Feb 2013 11:40:21 +0100 Subject: [PATCH 0513/1752] A comment in main() clarified. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index c4b6b4c7cf1..3c9c6203ea7 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2676,7 +2676,7 @@ int main(int argc, char **argv) { redisAsciiArt(); if (!server.sentinel_mode) { - /* Things only needed when not running in Sentinel mode. */ + /* Things not needed when running in Sentinel mode. */ redisLog(REDIS_WARNING,"Server started, Redis version " REDIS_VERSION); #ifdef __linux__ linuxOvercommitMemoryWarning(); From a74056f1e03a5f7c2b2c96b39372c6ce64ec327c Mon Sep 17 00:00:00 2001 From: Stam He Date: Sat, 2 Mar 2013 17:16:46 +0800 Subject: [PATCH 0514/1752] point 2 of slave-serve-stale-data miss '-' between 'stale' and 'data' --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 2862a81fcb8..bfe90493bcd 100644 --- a/redis.conf +++ b/redis.conf @@ -175,7 +175,7 @@ dir ./ # still reply to client requests, possibly with out of date data, or the # data set may just be empty if this is the first synchronization. # -# 2) if slave-serve-stale data is set to 'no' the slave will reply with +# 2) if slave-serve-stale-data is set to 'no' the slave will reply with # an error "SYNC with master in progress" to all the kind of commands # but to INFO and SLAVEOF. # From e618eb4fb379d8dfc92a5f8aa4e351b0244e3819 Mon Sep 17 00:00:00 2001 From: 0x20h Date: Sun, 24 Feb 2013 17:43:56 +0100 Subject: [PATCH 0515/1752] suppress external diff program when using git diff. --- src/mkreleasehdr.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mkreleasehdr.sh b/src/mkreleasehdr.sh index f3bbac0156f..1ae95886b4a 100755 --- a/src/mkreleasehdr.sh +++ b/src/mkreleasehdr.sh @@ -1,6 +1,6 @@ #!/bin/sh GIT_SHA1=`(git show-ref --head --hash=8 2> /dev/null || echo 00000000) | head -n1` -GIT_DIRTY=`git diff 2> /dev/null | wc -l` +GIT_DIRTY=`git diff --no-ext-diff 2> /dev/null | wc -l` BUILD_ID=`uname -n`"-"`date +%s` test -f release.h || touch release.h (cat release.h | grep SHA1 | grep $GIT_SHA1) && \ From df3b492045badd2751e6392e7fd10922a038c44d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 4 Mar 2013 11:14:32 +0100 Subject: [PATCH 0516/1752] redis-cli: use keepalive socket option. This should improve things in two ways: 1) Prevent timeouts caused by the execution of long commands. 2) Improve detection of real connection errors. This is mostly effective only on Linux because of the bogus default keepalive settings. In Linux we have OS-specific calls to set the keepalive interval to reasonable values. --- src/redis-cli.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index c182ac17d1b..a7795513137 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -56,6 +56,7 @@ #define OUTPUT_STANDARD 0 #define OUTPUT_RAW 1 #define OUTPUT_CSV 2 +#define REDIS_CLI_KEEPALIVE_INTERVAL 15 /* seconds */ static redisContext *context; static struct config { @@ -332,6 +333,12 @@ static int cliConnect(int force) { return REDIS_ERR; } + /* Set aggressive KEEP_ALIVE socket option in the Redis context socket + * in order to prevent timeouts caused by the execution of long + * commands. At the same time this improves the detection of real + * errors. */ + anetKeepAlive(NULL, context->fd, REDIS_CLI_KEEPALIVE_INTERVAL); + /* Do AUTH and select the right DB. */ if (cliAuth() != REDIS_OK) return REDIS_ERR; From aadb8aaa0823f207a16efddaddc2bb09d72eaee6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 11:50:38 +0100 Subject: [PATCH 0517/1752] Allow AUTH while loading the DB in memory. While Redis is loading the AOF or RDB file in memory only a subset of commands are allowed. This commit adds AUTH to this subset. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 3c9c6203ea7..c20837ed74c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -207,7 +207,7 @@ struct redisCommand redisCommandTable[] = { {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, - {"auth",authCommand,2,"rs",0,NULL,0,0,0,0,0}, + {"auth",authCommand,2,"rsl",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"r",0,NULL,0,0,0,0,0}, {"echo",echoCommand,2,"r",0,NULL,0,0,0,0,0}, {"save",saveCommand,1,"ars",0,NULL,0,0,0,0,0}, From 5aeb1f71a34e987949e29689b0eb2e0ca87f1d71 Mon Sep 17 00:00:00 2001 From: charsyam Date: Tue, 5 Feb 2013 21:47:14 -0800 Subject: [PATCH 0518/1752] Don't segfault on unbalanced quotes. --- src/config.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config.c b/src/config.c index 20f3c413235..7825ea84798 100644 --- a/src/config.c +++ b/src/config.c @@ -73,6 +73,10 @@ void loadServerConfigFromString(char *config) { /* Split into arguments */ argv = sdssplitargs(lines[i],&argc); + if (argv == NULL) { + err = "can't parse this line"; + goto loaderr; + } sdstolower(argv[0]); /* Execute config directives */ From a6f557657c47933b2a0c4ab4fc9a2bac81395af4 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 12:14:42 +0100 Subject: [PATCH 0519/1752] sdssplitargs(): now returns NULL only on error. An empty input string also resulted into the function returning NULL making it harder for the caller to distinguish between error and empty string without checking the original input string length. --- src/sds.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/sds.c b/src/sds.c index 85858a4f055..74fb274ac47 100644 --- a/src/sds.c +++ b/src/sds.c @@ -480,6 +480,11 @@ int hex_digit_to_int(char c) { * * Note that sdscatrepr() is able to convert back a string into * a quoted string in the same format sdssplitargs() is able to parse. + * + * The function returns the allocated tokens on success, even when the + * input string is empty, or NULL if the input contains unbalanced + * quotes or closed quotes followed by non space characters + * as in: "foo"bar or "foo' */ sds *sdssplitargs(const char *line, int *argc) { const char *p = line; @@ -576,6 +581,9 @@ sds *sdssplitargs(const char *line, int *argc) { (*argc)++; current = NULL; } else { + /* Even on empty input string returns something not NULL that + * can be freed by sdssplitargs_free. */ + if (vector == NULL) vector = zmalloc(sizeof(void*)); return vector; } } From a70e3ffb16c6a4f05e7b8c36478a4396abe60aad Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 12:19:38 +0100 Subject: [PATCH 0520/1752] sdssplitargs(): on error set *argc to 0. This makes programs not checking the return value for NULL much safer since with this change: 1) It is still possible to iterate the zero-length result without crashes. 2) sdssplitargs_free will work against NULL and 0 count. --- src/sds.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sds.c b/src/sds.c index 74fb274ac47..7d1227e71f1 100644 --- a/src/sds.c +++ b/src/sds.c @@ -593,6 +593,7 @@ sds *sdssplitargs(const char *line, int *argc) { sdsfree(vector[*argc]); zfree(vector); if (current) sdsfree(current); + *argc = 0; return NULL; } From 68532cf01e880ef94dbd484045b06b38282895c3 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 12:24:12 +0100 Subject: [PATCH 0521/1752] More specific error message in loadServerConfigFromString(). --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 7825ea84798..fd49e05dd6f 100644 --- a/src/config.c +++ b/src/config.c @@ -74,7 +74,7 @@ void loadServerConfigFromString(char *config) { /* Split into arguments */ argv = sdssplitargs(lines[i],&argc); if (argv == NULL) { - err = "can't parse this line"; + err = "Unbalanced quotes in configuration line"; goto loaderr; } sdstolower(argv[0]); From f72a3406a70e1f650c407e4ce23c38c076c66cfc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 12:34:43 +0100 Subject: [PATCH 0522/1752] sds.c: sdssplitargs_free() removed as it was a duplicate. --- src/sds.c | 16 +++++----------- src/sds.h | 1 - 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/sds.c b/src/sds.c index 7d1227e71f1..4cf700b9afd 100644 --- a/src/sds.c +++ b/src/sds.c @@ -475,8 +475,10 @@ int hex_digit_to_int(char c) { * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. The caller should sdsfree() all the returned - * strings and finally zfree() the array itself. + * of sds is returned. + * + * The caller should free the resulting array of sds strings with + * sdsfreesplitres(). * * Note that sdscatrepr() is able to convert back a string into * a quoted string in the same format sdssplitargs() is able to parse. @@ -581,8 +583,7 @@ sds *sdssplitargs(const char *line, int *argc) { (*argc)++; current = NULL; } else { - /* Even on empty input string returns something not NULL that - * can be freed by sdssplitargs_free. */ + /* Even on empty input string return something not NULL. */ if (vector == NULL) vector = zmalloc(sizeof(void*)); return vector; } @@ -597,13 +598,6 @@ sds *sdssplitargs(const char *line, int *argc) { return NULL; } -void sdssplitargs_free(sds *argv, int argc) { - int j; - - for (j = 0 ;j < argc; j++) sdsfree(argv[j]); - zfree(argv); -} - /* Modify the string substituting all the occurrences of the set of * characters specified in the 'from' string to the corresponding character * in the 'to' array. diff --git a/src/sds.h b/src/sds.h index e8d3065036f..c5a4f30a931 100644 --- a/src/sds.h +++ b/src/sds.h @@ -88,7 +88,6 @@ void sdstoupper(sds s); sds sdsfromlonglong(long long value); sds sdscatrepr(sds s, const char *p, size_t len); sds *sdssplitargs(const char *line, int *argc); -void sdssplitargs_free(sds *argv, int argc); sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); /* Low level functions exposed to the user API */ From 00a5e86e44722196a52c2315e1747c3367de931d Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 12:38:32 +0100 Subject: [PATCH 0523/1752] redis-cli: use sdsfreesplitres() instead of hand-coding it. --- src/redis-cli.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index a7795513137..d93c6cd5fd6 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -850,8 +850,7 @@ static void repl() { } } /* Free the argument vector */ - while(argc--) sdsfree(argv[argc]); - zfree(argv); + sdsfreesplitres(argv,argc); } /* linenoise() returns malloc-ed lines like readline() */ free(line); From 5bcfa2ebbc70b9b1c8bbecf3f3418902cb177734 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 12:40:48 +0100 Subject: [PATCH 0524/1752] Handle a non-impossible empty argv in loadServerConfigFromString(). Usually this does not happens since we trim for " \t\r\n", but if there are other chars that return true with isspace(), we may end with an empty argv. Better to handle the condition in an explicit way. --- src/config.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index fd49e05dd6f..9d2b3d7684d 100644 --- a/src/config.c +++ b/src/config.c @@ -68,7 +68,7 @@ void loadServerConfigFromString(char *config) { linenum = i+1; lines[i] = sdstrim(lines[i]," \t\r\n"); - /* Skip comments and blank lines*/ + /* Skip comments and blank lines */ if (lines[i][0] == '#' || lines[i][0] == '\0') continue; /* Split into arguments */ @@ -77,6 +77,12 @@ void loadServerConfigFromString(char *config) { err = "Unbalanced quotes in configuration line"; goto loaderr; } + + /* Skip this line if the resulting command vector is empty. */ + if (argc == 0) { + sdsfreesplitres(argv,argc); + return; + } sdstolower(argv[0]); /* Execute config directives */ From deccd104d8a499ad3afd86243b9b506c15a759a9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 15:47:16 +0100 Subject: [PATCH 0525/1752] Add a warning about command renaming in redis.conf. --- redis.conf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/redis.conf b/redis.conf index bfe90493bcd..f135463cd7a 100644 --- a/redis.conf +++ b/redis.conf @@ -294,6 +294,9 @@ slave-priority 100 # an empty string: # # rename-command CONFIG "" +# +# Please note that changing the name of commands that are logged into the +# AOF file or transmitted to slaves may cause problems. ################################### LIMITS #################################### From bc1b2e8f96bac798242a6f61720050d79e72c75e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Mar 2013 16:28:26 +0100 Subject: [PATCH 0526/1752] API to lookup commands with their original name. A new server.orig_commands table was added to the server structure, this contains a copy of the commant table unaffected by rename-command statements in redis.conf. A new API lookupCommandOrOriginal() was added that checks both tables, new first, old later, so that rewriteClientCommandVector() and friends can lookup commands with their new or original name in order to fix the client->cmd pointer when the argument vector is renamed. This fixes the segfault of issue #986, but does not fix a wider range of problems resulting from renaming commands that actually operate on data and are registered into the AOF file or propagated to slaves... That is command renaming should be handled with care. --- src/networking.c | 4 ++-- src/redis.c | 24 +++++++++++++++++++++--- src/redis.h | 4 +++- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/src/networking.c b/src/networking.c index b909360115e..694d953e4b4 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1291,7 +1291,7 @@ void rewriteClientCommandVector(redisClient *c, int argc, ...) { /* Replace argv and argc with our new versions. */ c->argv = argv; c->argc = argc; - c->cmd = lookupCommand(c->argv[0]->ptr); + c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr); redisAssertWithInfo(c,NULL,c->cmd != NULL); va_end(ap); } @@ -1309,7 +1309,7 @@ void rewriteClientCommandArgument(redisClient *c, int i, robj *newval) { /* If this is the command name make sure to fix c->cmd. */ if (i == 0) { - c->cmd = lookupCommand(c->argv[0]->ptr); + c->cmd = lookupCommandOrOriginal(c->argv[0]->ptr); redisAssertWithInfo(c,NULL,c->cmd != NULL); } } diff --git a/src/redis.c b/src/redis.c index c20837ed74c..9727028f041 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1210,6 +1210,7 @@ void initServerConfig() { * initial configuration, since command names may be changed via * redis.conf using the rename-command directive. */ server.commands = dictCreate(&commandTableDictType,NULL); + server.orig_commands = dictCreate(&commandTableDictType,NULL); populateCommandTable(); server.delCommand = lookupCommandByCString("del"); server.multiCommand = lookupCommandByCString("multi"); @@ -1403,7 +1404,7 @@ void populateCommandTable(void) { for (j = 0; j < numcommands; j++) { struct redisCommand *c = redisCommandTable+j; char *f = c->sflags; - int retval; + int retval1, retval2; while(*f != '\0') { switch(*f) { @@ -1424,8 +1425,11 @@ void populateCommandTable(void) { f++; } - retval = dictAdd(server.commands, sdsnew(c->name), c); - assert(retval == DICT_OK); + retval1 = dictAdd(server.commands, sdsnew(c->name), c); + /* Populate an additional dictionary that will be unaffected + * by rename-command statements in redis.conf. */ + retval2 = dictAdd(server.orig_commands, sdsnew(c->name), c); + redisAssert(retval1 == DICT_OK && retval2 == DICT_OK); } } @@ -1493,6 +1497,20 @@ struct redisCommand *lookupCommandByCString(char *s) { return cmd; } +/* Lookup the command in the current table, if not found also check in + * the original table containing the original command names unaffected by + * redis.conf rename-command statement. + * + * This is used by functions rewriting the argument vector such as + * rewriteClientCommandVector() in order to set client->cmd pointer + * correctly even if the command was renamed. */ +struct redisCommand *lookupCommandOrOriginal(sds name) { + struct redisCommand *cmd = dictFetchValue(server.commands, name); + + if (!cmd) cmd = dictFetchValue(server.orig_commands,name); + return cmd; +} + /* Propagate the specified command (in the context of the specified database id) * to AOF and Slaves. * diff --git a/src/redis.h b/src/redis.h index 90ff59abed7..37cef00c375 100644 --- a/src/redis.h +++ b/src/redis.h @@ -518,7 +518,8 @@ struct redisServer { /* General */ int hz; /* serverCron() calls frequency in hertz */ redisDb *db; - dict *commands; /* Command table hash table */ + dict *commands; /* Command table */ + dict *orig_commands; /* Command table before command renaming. */ aeEventLoop *el; unsigned lruclock:22; /* Clock incrementing every minute, for LRU */ unsigned lruclock_padding:10; @@ -995,6 +996,7 @@ int processCommand(redisClient *c); void setupSignalHandlers(void); struct redisCommand *lookupCommand(sds name); struct redisCommand *lookupCommandByCString(char *s); +struct redisCommand *lookupCommandOrOriginal(sds name); void call(redisClient *c, int flags); void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags); void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int target); From 6ae8fb477af65fd78228dbe5f0cf1c973cc48f24 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Tue, 5 Mar 2013 00:47:05 +0800 Subject: [PATCH 0527/1752] Removed useless "return" statements in pubsub.c (original commit message edited) --- src/pubsub.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index 3e58e3857eb..718f819ba27 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -278,7 +278,6 @@ void subscribeCommand(redisClient *c) { void unsubscribeCommand(redisClient *c) { if (c->argc == 1) { pubsubUnsubscribeAllChannels(c,1); - return; } else { int j; @@ -297,7 +296,6 @@ void psubscribeCommand(redisClient *c) { void punsubscribeCommand(redisClient *c) { if (c->argc == 1) { pubsubUnsubscribeAllPatterns(c,1); - return; } else { int j; From 21822fd0e9b46489c0246349faa5544e9a29083d Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Mar 2013 11:54:33 +0100 Subject: [PATCH 0528/1752] Move Redis databases background processing to databasesCron(). --- src/redis.c | 37 +++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/src/redis.c b/src/redis.c index 9727028f041..ac6958eab70 100644 --- a/src/redis.c +++ b/src/redis.c @@ -793,6 +793,27 @@ void clientsCron(void) { } } +/* This function handles 'background' operations we are required to do + * incrementally in Redis databases, such as active key expiring, resizing, + * rehashing. */ +void databasesCron(void) { + /* Expire a few keys per cycle, only if this is a master. + * On slaves we wait for DEL operations synthesized by the master + * in order to guarantee a strict consistency. */ + if (server.masterhost == NULL) activeExpireCycle(); + + /* We don't want to resize the hash tables while a background saving + * is in progress: the saving child is created using fork() that is + * implemented with a copy-on-write semantic in most modern systems, so + * if we resize the HT while there is the saving child at work actually + * a lot of memory movements in the parent will cause a lot of pages + * copied. */ + if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { + tryResizeHashTables(); + if (server.activerehashing) incrementallyRehash(); + } +} + /* This is our timer interrupt, called server.hz times per second. * Here is where we do a number of things that need to be done asynchronously. * For instance: @@ -871,17 +892,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { } } - /* We don't want to resize the hash tables while a background saving - * is in progress: the saving child is created using fork() that is - * implemented with a copy-on-write semantic in most modern systems, so - * if we resize the HT while there is the saving child at work actually - * a lot of memory movements in the parent will cause a lot of pages - * copied. */ - if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { - tryResizeHashTables(); - if (server.activerehashing) incrementallyRehash(); - } - /* Show information about connected clients */ if (!server.sentinel_mode) { run_with_period(5000) { @@ -962,11 +972,6 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * cron function is called. */ if (server.aof_flush_postponed_start) flushAppendOnlyFile(0); - /* Expire a few keys per cycle, only if this is a master. - * On slaves we wait for DEL operations synthesized by the master - * in order to guarantee a strict consistency. */ - if (server.masterhost == NULL) activeExpireCycle(); - /* Close clients that need to be closed asynchronous */ freeClientsInAsyncFreeQueue(); From aa7f74c7e9df928b55ce015391c2cd8021fe0849 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Mar 2013 13:59:50 +0100 Subject: [PATCH 0529/1752] Actually call databasesCron() inside serverCron(). --- src/redis.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/redis.c b/src/redis.c index ac6958eab70..823ea3a0d5c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -906,6 +906,9 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { /* We need to do a few operations on clients asynchronously. */ clientsCron(); + /* Handle background operations on Redis databases. */ + databasesCron(); + /* Start a scheduled AOF rewrite if this was requested by the user while * a BGSAVE was in progress. */ if (server.rdb_child_pid == -1 && server.aof_child_pid == -1 && From a4bb4b29fb64d827a996a9073220696dc05c99cb Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Mar 2013 14:01:12 +0100 Subject: [PATCH 0530/1752] Only resize/rehash a few databases per cron iteration. This is the first step to lower the CPU usage when many databases are configured. The other is to also process a limited number of DBs per call in the active expire cycle. --- src/redis.c | 85 +++++++++++++++++++++++++++++++---------------------- src/redis.h | 1 + 2 files changed, 51 insertions(+), 35 deletions(-) diff --git a/src/redis.c b/src/redis.c index 823ea3a0d5c..654e5c50486 100644 --- a/src/redis.c +++ b/src/redis.c @@ -571,36 +571,32 @@ int htNeedsResize(dict *dict) { /* If the percentage of used slots in the HT reaches REDIS_HT_MINFILL * we resize the hash table to save memory */ -void tryResizeHashTables(void) { - int j; - - for (j = 0; j < server.dbnum; j++) { - if (htNeedsResize(server.db[j].dict)) - dictResize(server.db[j].dict); - if (htNeedsResize(server.db[j].expires)) - dictResize(server.db[j].expires); - } +void tryResizeHashTables(int dbid) { + if (htNeedsResize(server.db[dbid].dict)) + dictResize(server.db[dbid].dict); + if (htNeedsResize(server.db[dbid].expires)) + dictResize(server.db[dbid].expires); } /* Our hash table implementation performs rehashing incrementally while * we write/read from the hash table. Still if the server is idle, the hash * table will use two tables for a long time. So we try to use 1 millisecond - * of CPU time at every serverCron() loop in order to rehash some key. */ -void incrementallyRehash(void) { - int j; - - for (j = 0; j < server.dbnum; j++) { - /* Keys dictionary */ - if (dictIsRehashing(server.db[j].dict)) { - dictRehashMilliseconds(server.db[j].dict,1); - break; /* already used our millisecond for this loop... */ - } - /* Expires */ - if (dictIsRehashing(server.db[j].expires)) { - dictRehashMilliseconds(server.db[j].expires,1); - break; /* already used our millisecond for this loop... */ - } + * of CPU time at every call of this function to perform some rehahsing. + * + * The function returns 1 if some rehashing was performed, otherwise 0 + * is returned. */ +int incrementallyRehash(int dbid) { + /* Keys dictionary */ + if (dictIsRehashing(server.db[dbid].dict)) { + dictRehashMilliseconds(server.db[dbid].dict,1); + return 1; /* already used our millisecond for this loop... */ + } + /* Expires */ + if (dictIsRehashing(server.db[dbid].expires)) { + dictRehashMilliseconds(server.db[dbid].expires,1); + return 1; /* already used our millisecond for this loop... */ } + return 0; } /* This function is called once a background process of some kind terminates, @@ -797,20 +793,39 @@ void clientsCron(void) { * incrementally in Redis databases, such as active key expiring, resizing, * rehashing. */ void databasesCron(void) { - /* Expire a few keys per cycle, only if this is a master. - * On slaves we wait for DEL operations synthesized by the master - * in order to guarantee a strict consistency. */ + /* Expire keys by random sampling. Not required for slaves + * as master will synthesize DELs for us. */ if (server.masterhost == NULL) activeExpireCycle(); - /* We don't want to resize the hash tables while a background saving - * is in progress: the saving child is created using fork() that is - * implemented with a copy-on-write semantic in most modern systems, so - * if we resize the HT while there is the saving child at work actually - * a lot of memory movements in the parent will cause a lot of pages - * copied. */ + /* Perform hash tables rehashing if needed, but only if there are no + * other processes saving the DB on disk. Otherwise rehashing is bad + * as will cause a lot of copy-on-write of memory pages. */ if (server.rdb_child_pid == -1 && server.aof_child_pid == -1) { - tryResizeHashTables(); - if (server.activerehashing) incrementallyRehash(); + /* We use global counters so if we stop the computation at a given + * DB we'll be able to start from the successive in the next + * cron loop iteration. */ + static int resize_db = 0; + static int rehash_db = 0; + int j; + + /* Resize */ + for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { + tryResizeHashTables(resize_db % server.dbnum); + resize_db++; + } + + /* Rehash */ + if (server.activerehashing) { + for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { + int work_done = incrementallyRehash(rehash_db % server.dbnum); + rehash_db++; + if (work_done) { + /* If the function did some work, stop here, we'll do + * more at the next cron loop. */ + break; + } + } + } } } diff --git a/src/redis.h b/src/redis.h index 37cef00c375..f7aca924d1e 100644 --- a/src/redis.h +++ b/src/redis.h @@ -76,6 +76,7 @@ #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_EXPIRELOOKUPS_PER_CRON 10 /* lookup 10 expires per loop */ #define REDIS_EXPIRELOOKUPS_TIME_PERC 25 /* CPU max % for keys collection */ +#define REDIS_DBCRON_DBS_PER_SEC 16 #define REDIS_MAX_WRITE_PER_EVENT (1024*64) #define REDIS_SHARED_SELECT_CMDS 10 #define REDIS_SHARED_INTEGERS 10000 From 8abfe5fe8b17bdd388a7bd1bdf52563cd8b68535 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Mar 2013 17:41:20 +0100 Subject: [PATCH 0531/1752] Use unsigned integers for DB ids, for defined wrap-to-zero. --- src/redis.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 654e5c50486..aaba09220c0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -804,9 +804,9 @@ void databasesCron(void) { /* We use global counters so if we stop the computation at a given * DB we'll be able to start from the successive in the next * cron loop iteration. */ - static int resize_db = 0; - static int rehash_db = 0; - int j; + static unsigned int resize_db = 0; + static unsigned int rehash_db = 0; + unsigned int j; /* Resize */ for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { From 1a73260dce3840c1c6aa3f2cff39f508847536f5 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 8 Mar 2013 17:48:58 +0100 Subject: [PATCH 0532/1752] activeExpireCycle(): process only a small number of DBs per iteration. This small number of DBs is set to 16 so actually in the default configuraiton Redis should behave exactly like in the past. However the difference is that when the user configures a very large number of DBs we don't do an O(N) operation, consuming a non trivial amount of CPU per serverCron() iteration. --- src/redis.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index aaba09220c0..0f378af798e 100644 --- a/src/redis.c +++ b/src/redis.c @@ -617,9 +617,13 @@ void updateDictResizePolicy(void) { /* Try to expire a few timed out keys. The algorithm used is adaptive and * will use few CPU cycles if there are few expiring keys, otherwise * it will get more aggressive to avoid that too much memory is used by - * keys that can be removed from the keyspace. */ + * keys that can be removed from the keyspace. + * + * No more than REDIS_DBCRON_DBS_PER_SEC databases are tested at every + * iteration. */ void activeExpireCycle(void) { - int j, iteration = 0; + static unsigned int current_db = 0; + unsigned int j, iteration = 0; long long start = ustime(), timelimit; /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time @@ -629,9 +633,14 @@ void activeExpireCycle(void) { timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/server.hz/100; if (timelimit <= 0) timelimit = 1; - for (j = 0; j < server.dbnum; j++) { + for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { int expired; - redisDb *db = server.db+j; + redisDb *db = server.db+(current_db % server.dbnum); + + /* Increment the DB now so we are sure if we run out of time + * in the current DB we'll restart from the next. This allows to + * distribute the time evenly across DBs. */ + current_db++; /* Continue to expire if at the end of the cycle more than 25% * of the keys were expired. */ From 9afb7789b3f8b33039d82a5f96a6f67936e56ea9 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 9 Mar 2013 11:44:20 +0100 Subject: [PATCH 0533/1752] REDIS_DBCRON_DBS_PER_SEC -> REDIS_DBCRON_DBS_PER_CALL --- src/redis.c | 8 ++++---- src/redis.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0f378af798e..5d92d058984 100644 --- a/src/redis.c +++ b/src/redis.c @@ -619,7 +619,7 @@ void updateDictResizePolicy(void) { * it will get more aggressive to avoid that too much memory is used by * keys that can be removed from the keyspace. * - * No more than REDIS_DBCRON_DBS_PER_SEC databases are tested at every + * No more than REDIS_DBCRON_DBS_PER_CALL databases are tested at every * iteration. */ void activeExpireCycle(void) { static unsigned int current_db = 0; @@ -633,7 +633,7 @@ void activeExpireCycle(void) { timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/server.hz/100; if (timelimit <= 0) timelimit = 1; - for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { + for (j = 0; j < REDIS_DBCRON_DBS_PER_CALL; j++) { int expired; redisDb *db = server.db+(current_db % server.dbnum); @@ -818,14 +818,14 @@ void databasesCron(void) { unsigned int j; /* Resize */ - for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { + for (j = 0; j < REDIS_DBCRON_DBS_PER_CALL; j++) { tryResizeHashTables(resize_db % server.dbnum); resize_db++; } /* Rehash */ if (server.activerehashing) { - for (j = 0; j < REDIS_DBCRON_DBS_PER_SEC; j++) { + for (j = 0; j < REDIS_DBCRON_DBS_PER_CALL; j++) { int work_done = incrementallyRehash(rehash_db % server.dbnum); rehash_db++; if (work_done) { diff --git a/src/redis.h b/src/redis.h index f7aca924d1e..f59497efe1a 100644 --- a/src/redis.h +++ b/src/redis.h @@ -76,7 +76,7 @@ #define REDIS_CONFIGLINE_MAX 1024 #define REDIS_EXPIRELOOKUPS_PER_CRON 10 /* lookup 10 expires per loop */ #define REDIS_EXPIRELOOKUPS_TIME_PERC 25 /* CPU max % for keys collection */ -#define REDIS_DBCRON_DBS_PER_SEC 16 +#define REDIS_DBCRON_DBS_PER_CALL 16 #define REDIS_MAX_WRITE_PER_EVENT (1024*64) #define REDIS_SHARED_SELECT_CMDS 10 #define REDIS_SHARED_INTEGERS 10000 From 6a8288766c3ca5e381f815ffe483e5457630dd95 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 9 Mar 2013 11:48:54 +0100 Subject: [PATCH 0534/1752] Optimize inner loop of activeExpireCycle() for no-expires case. --- src/redis.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 5d92d058984..29aaf79a02a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -645,9 +645,13 @@ void activeExpireCycle(void) { /* Continue to expire if at the end of the cycle more than 25% * of the keys were expired. */ do { - unsigned long num = dictSize(db->expires); - unsigned long slots = dictSlots(db->expires); - long long now = mstime(); + unsigned long num, slots; + long long now; + + /* If there is nothing to expire try next DB ASAP. */ + if ((num = dictSize(db->expires)) == 0) break; + slots = dictSlots(db->expires); + now = mstime(); /* When there are less than 1% filled slots getting random * keys is expensive, so stop here waiting for better times... From 7e7ee815b0ef75696874c7fa96d2d11653df1548 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Mar 2013 10:42:14 +0100 Subject: [PATCH 0535/1752] Make comment name match var name in activeExpireCycle(). --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 29aaf79a02a..3421be90cf2 100644 --- a/src/redis.c +++ b/src/redis.c @@ -687,7 +687,7 @@ void activeExpireCycle(void) { * expire. So after a given amount of milliseconds return to the * caller waiting for the other active expire cycle. */ iteration++; - if ((iteration & 0xf) == 0 && /* check once every 16 cycles. */ + if ((iteration & 0xf) == 0 && /* check once every 16 iterations. */ (ustime()-start) > timelimit) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } From c5afbb6e2a2079685cd9ebf6b9069dea2fe1e471 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Mar 2013 10:51:03 +0100 Subject: [PATCH 0536/1752] In databasesCron() never test more DBs than we have. --- src/redis.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 3421be90cf2..884b3f16b75 100644 --- a/src/redis.c +++ b/src/redis.c @@ -624,8 +624,12 @@ void updateDictResizePolicy(void) { void activeExpireCycle(void) { static unsigned int current_db = 0; unsigned int j, iteration = 0; + unsigned int dbs_per_call = REDIS_DBCRON_DBS_PER_CALL; long long start = ustime(), timelimit; + /* Don't test more DBs than we have. */ + if (dbs_per_call > server.dbnum) dbs_per_call = server.dbnum; + /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time * per iteration. Since this function gets called with a frequency of * server.hz times per second, the following is the max amount of @@ -633,7 +637,7 @@ void activeExpireCycle(void) { timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/server.hz/100; if (timelimit <= 0) timelimit = 1; - for (j = 0; j < REDIS_DBCRON_DBS_PER_CALL; j++) { + for (j = 0; j < dbs_per_call; j++) { int expired; redisDb *db = server.db+(current_db % server.dbnum); @@ -819,17 +823,21 @@ void databasesCron(void) { * cron loop iteration. */ static unsigned int resize_db = 0; static unsigned int rehash_db = 0; + unsigned int dbs_per_call = REDIS_DBCRON_DBS_PER_CALL; unsigned int j; + /* Don't test more DBs than we have. */ + if (dbs_per_call > server.dbnum) dbs_per_call = server.dbnum; + /* Resize */ - for (j = 0; j < REDIS_DBCRON_DBS_PER_CALL; j++) { + for (j = 0; j < dbs_per_call; j++) { tryResizeHashTables(resize_db % server.dbnum); resize_db++; } /* Rehash */ if (server.activerehashing) { - for (j = 0; j < REDIS_DBCRON_DBS_PER_CALL; j++) { + for (j = 0; j < dbs_per_call; j++) { int work_done = incrementallyRehash(rehash_db % server.dbnum); rehash_db++; if (work_done) { From 8e7d962291b2aa9131229afb0fc5b8e2b9efaad9 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Mar 2013 11:10:33 +0100 Subject: [PATCH 0537/1752] activeExpireCycle() smarter with many DBs and under expire pressure. activeExpireCycle() tries to test just a few DBs per iteration so that it scales if there are many configured DBs in the Redis instance. However this commit makes it a bit smarter when one a few of those DBs are under expiration pressure and there are many many keys to expire. What we do is to remember if in the last iteration had to return because we ran out of time. In that case the next iteration we'll test all the configured DBs so that we are sure we'll test again the DB under pressure. Before of this commit after some mass-expire in a given DB the function tested just a few of the next DBs, possibly empty, a few per iteration, so it took a long time for the function to reach again the DB under pressure. This resulted in a lot of memory being used by already expired keys and never accessed by clients. --- src/redis.c | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index 884b3f16b75..f7118f6ab31 100644 --- a/src/redis.c +++ b/src/redis.c @@ -622,19 +622,31 @@ void updateDictResizePolicy(void) { * No more than REDIS_DBCRON_DBS_PER_CALL databases are tested at every * iteration. */ void activeExpireCycle(void) { - static unsigned int current_db = 0; + /* This function has some global state in order to continue the work + * incrementally across calls. */ + static unsigned int current_db = 0; /* Last DB tested. */ + static int timelimit_exit = 0; /* Time limit hit in previous call? */ + unsigned int j, iteration = 0; unsigned int dbs_per_call = REDIS_DBCRON_DBS_PER_CALL; long long start = ustime(), timelimit; - /* Don't test more DBs than we have. */ - if (dbs_per_call > server.dbnum) dbs_per_call = server.dbnum; + /* We usually should test REDIS_DBCRON_DBS_PER_CALL per iteration, with + * two exceptions: + * + * 1) Don't test more DBs than we have. + * 2) If last time we hit the time limit, we want to scan all DBs + * in this iteration, as there is work to do in some DB and we don't want + * expired keys to use memory for too much time. */ + if (dbs_per_call > server.dbnum || timelimit_exit) + dbs_per_call = server.dbnum; /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time * per iteration. Since this function gets called with a frequency of * server.hz times per second, the following is the max amount of * microseconds we can spend in this function. */ timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/server.hz/100; + timelimit_exit = 0; if (timelimit <= 0) timelimit = 1; for (j = 0; j < dbs_per_call; j++) { @@ -692,7 +704,11 @@ void activeExpireCycle(void) { * caller waiting for the other active expire cycle. */ iteration++; if ((iteration & 0xf) == 0 && /* check once every 16 iterations. */ - (ustime()-start) > timelimit) return; + (ustime()-start) > timelimit) + { + timelimit_exit = 1; + return; + } } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } } From 378fcaad55f57a2ccfe1d36974bd6c4bd8bdb848 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Mar 2013 09:57:49 +0100 Subject: [PATCH 0538/1752] redis-cli --bigkeys: don't crash with empty DBs. --- src/redis-cli.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index d93c6cd5fd6..aaf3eeeb967 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1205,7 +1205,11 @@ static void findBigKeys(void) { fprintf(stderr, "RANDOMKEY error: %s\n", reply1->str); exit(1); + } else if (reply1->type == REDIS_REPLY_NIL) { + fprintf(stderr, "It looks like the database is empty!\n"); + exit(1); } + /* Get the key type */ reply2 = redisCommand(context,"TYPE %s",reply1->str); assert(reply2 && reply2->type == REDIS_REPLY_STATUS); From b2e42a90d3b4d08c3d8bc84f88a249982fe30efd Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Mar 2013 18:34:08 +0100 Subject: [PATCH 0539/1752] Set default for stop_writes_on_bgsave_err in initServerConfig(). It was placed for error in initServer() that's called after the configuation is already loaded, causing issue #1000. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index f7118f6ab31..5d667cddb14 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1196,6 +1196,7 @@ void initServerConfig() { server.requirepass = NULL; server.rdb_compression = 1; server.rdb_checksum = 1; + server.stop_writes_on_bgsave_err = 1; server.activerehashing = 1; server.notify_keyspace_events = 0; server.maxclients = REDIS_MAX_CLIENTS; @@ -1420,7 +1421,6 @@ void initServer() { server.ops_sec_last_sample_ops = 0; server.unixtime = time(NULL); server.lastbgsave_status = REDIS_OK; - server.stop_writes_on_bgsave_err = 1; if(aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR) { redisPanic("Can't create the serverCron time event."); exit(1); From edb3c103617e47b2e61c4326035805cd5d6fc893 Mon Sep 17 00:00:00 2001 From: Damian Janowski Date: Tue, 12 Mar 2013 14:37:50 -0300 Subject: [PATCH 0540/1752] Abort when opening the RDB file results in an error other than ENOENT. This fixes cases where the RDB file does exist but can't be accessed for any reason. For instance, when the Redis process doesn't have enough permissions on the file. --- src/rdb.c | 1 - src/redis.c | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index fbf9618dc16..15f17914dc8 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1069,7 +1069,6 @@ int rdbLoad(char *filename) { fp = fopen(filename,"r"); if (!fp) { - errno = ENOENT; return REDIS_ERR; } rioInitWithFile(&rdb,fp); diff --git a/src/redis.c b/src/redis.c index 5d667cddb14..93d55f5f825 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2654,7 +2654,7 @@ void loadDataFromDisk(void) { redisLog(REDIS_NOTICE,"DB loaded from disk: %.3f seconds", (float)(ustime()-start)/1000000); } else if (errno != ENOENT) { - redisLog(REDIS_WARNING,"Fatal error loading the DB. Exiting."); + redisLog(REDIS_WARNING,"Fatal error loading the DB: %s. Exiting.",strerror(errno)); exit(1); } } From 64b91c1493213be4717b4bec12eb04389285f702 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Mar 2013 19:46:33 +0100 Subject: [PATCH 0541/1752] rdbLoad(): rework code to save vertical space. --- src/rdb.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 15f17914dc8..72d36eacd24 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1067,10 +1067,8 @@ int rdbLoad(char *filename) { FILE *fp; rio rdb; - fp = fopen(filename,"r"); - if (!fp) { - return REDIS_ERR; - } + if ((fp = fopen(filename,"r")) == NULL) return REDIS_ERR; + rioInitWithFile(&rdb,fp); if (server.rdb_checksum) rdb.update_cksum = rioGenericUpdateChecksum; From 7b6fb960cc571dd92b1465c1c35cd423c77d2448 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Mar 2013 19:55:33 +0100 Subject: [PATCH 0542/1752] Test: check that Redis starts empty without an RDB file. --- tests/integration/rdb.tcl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 85c5db9f5a3..3a21c143cad 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -23,3 +23,10 @@ start_server [list overrides [list "dir" $server_path "dbfilename" "encodings.rd } } +set server_path [tmpdir "server.rdb-startup-test"] + +start_server [list overrides [list "dir" $server_path]] { + test {Server started empty with non-existing RDB file} { + r debug digest + } {0000000000000000000000000000000000000000} +} From 67a745515d827521d906c53fd5ec51c93bd8d05c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Mar 2013 10:04:33 +0100 Subject: [PATCH 0543/1752] Test: more RDB loading checks. A test for issue #1001 is included. --- tests/integration/rdb.tcl | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 3a21c143cad..2671167e4c0 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -29,4 +29,36 @@ start_server [list overrides [list "dir" $server_path]] { test {Server started empty with non-existing RDB file} { r debug digest } {0000000000000000000000000000000000000000} + # Save an RDB file, needed for the next test. + r save +} + +start_server [list overrides [list "dir" $server_path]] { + test {Server started empty with empty RDB file} { + r debug digest + } {0000000000000000000000000000000000000000} +} + +# Helper function to start a server and kill it, just to check the error +# logged. +set defaults {} +proc start_server_and_kill_it {overrides code} { + upvar defaults defaults srv srv server_path server_path + set config [concat $defaults $overrides] + set srv [start_server [list overrides $config]] + uplevel 1 $code + kill_server $srv +} + +# Make the RDB file unreadable +file attributes [file join $server_path dump.rdb] -permissions 0222 + +# Now make sure the server aborted with an error +start_server_and_kill_it [list "dir" $server_path] { + wait_for_condition 50 100 { + [string match {*Fatal error loading*} \ + [exec tail -n1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB was unreadable!" + } } From ba8367ee491660e61e3b6fbcd7f5697f921184eb Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Mar 2013 11:12:45 +0100 Subject: [PATCH 0544/1752] Test: make sure broken RDB checksum is detected. --- tests/integration/rdb.tcl | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 2671167e4c0..9322ef2af03 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -62,3 +62,22 @@ start_server_and_kill_it [list "dir" $server_path] { fail "Server started even if RDB was unreadable!" } } + +# Fix permissions of the RDB file, but corrupt its CRC64 checksum. +file attributes [file join $server_path dump.rdb] -permissions 0666 +set filesize [file size [file join $server_path dump.rdb]] +set fd [open [file join $server_path dump.rdb] r+] +fconfigure $fd -translation binary +seek $fd -8 end +puts -nonewline $fd "foobar00"; # Corrupt the checksum +close $fd + +# Now make sure the server aborted with an error +start_server_and_kill_it [list "dir" $server_path] { + wait_for_condition 50 100 { + [string match {*RDB checksum*} \ + [exec tail -n1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB was corrupted!" + } +} From 58708fa65a30920b97a1df07d8549f5b61810ce0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Mar 2013 12:51:10 +0100 Subject: [PATCH 0545/1752] Replication: master_link_down_since_seconds initial value should be huge. server.repl_down_since used to be initialized to the current time at startup. This is wrong since the replication never started. Clients testing this filed to check if data is uptodate should never believe data is recent if we never ever connected to our master. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 93d55f5f825..86bdedc69c9 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1236,7 +1236,7 @@ void initServerConfig() { server.repl_syncio_timeout = REDIS_REPL_SYNCIO_TIMEOUT; server.repl_serve_stale_data = 1; server.repl_slave_ro = 1; - server.repl_down_since = time(NULL); + server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ server.repl_disable_tcp_nodelay = 0; server.slave_priority = REDIS_DEFAULT_SLAVE_PRIORITY; server.master_repl_offset = 0; From ec18d4bfba89f54508c7834e6f275e57681c0d02 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 22 Mar 2013 17:54:32 +0100 Subject: [PATCH 0546/1752] redis-cli --stat, stolen from redis-tools. Redis-tools is a connection of tools no longer mantained that was intented as a way to economically make sense of Redis in the pre-vmware sponsorship era. However there was a nice redis-stat utility, this commit imports one of the functionalities of this tool here in redis-cli as it seems to be pretty useful. Usage: redis-cli --stat The output is similar to vmstat in the format, but with Redis specific stuff of course. From the point of view of the monitored instance, only INFO is used in order to grab data. --- src/redis-cli.c | 180 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 180 insertions(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index aaf3eeeb967..3953c10f120 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -42,6 +42,7 @@ #include #include #include +#include #include "hiredis.h" #include "sds.h" @@ -76,6 +77,7 @@ static struct config { int slave_mode; int pipe_mode; int getrdb_mode; + int stat_mode; char *rdb_filename; int bigkeys; int stdinarg; /* get last arg from stdin. (-x option) */ @@ -630,6 +632,36 @@ static int cliSendCommand(int argc, char **argv, int repeat) { return REDIS_OK; } +/* Send the INFO command, reconnecting the link if needed. */ +static redisReply *reconnectingInfo(void) { + redisContext *c = context; + redisReply *reply = NULL; + int tries = 0; + + assert(!c->err); + while(reply == NULL) { + while (c->err & (REDIS_ERR_IO | REDIS_ERR_EOF)) { + printf("Reconnecting (%d)...\r", ++tries); + fflush(stdout); + + redisFree(c); + c = redisConnect(config.hostip,config.hostport); + usleep(1000000); + } + + reply = redisCommand(c,"INFO"); + if (c->err && !(c->err & (REDIS_ERR_IO | REDIS_ERR_EOF))) { + fprintf(stderr, "Error: %s\n", c->errstr); + exit(1); + } else if (tries > 0) { + printf("\n"); + } + } + + context = c; + return reply; +} + /*------------------------------------------------------------------------------ * User interface *--------------------------------------------------------------------------- */ @@ -670,6 +702,8 @@ static int parseOptions(int argc, char **argv) { config.latency_mode = 1; } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; + } else if (!strcmp(argv[i],"--stat")) { + config.stat_mode = 1; } else if (!strcmp(argv[i],"--rdb") && !lastarg) { config.getrdb_mode = 1; config.rdb_filename = argv[++i]; @@ -1265,6 +1299,145 @@ static void findBigKeys(void) { } } +/* Return the specified INFO field from the INFO command output "info". + * A new buffer is allocated for the result, that needs to be free'd. + * If the field is not found NULL is returned. */ +static char *getInfoField(char *info, char *field) { + char *p = strstr(info,field); + char *n1, *n2; + char *result; + + if (!p) return NULL; + p += strlen(field)+1; + n1 = strchr(p,'\r'); + n2 = strchr(p,','); + if (n2 && n2 < n1) n1 = n2; + result = malloc(sizeof(char)*(n1-p)+1); + memcpy(result,p,(n1-p)); + result[n1-p] = '\0'; + return result; +} + +/* Like the above function but automatically convert the result into + * a long. On error (missing field) LONG_MIN is returned. */ +static long getLongInfoField(char *info, char *field) { + char *value = getInfoField(info,field); + long l; + + if (!value) return LONG_MIN; + l = strtol(value,NULL,10); + free(value); + return l; +} + +/* Convert number of bytes into a human readable string of the form: + * 100B, 2G, 100M, 4K, and so forth. */ +void bytesToHuman(char *s, long long n) { + double d; + + if (n < 0) { + *s = '-'; + s++; + n = -n; + } + if (n < 1024) { + /* Bytes */ + sprintf(s,"%lluB",n); + return; + } else if (n < (1024*1024)) { + d = (double)n/(1024); + sprintf(s,"%.2fK",d); + } else if (n < (1024LL*1024*1024)) { + d = (double)n/(1024*1024); + sprintf(s,"%.2fM",d); + } else if (n < (1024LL*1024*1024*1024)) { + d = (double)n/(1024LL*1024*1024); + sprintf(s,"%.2fG",d); + } +} + +static void statMode() { + redisReply *reply; + long aux, requests = 0; + int i = 0; + + while(1) { + char buf[64]; + int j; + + reply = reconnectingInfo(); + if (reply->type == REDIS_REPLY_ERROR) { + printf("ERROR: %s\n", reply->str); + exit(1); + } + + if ((i++ % 20) == 0) { + printf( +"------- data ------ --------------------- load -------------------- - child -\n" +"keys mem clients blocked requests connections \n"); + } + + /* Keys */ + aux = 0; + for (j = 0; j < 20; j++) { + long k; + + sprintf(buf,"db%d:keys",j); + k = getLongInfoField(reply->str,buf); + if (k == LONG_MIN) continue; + aux += k; + } + sprintf(buf,"%ld",aux); + printf("%-11s",buf); + + /* Used memory */ + aux = getLongInfoField(reply->str,"used_memory"); + bytesToHuman(buf,aux); + printf("%-8s",buf); + + /* Clients */ + aux = getLongInfoField(reply->str,"connected_clients"); + sprintf(buf,"%ld",aux); + printf(" %-8s",buf); + + /* Blocked (BLPOPPING) Clients */ + aux = getLongInfoField(reply->str,"blocked_clients"); + sprintf(buf,"%ld",aux); + printf("%-8s",buf); + + /* Requets */ + aux = getLongInfoField(reply->str,"total_commands_processed"); + sprintf(buf,"%ld (+%ld)",aux,requests == 0 ? 0 : aux-requests); + printf("%-19s",buf); + requests = aux; + + /* Connections */ + aux = getLongInfoField(reply->str,"total_connections_received"); + sprintf(buf,"%ld",aux); + printf(" %-12s",buf); + + /* Children */ + aux = getLongInfoField(reply->str,"bgsave_in_progress"); + aux |= getLongInfoField(reply->str,"aof_rewrite_in_progress") << 1; + switch(aux) { + case 0: break; + case 1: + printf("SAVE"); + break; + case 2: + printf("AOF"); + break; + case 3: + printf("SAVE+AOF"); + break; + } + + printf("\n"); + freeReplyObject(reply); + usleep(config.interval); + } +} + int main(int argc, char **argv) { int firstarg; @@ -1329,6 +1502,13 @@ int main(int argc, char **argv) { findBigKeys(); } + /* Stat mode */ + if (config.stat_mode) { + if (cliConnect(0) == REDIS_ERR) exit(1); + if (config.interval == 0) config.interval = 1000000; + statMode(); + } + /* Start interactive mode when no command is provided */ if (argc == 0 && !config.eval) { /* Note that in repl mode we don't abort on connection error. From 6b7562e2041bc2d33eded25cf465420bcdde53a7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 25 Mar 2013 11:56:08 +0100 Subject: [PATCH 0547/1752] Test: obuf-limits test false positive removed. Fixes #621. --- tests/unit/obuf-limits.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/obuf-limits.tcl b/tests/unit/obuf-limits.tcl index e965e892dca..5d625cf4530 100644 --- a/tests/unit/obuf-limits.tcl +++ b/tests/unit/obuf-limits.tcl @@ -15,7 +15,7 @@ start_server {tags {"obuf-limits"}} { if {![regexp {omem=([0-9]+)} $c - omem]} break if {$omem > 200000} break } - assert {$omem >= 99000 && $omem < 200000} + assert {$omem >= 90000 && $omem < 200000} $rd1 close } From 382420ec6eeecccde60481bca82666c6c3075058 Mon Sep 17 00:00:00 2001 From: NanXiao Date: Thu, 14 Mar 2013 13:52:43 +0800 Subject: [PATCH 0548/1752] Update config.c Fix bug in configGetCommand function: get correct masterauth value. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 9d2b3d7684d..c430cfe8ff2 100644 --- a/src/config.c +++ b/src/config.c @@ -816,7 +816,7 @@ void configGetCommand(redisClient *c) { /* String values */ config_get_string_field("dbfilename",server.rdb_filename); config_get_string_field("requirepass",server.requirepass); - config_get_string_field("masterauth",server.requirepass); + config_get_string_field("masterauth",server.masterauth); config_get_string_field("bind",server.bindaddr); config_get_string_field("unixsocket",server.unixsocket); config_get_string_field("logfile",server.logfile); From 150f7e43ca658b609646c846136db823160a32f4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Mar 2013 13:51:17 +0100 Subject: [PATCH 0549/1752] Allow SELECT while loading the DB. Fixes issue #1024. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 86bdedc69c9..3cebf9c78a1 100644 --- a/src/redis.c +++ b/src/redis.c @@ -197,7 +197,7 @@ struct redisCommand redisCommandTable[] = { {"mset",msetCommand,-3,"wm",0,NULL,1,-1,2,0,0}, {"msetnx",msetnxCommand,-3,"wm",0,NULL,1,-1,2,0,0}, {"randomkey",randomkeyCommand,1,"rR",0,NULL,0,0,0,0,0}, - {"select",selectCommand,2,"r",0,NULL,0,0,0,0,0}, + {"select",selectCommand,2,"rl",0,NULL,0,0,0,0,0}, {"move",moveCommand,3,"w",0,NULL,1,1,1,0,0}, {"rename",renameCommand,3,"w",0,renameGetKeys,1,2,1,0,0}, {"renamenx",renamenxCommand,3,"w",0,renameGetKeys,1,2,1,0,0}, From d91dfaf6dc9e9167caeac58ef0edb7eb87fea9a1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Mar 2013 10:27:45 +0100 Subject: [PATCH 0550/1752] Transactions: use the propagate() API to propagate MULTI. The behavior is the same, but the code is now cleaner and uses the proper interface instead of dealing directly with AOF/replication functions. --- src/multi.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/multi.c b/src/multi.c index ba8baf0a22b..2ceca8caf46 100644 --- a/src/multi.c +++ b/src/multi.c @@ -103,12 +103,11 @@ void discardCommand(redisClient *c) { /* Send a MULTI command to all the slaves and AOF file. Check the execCommand * implementation for more information. */ -void execCommandReplicateMulti(redisClient *c) { +void execCommandPropagateMulti(redisClient *c) { robj *multistring = createStringObject("MULTI",5); - if (server.aof_state != REDIS_AOF_OFF) - feedAppendOnlyFile(server.multiCommand,c->db->id,&multistring,1); - replicationFeedSlaves(server.slaves,c->db->id,&multistring,1); + propagate(server.multiCommand,c->db->id,&multistring,1, + REDIS_PROPAGATE_AOF|REDIS_PROPAGATE_REPL); decrRefCount(multistring); } @@ -139,11 +138,11 @@ void execCommand(redisClient *c) { goto handle_monitor; } - /* Replicate a MULTI request now that we are sure the block is executed. + /* Propagate a MULTI request now that we are sure the block is executed. * This way we'll deliver the MULTI/..../EXEC block as a whole and * both the AOF and the replication link will have the same consistency * and atomicity guarantees. */ - execCommandReplicateMulti(c); + execCommandPropagateMulti(c); /* Exec all the queued commands */ unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */ From 82516e8168cdc335123bcc8adb27d782d0ea7128 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Mar 2013 10:48:15 +0100 Subject: [PATCH 0551/1752] Transactions: use discardTransaction() in EXEC implementation. --- src/multi.c | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/multi.c b/src/multi.c index 2ceca8caf46..191307a67fe 100644 --- a/src/multi.c +++ b/src/multi.c @@ -131,10 +131,7 @@ void execCommand(redisClient *c) { if (c->flags & (REDIS_DIRTY_CAS|REDIS_DIRTY_EXEC)) { addReply(c, c->flags & REDIS_DIRTY_EXEC ? shared.execaborterr : shared.nullmultibulk); - freeClientMultiState(c); - initClientMultiState(c); - c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS|REDIS_DIRTY_EXEC); - unwatchAllKeys(c); + discardTransaction(c); goto handle_monitor; } @@ -164,9 +161,7 @@ void execCommand(redisClient *c) { c->argv = orig_argv; c->argc = orig_argc; c->cmd = orig_cmd; - freeClientMultiState(c); - initClientMultiState(c); - c->flags &= ~(REDIS_MULTI|REDIS_DIRTY_CAS|REDIS_DIRTY_EXEC); + discardTransaction(c); /* Make sure the EXEC command is always replicated / AOF, since we * always send the MULTI command (we can't know beforehand if the * next operations will contain at least a modification to the DB). */ From 2618101ad587be9186738a218cbb4831c1f77cb4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Mar 2013 10:58:10 +0100 Subject: [PATCH 0552/1752] Transactions: propagate MULTI/EXEC only when needed. MULTI/EXEC is now propagated to the AOF / Slaves only once we encounter the first command that is not a read-only one inside the transaction. The old behavior was to always propagate an empty MULTI/EXEC block when the transaction was composed just of read only commands, or even completely empty. This created two problems: 1) It's a bandwidth waste in the replication link and a space waste inside the AOF file. 2) We used to always increment server.dirty to force the propagation of the EXEC command, resulting into triggering RDB saves more often than needed. Note: even read-only commands may also trigger writes that will be propagated, when we access a key that is found expired and Redis will synthesize a DEL operation. However there is no need for this to stay inside the transaction itself, but only to be ordered. So for instance something like: MULTI GET foo SET key zap EXEC May be propagated into: DEL foo MULTI SET key zap EXEC While the DEL is outside the transaction, the commands are delivered in the right order and it is not possible for other commands to be inserted between DEL and MULTI. --- src/multi.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/multi.c b/src/multi.c index 191307a67fe..7000c484140 100644 --- a/src/multi.c +++ b/src/multi.c @@ -116,6 +116,7 @@ void execCommand(redisClient *c) { robj **orig_argv; int orig_argc; struct redisCommand *orig_cmd; + int must_propagate = 0; /* Need to propagate MULTI/EXEC to AOF / slaves? */ if (!(c->flags & REDIS_MULTI)) { addReplyError(c,"EXEC without MULTI"); @@ -135,12 +136,6 @@ void execCommand(redisClient *c) { goto handle_monitor; } - /* Propagate a MULTI request now that we are sure the block is executed. - * This way we'll deliver the MULTI/..../EXEC block as a whole and - * both the AOF and the replication link will have the same consistency - * and atomicity guarantees. */ - execCommandPropagateMulti(c); - /* Exec all the queued commands */ unwatchAllKeys(c); /* Unwatch ASAP otherwise we'll waste CPU cycles */ orig_argv = c->argv; @@ -151,6 +146,16 @@ void execCommand(redisClient *c) { c->argc = c->mstate.commands[j].argc; c->argv = c->mstate.commands[j].argv; c->cmd = c->mstate.commands[j].cmd; + + /* Propagate a MULTI request once we encounter the first write op. + * This way we'll deliver the MULTI/..../EXEC block as a whole and + * both the AOF and the replication link will have the same consistency + * and atomicity guarantees. */ + if (!must_propagate && !(c->cmd->flags & REDIS_CMD_READONLY)) { + execCommandPropagateMulti(c); + must_propagate = 1; + } + call(c,REDIS_CALL_FULL); /* Commands may alter argc/argv, restore mstate. */ @@ -162,10 +167,9 @@ void execCommand(redisClient *c) { c->argc = orig_argc; c->cmd = orig_cmd; discardTransaction(c); - /* Make sure the EXEC command is always replicated / AOF, since we - * always send the MULTI command (we can't know beforehand if the - * next operations will contain at least a modification to the DB). */ - server.dirty++; + /* Make sure the EXEC command will be propagated as well if MULTI + * was already propagated. */ + if (must_propagate) server.dirty++; handle_monitor: /* Send EXEC to clients waiting data from MONITOR. We do it here From 10c6195e3fbd6f43a9102bc8308c79e1504e04d5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Mar 2013 11:09:22 +0100 Subject: [PATCH 0553/1752] Flag PUBLISH as read-only in the command table. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 3cebf9c78a1..ca2a277548d 100644 --- a/src/redis.c +++ b/src/redis.c @@ -237,7 +237,7 @@ struct redisCommand redisCommandTable[] = { {"unsubscribe",unsubscribeCommand,-1,"rpslt",0,NULL,0,0,0,0,0}, {"psubscribe",psubscribeCommand,-2,"rpslt",0,NULL,0,0,0,0,0}, {"punsubscribe",punsubscribeCommand,-1,"rpslt",0,NULL,0,0,0,0,0}, - {"publish",publishCommand,3,"pflt",0,NULL,0,0,0,0,0}, + {"publish",publishCommand,3,"pfltr",0,NULL,0,0,0,0,0}, {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0}, {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,4,"awm",0,NULL,1,1,1,0,0}, From a452b548497925164757e1309e1caadfaded95a3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 26 Mar 2013 11:42:14 +0100 Subject: [PATCH 0554/1752] TTL / PTTL commands: two bugs fixed. This commit fixes two corner cases for the TTL command. 1) When the key was already logically expired (expire time older than current time) the command returned -1 instead of -2. 2) When the key was existing and the expire was found to be exactly 0 (the key was just about to expire), the command reported -1 (that is, no expire) instead of a TTL of zero (that is, about to expire). --- src/db.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index f448e0330fb..ef1af2fe11d 100644 --- a/src/db.c +++ b/src/db.c @@ -612,17 +612,17 @@ void pexpireatCommand(redisClient *c) { void ttlGenericCommand(redisClient *c, int output_ms) { long long expire, ttl = -1; - expire = getExpire(c->db,c->argv[1]); /* If the key does not exist at all, return -2 */ - if (expire == -1 && lookupKeyRead(c->db,c->argv[1]) == NULL) { + if (lookupKeyRead(c->db,c->argv[1]) == NULL) { addReplyLongLong(c,-2); return; } /* The key exists. Return -1 if it has no expire, or the actual * TTL value otherwise. */ + expire = getExpire(c->db,c->argv[1]); if (expire != -1) { ttl = expire-mstime(); - if (ttl < 0) ttl = -1; + if (ttl < 0) ttl = 0; } if (ttl == -1) { addReplyLongLong(c,-1); From 7009ff98b86cfecd4100f67e2835558e4380df01 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Mar 2013 11:29:47 +0100 Subject: [PATCH 0555/1752] Test: new functions to capture and analyze the replication stream. --- tests/test_helper.tcl | 53 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 483774219b4..6aeff6ac26f 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -408,6 +408,59 @@ for {set j 0} {$j < [llength $argv]} {incr j} { } } +proc attach_to_replication_stream {} { + set s [socket [srv 0 "host"] [srv 0 "port"]] + fconfigure $s -translation binary + puts -nonewline $s "SYNC\r\n" + flush $s + + # Get the count + set count [gets $s] + set prefix [string range $count 0 0] + if {$prefix ne {$}} { + error "attach_to_replication_stream error. Received '$count' as count." + } + set count [string range $count 1 end] + + # Consume the bulk payload + while {$count} { + set buf [read $s $count] + set count [expr {$count-[string length $buf]}] + } + return $s +} + +proc read_from_replication_stream {s} { + fconfigure $s -blocking 0 + set attempt 0 + while {[gets $s count] == -1} { + if {[incr attempt] == 10} return "" + after 100 + } + fconfigure $s -blocking 1 + set count [string range $count 1 end] + + # Return a list of arguments for the command. + set res {} + for {set j 0} {$j < $count} {incr j} { + read $s 1 + set arg [::redis::redis_bulk_read $s] + if {$j == 0} {set arg [string tolower $arg]} + lappend res $arg + } + return $res +} + +proc assert_replication_stream {s patterns} { + for {set j 0} {$j < [llength $patterns]} {incr j} { + assert_match [lindex $patterns $j] [read_from_replication_stream $s] + } +} + +proc close_replication_stream {s} { + close $s +} + # With the parallel test running multiple Redis instances at the same time # we need a fast enough computer, otherwise a lot of tests may generate # false positives. From 61b1d6da1893e0e4142c261fa5d7201b44cd84a5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Mar 2013 11:30:23 +0100 Subject: [PATCH 0556/1752] Test: Restore DB back to 9 after testing MULTI/EXEC with DB 5. --- tests/unit/multi.tcl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index f7a1f6468ec..798f589b04c 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -205,7 +205,10 @@ start_server {tags {"multi"}} { r select 5 r multi r ping - r exec + set res [r exec] + # Restore original DB + r select 9 + set res } {PONG} test {WATCH will consider touched keys target of EXPIRE} { From c6a9a20d3bc9adc8fa0c43e7ec42b7ff590946ae Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Mar 2013 11:31:27 +0100 Subject: [PATCH 0557/1752] Test: test replication of MULTI/EXEC. --- tests/unit/multi.tcl | 54 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/tests/unit/multi.tcl b/tests/unit/multi.tcl index 798f589b04c..6655bf62c2b 100644 --- a/tests/unit/multi.tcl +++ b/tests/unit/multi.tcl @@ -252,4 +252,58 @@ start_server {tags {"multi"}} { r incr x r exec } {11} + + test {MULTI / EXEC is propagated correctly (single write command)} { + set repl [attach_to_replication_stream] + r multi + r set foo bar + r exec + assert_replication_stream $repl { + {select *} + {multi} + {set foo bar} + {exec} + } + close_replication_stream $repl + } + + test {MULTI / EXEC is propagated correctly (empty transaction)} { + set repl [attach_to_replication_stream] + r multi + r exec + r set foo bar + assert_replication_stream $repl { + {select *} + {set foo bar} + } + close_replication_stream $repl + } + + test {MULTI / EXEC is propagated correctly (read-only commands)} { + r set foo value1 + set repl [attach_to_replication_stream] + r multi + r get foo + r exec + r set foo value2 + assert_replication_stream $repl { + {select *} + {set foo value2} + } + close_replication_stream $repl + } + + test {MULTI / EXEC is propagated correctly (write command, no effect)} { + r del bar foo bar + set repl [attach_to_replication_stream] + r multi + r del foo + r exec + assert_replication_stream $repl { + {select *} + {multi} + {exec} + } + close_replication_stream $repl + } } From 10d8e6a712fd6566f01115045aab82ae2bd4a641 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 27 Mar 2013 17:55:02 +0100 Subject: [PATCH 0558/1752] DEBUG set-active-expire added. We need the ability to disable the activeExpireCycle() (active expired key collection) call for testing purposes. --- src/debug.c | 9 +++++++-- src/redis.c | 4 +++- src/redis.h | 1 + 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/debug.c b/src/debug.c index b71279795c6..cd3a22a928c 100644 --- a/src/debug.c +++ b/src/debug.c @@ -332,9 +332,14 @@ void debugCommand(redisClient *c) { usleep(utime); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"set-active-expire") && + c->argc == 3) + { + server.active_expire_enabled = atoi(c->argv[2]->ptr); + addReply(c,shared.ok); } else { - addReplyError(c, - "Syntax error, try DEBUG [SEGFAULT|OBJECT |SWAPIN |SWAPOUT |RELOAD]"); + addReplyErrorFormat(c, "Unknown DEBUG subcommand '%s'", + (char*)c->argv[1]->ptr); } } diff --git a/src/redis.c b/src/redis.c index ca2a277548d..0f3d9bc3220 100644 --- a/src/redis.c +++ b/src/redis.c @@ -828,7 +828,8 @@ void clientsCron(void) { void databasesCron(void) { /* Expire keys by random sampling. Not required for slaves * as master will synthesize DELs for us. */ - if (server.masterhost == NULL) activeExpireCycle(); + if (server.active_expire_enabled && server.masterhost == NULL) + activeExpireCycle(); /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad @@ -1167,6 +1168,7 @@ void initServerConfig() { server.verbosity = REDIS_NOTICE; server.maxidletime = REDIS_MAXIDLETIME; server.tcpkeepalive = 0; + server.active_expire_enabled = 1; server.client_max_querybuf_len = REDIS_MAX_QUERYBUF_LEN; server.saveparams = NULL; server.loading = 0; diff --git a/src/redis.h b/src/redis.h index f59497efe1a..5b4edf41dd1 100644 --- a/src/redis.h +++ b/src/redis.h @@ -580,6 +580,7 @@ struct redisServer { int verbosity; /* Loglevel in redis.conf */ int maxidletime; /* Client timeout in seconds */ int tcpkeepalive; /* Set SO_KEEPALIVE if non-zero. */ + int active_expire_enabled; /* Can be disabled for testing purposes. */ size_t client_max_querybuf_len; /* Limit for client query buffer length */ int dbnum; /* Total number of configured DBs */ int daemonize; /* True if running as a daemon */ From 978185bf679725d69b2f4d9613d4761e5fa95623 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Mar 2013 11:36:49 +0100 Subject: [PATCH 0559/1752] Test: verify that lazy-expire works. --- tests/unit/expire.tcl | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index 57497fe5e09..f1b0e011001 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -160,6 +160,24 @@ start_server {tags {"expire"}} { list $size1 $size2 } {3 0} + test {Redis should lazy expire keys} { + r flushdb + r debug set-active-expire 0 + r psetex key1 500 a + r psetex key2 500 a + r psetex key3 500 a + set size1 [r dbsize] + # Redis expires random keys ten times every second so we are + # fairly sure that all the three keys should be evicted after + # one second. + after 1000 + set size2 [r dbsize] + r mget key1 key2 key3 + set size3 [r dbsize] + r debug set-active-expire 1 + list $size1 $size2 $size3 + } {3 3 0} + test {5 keys in, 5 keys out} { r flushdb r set a c From 4de4080ea5c6952566221c31ff10d8e153d74fd5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Mar 2013 11:39:02 +0100 Subject: [PATCH 0560/1752] Better DEBUG error message when num of arguments is wrong. --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index cd3a22a928c..2f62bedb055 100644 --- a/src/debug.c +++ b/src/debug.c @@ -338,7 +338,7 @@ void debugCommand(redisClient *c) { server.active_expire_enabled = atoi(c->argv[2]->ptr); addReply(c,shared.ok); } else { - addReplyErrorFormat(c, "Unknown DEBUG subcommand '%s'", + addReplyErrorFormat(c, "Unknown DEBUG subcommand or wrong number of arguments for '%s'", (char*)c->argv[1]->ptr); } } From be832929c8bc7b3ce14b2f0f63a820c4894b1368 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Mar 2013 11:46:14 +0100 Subject: [PATCH 0561/1752] Test: regression test for issue #1026. --- tests/unit/expire.tcl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/expire.tcl b/tests/unit/expire.tcl index f1b0e011001..ff3dacb3370 100644 --- a/tests/unit/expire.tcl +++ b/tests/unit/expire.tcl @@ -178,6 +178,16 @@ start_server {tags {"expire"}} { list $size1 $size2 $size3 } {3 3 0} + test {EXPIRE should not resurrect keys (issue #1026)} { + r debug set-active-expire 0 + r set foo bar + r pexpire foo 500 + after 1000 + r expire foo 10 + r debug set-active-expire 1 + r exists foo + } {0} + test {5 keys in, 5 keys out} { r flushdb r set a c From e6e07042a1894605a4120d5341c29a9fad2ecb9d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Mar 2013 12:45:07 +0100 Subject: [PATCH 0562/1752] EXPIRE should not resurrect keys. Issue #1026. --- src/db.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/db.c b/src/db.c index ef1af2fe11d..61f3b4ffb0e 100644 --- a/src/db.c +++ b/src/db.c @@ -548,7 +548,6 @@ int expireIfNeeded(redisDb *db, robj *key) { * unit is either UNIT_SECONDS or UNIT_MILLISECONDS, and is only used for * the argv[2] parameter. The basetime is always specified in milliseconds. */ void expireGenericCommand(redisClient *c, long long basetime, int unit) { - dictEntry *de; robj *key = c->argv[1], *param = c->argv[2]; long long when; /* unix time in milliseconds when the key will expire. */ @@ -558,11 +557,12 @@ void expireGenericCommand(redisClient *c, long long basetime, int unit) { if (unit == UNIT_SECONDS) when *= 1000; when += basetime; - de = dictFind(c->db->dict,key->ptr); - if (de == NULL) { + /* No key, return zero. */ + if (lookupKeyRead(c->db,key) == NULL) { addReply(c,shared.czero); return; } + /* EXPIRE with negative TTL, or EXPIREAT with a timestamp into the past * should never be executed as a DEL when load the AOF or in the context * of a slave instance. From 24aa61785d08eecdba8d082cb4d164858d12e7c1 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Mar 2013 15:40:19 +0100 Subject: [PATCH 0563/1752] Extended SET command implemented (issue #931). --- src/redis.c | 2 +- src/t_string.c | 66 ++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0f3d9bc3220..07e215c69bc 100644 --- a/src/redis.c +++ b/src/redis.c @@ -114,7 +114,7 @@ struct redisCommand *commandTable; */ struct redisCommand redisCommandTable[] = { {"get",getCommand,2,"r",0,NULL,1,1,1,0,0}, - {"set",setCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0}, + {"set",setCommand,-3,"wm",0,noPreloadGetKeys,1,1,1,0,0}, {"setnx",setnxCommand,3,"wm",0,noPreloadGetKeys,1,1,1,0,0}, {"setex",setexCommand,4,"wm",0,noPreloadGetKeys,1,1,1,0,0}, {"psetex",psetexCommand,4,"wm",0,noPreloadGetKeys,1,1,1,0,0}, diff --git a/src/t_string.c b/src/t_string.c index 1d25e5ad131..b619cbc7cf7 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -42,7 +42,27 @@ static int checkStringLength(redisClient *c, long long size) { return REDIS_OK; } -void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expire, int unit) { +/* The setGenericCommand() function implements the SET operation with different + * options and variants. This function is called in order to implement the + * following commands: SET, SETEX, PSETEX, SETNX. + * + * 'flags' changes the behavior of the command (NX or XX, see belove). + * + * 'expire' represents an expire to set in form of a Redis object as passed + * by the user. It is interpreted according to the specified 'unit'. + * + * 'ok_reply' and 'abort_reply' is what the function will reply to the client + * if the operation is performed, or when it is not because of NX or + * XX flags. + * + * If ok_reply is NULL "+OK" is used. + * If abort_reply is NULL, "$-1" is used. */ + +#define REDIS_SET_NO_FLAGS 0 +#define REDIS_SET_NX (1<<0) /* Set if key not exists. */ +#define REDIS_SET_XX (1<<1) /* Set if key exists. */ + +void setGenericCommand(redisClient *c, int flags, robj *key, robj *val, robj *expire, int unit, robj *ok_reply, robj *abort_reply) { long long milliseconds = 0; /* initialized to avoid any harmness warning */ if (expire) { @@ -55,8 +75,10 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir if (unit == UNIT_SECONDS) milliseconds *= 1000; } - if (nx && lookupKeyWrite(c->db,key) != NULL) { - addReply(c,shared.czero); + if ((flags & REDIS_SET_NX && lookupKeyWrite(c->db,key) != NULL) || + (flags & REDIS_SET_XX && lookupKeyWrite(c->db,key) == NULL)) + { + addReply(c, abort_reply ? abort_reply : shared.nullbulk); return; } setKey(c->db,key,val); @@ -65,27 +87,55 @@ void setGenericCommand(redisClient *c, int nx, robj *key, robj *val, robj *expir notifyKeyspaceEvent(REDIS_NOTIFY_STRING,"set",key,c->db->id); if (expire) notifyKeyspaceEvent(REDIS_NOTIFY_GENERIC, "expire",key,c->db->id); - addReply(c, nx ? shared.cone : shared.ok); + addReply(c, ok_reply ? ok_reply : shared.ok); } +/* SET key value [NX] [XX] [EX ] [PX ] */ void setCommand(redisClient *c) { + int j; + robj *expire = NULL; + int unit = UNIT_SECONDS; + int flags = REDIS_SET_NO_FLAGS; + + for (j = 3; j < c->argc; j++) { + char *a = c->argv[j]->ptr; + robj *next = (j == c->argc-1) ? NULL : c->argv[j+1]; + + if (a[0] == 'n' && a[1] == 'x' && a[2] == '\0') { + flags |= REDIS_SET_NX; + } else if (a[0] == 'x' && a[1] == 'x' && a[2] == '\0') { + flags |= REDIS_SET_XX; + } else if (a[0] == 'e' && a[1] == 'x' && a[2] == '\0' && next) { + unit = UNIT_SECONDS; + expire = next; + j++; + } else if (a[0] == 'p' && a[1] == 'x' && a[2] == '\0' && next) { + unit = UNIT_MILLISECONDS; + expire = next; + j++; + } else { + addReply(c,shared.syntaxerr); + return; + } + } + c->argv[2] = tryObjectEncoding(c->argv[2]); - setGenericCommand(c,0,c->argv[1],c->argv[2],NULL,0); + setGenericCommand(c,flags,c->argv[1],c->argv[2],expire,unit,NULL,NULL); } void setnxCommand(redisClient *c) { c->argv[2] = tryObjectEncoding(c->argv[2]); - setGenericCommand(c,1,c->argv[1],c->argv[2],NULL,0); + setGenericCommand(c,REDIS_SET_NX,c->argv[1],c->argv[2],NULL,0,shared.cone,shared.czero); } void setexCommand(redisClient *c) { c->argv[3] = tryObjectEncoding(c->argv[3]); - setGenericCommand(c,0,c->argv[1],c->argv[3],c->argv[2],UNIT_SECONDS); + setGenericCommand(c,REDIS_SET_NO_FLAGS,c->argv[1],c->argv[3],c->argv[2],UNIT_SECONDS,NULL,NULL); } void psetexCommand(redisClient *c) { c->argv[3] = tryObjectEncoding(c->argv[3]); - setGenericCommand(c,0,c->argv[1],c->argv[3],c->argv[2],UNIT_MILLISECONDS); + setGenericCommand(c,REDIS_SET_NO_FLAGS,c->argv[1],c->argv[3],c->argv[2],UNIT_MILLISECONDS,NULL,NULL); } int getGenericCommand(redisClient *c) { From 0c447d350624f2504fcaa0aec1b7cbd5cea1057a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Mar 2013 16:25:17 +0100 Subject: [PATCH 0564/1752] Test: Extended SET tests. --- tests/unit/basic.tcl | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/unit/basic.tcl b/tests/unit/basic.tcl index 86498d1fa53..c766b3de998 100644 --- a/tests/unit/basic.tcl +++ b/tests/unit/basic.tcl @@ -712,4 +712,46 @@ start_server {tags {"basic"}} { assert_equal [string range $bin $_start $_end] [r getrange bin $start $end] } } + + test {Extended SET can detect syntax errors} { + set e {} + catch {r set foo bar non-existing-option} e + set e + } {*syntax*} + + test {Extended SET NX option} { + r del foo + set v1 [r set foo 1 nx] + set v2 [r set foo 2 nx] + list $v1 $v2 [r get foo] + } {OK {} 1} + + test {Extended SET XX option} { + r del foo + set v1 [r set foo 1 xx] + r set foo bar + set v2 [r set foo 2 xx] + list $v1 $v2 [r get foo] + } {{} OK 2} + + test {Extended SET EX option} { + r del foo + r set foo bar ex 10 + set ttl [r ttl foo] + assert {$ttl <= 10 && $ttl > 5} + } + + test {Extended SET PX option} { + r del foo + r set foo bar px 10000 + set ttl [r ttl foo] + assert {$ttl <= 10 && $ttl > 5} + } + + test {Extended SET using multiple options at once} { + r set foo val + assert {[r set foo bar xx px 10000] eq {OK}} + set ttl [r ttl foo] + assert {$ttl <= 10 && $ttl > 5} + } } From cf8484b723219128c7e669b19d9f1a18c0dd0cc1 Mon Sep 17 00:00:00 2001 From: charsyam Date: Thu, 28 Mar 2013 14:41:46 -0700 Subject: [PATCH 0565/1752] Support for case unsensitive SET options. --- src/t_string.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/t_string.c b/src/t_string.c index b619cbc7cf7..cbd069d3c9d 100644 --- a/src/t_string.c +++ b/src/t_string.c @@ -101,15 +101,19 @@ void setCommand(redisClient *c) { char *a = c->argv[j]->ptr; robj *next = (j == c->argc-1) ? NULL : c->argv[j+1]; - if (a[0] == 'n' && a[1] == 'x' && a[2] == '\0') { + if ((a[0] == 'n' || a[0] == 'N') && + (a[1] == 'x' || a[1] == 'X') && a[2] == '\0') { flags |= REDIS_SET_NX; - } else if (a[0] == 'x' && a[1] == 'x' && a[2] == '\0') { + } else if ((a[0] == 'x' || a[0] == 'X') && + (a[1] == 'x' || a[1] == 'X') && a[2] == '\0') { flags |= REDIS_SET_XX; - } else if (a[0] == 'e' && a[1] == 'x' && a[2] == '\0' && next) { + } else if ((a[0] == 'e' || a[0] == 'E') && + (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' && next) { unit = UNIT_SECONDS; expire = next; j++; - } else if (a[0] == 'p' && a[1] == 'x' && a[2] == '\0' && next) { + } else if ((a[0] == 'p' || a[0] == 'P') && + (a[1] == 'x' || a[1] == 'X') && a[2] == '\0' && next) { unit = UNIT_MILLISECONDS; expire = next; j++; From 0c8690d46ec2d9b2806c8c8f4f9955051bc1dc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sat, 16 Mar 2013 18:33:42 +1100 Subject: [PATCH 0566/1752] Slightly refactor CFLAGS/LDFLAGS/LIBS This way, we can avoid -rdynamic and -pthread warnings on darwin. --- src/Makefile | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/Makefile b/src/Makefile index 9202fb4bd97..98cf613ca8f 100644 --- a/src/Makefile +++ b/src/Makefile @@ -45,16 +45,20 @@ endif # Override default settings if possible -include .make-settings +FINAL_CFLAGS=$(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) +FINAL_LDFLAGS=$(LDFLAGS) $(REDIS_LDFLAGS) $(DEBUG) +FINAL_LIBS=-lm +DEBUG=-g -ggdb + ifeq ($(uname_S),SunOS) - FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) -D__EXTENSIONS__ -D_XPG6 - FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) -g -ggdb - FINAL_LIBS= -ldl -lnsl -lsocket -lm -lpthread - DEBUG= -g -ggdb + FINAL_CFLAGS+= -D__EXTENSIONS__ -D_XPG6 + FINAL_LIBS+= -ldl -lnsl -lsocket -lpthread +else ifeq ($(uname_S),Darwin) + else - FINAL_CFLAGS= $(STD) $(WARN) $(OPT) $(DEBUG) $(CFLAGS) $(REDIS_CFLAGS) - FINAL_LDFLAGS= $(LDFLAGS) $(REDIS_LDFLAGS) -g -rdynamic -ggdb - FINAL_LIBS= -lm -pthread - DEBUG= -g -rdynamic -ggdb + FINAL_LDFLAGS+= -rdynamic + FINAL_LIBS+= -pthread + DEBUG+= -rdynamic endif # Include paths to dependencies From 49eef7738ec33f50ac3e1709a4de345afeb60b24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sat, 16 Mar 2013 18:35:20 +1100 Subject: [PATCH 0567/1752] Spaces to tabs --- deps/Makefile | 4 ++-- src/Makefile | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/deps/Makefile b/deps/Makefile index d58ee5681ba..5a95545defd 100644 --- a/deps/Makefile +++ b/deps/Makefile @@ -54,8 +54,8 @@ linenoise: .make-prerequisites .PHONY: linenoise ifeq ($(uname_S),SunOS) - # Make isinf() available - LUA_CFLAGS= -D__C99FEATURES__=1 + # Make isinf() available + LUA_CFLAGS= -D__C99FEATURES__=1 endif LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI $(CFLAGS) diff --git a/src/Makefile b/src/Makefile index 98cf613ca8f..e06ef48f5ba 100644 --- a/src/Makefile +++ b/src/Makefile @@ -24,22 +24,22 @@ OPT= $(OPTIMIZATION) # Default allocator ifeq ($(uname_S),Linux) - MALLOC=jemalloc + MALLOC=jemalloc else - MALLOC=libc + MALLOC=libc endif # Backwards compatibility for selecting an allocator ifeq ($(USE_TCMALLOC),yes) - MALLOC=tcmalloc + MALLOC=tcmalloc endif ifeq ($(USE_TCMALLOC_MINIMAL),yes) - MALLOC=tcmalloc_minimal + MALLOC=tcmalloc_minimal endif ifeq ($(USE_JEMALLOC),yes) - MALLOC=jemalloc + MALLOC=jemalloc endif # Override default settings if possible @@ -65,19 +65,19 @@ endif FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src ifeq ($(MALLOC),tcmalloc) - FINAL_CFLAGS+= -DUSE_TCMALLOC - FINAL_LIBS+= -ltcmalloc + FINAL_CFLAGS+= -DUSE_TCMALLOC + FINAL_LIBS+= -ltcmalloc endif ifeq ($(MALLOC),tcmalloc_minimal) - FINAL_CFLAGS+= -DUSE_TCMALLOC - FINAL_LIBS+= -ltcmalloc_minimal + FINAL_CFLAGS+= -DUSE_TCMALLOC + FINAL_LIBS+= -ltcmalloc_minimal endif ifeq ($(MALLOC),jemalloc) - DEPENDENCY_TARGETS+= jemalloc - FINAL_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include - FINAL_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl + DEPENDENCY_TARGETS+= jemalloc + FINAL_CFLAGS+= -DUSE_JEMALLOC -I../deps/jemalloc/include + FINAL_LIBS+= ../deps/jemalloc/lib/libjemalloc.a -ldl endif REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) From ca0b1c2436d9f3e35519387beda40ca83c162534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sat, 16 Mar 2013 18:38:37 +1100 Subject: [PATCH 0568/1752] Inherit CC for Lua --- deps/lua/src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/lua/src/Makefile b/deps/lua/src/Makefile index 77d6a48b4b1..f5c45fcd3e8 100644 --- a/deps/lua/src/Makefile +++ b/deps/lua/src/Makefile @@ -7,7 +7,7 @@ # Your platform. See PLATS for possible values. PLAT= none -CC= gcc +CC?= gcc CFLAGS= -O2 -Wall $(MYCFLAGS) AR= ar rcu RANLIB= ranlib From c72459bf1835d6672c657f1059e26b78b3e0d663 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sat, 16 Mar 2013 18:40:22 +1100 Subject: [PATCH 0569/1752] make check is a common naming convention for tests --- src/Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Makefile b/src/Makefile index e06ef48f5ba..3444c1a1f7a 100644 --- a/src/Makefile +++ b/src/Makefile @@ -200,6 +200,8 @@ distclean: clean test: $(REDIS_SERVER_NAME) $(REDIS_CHECK_AOF_NAME) @(cd ..; ./runtest) +check: test + lcov: $(MAKE) gcov @(set -e; cd ..; ./runtest --clients 1) From 31844fe138c46c2001e1000f2e324af17be5097c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sat, 16 Mar 2013 18:44:38 +1100 Subject: [PATCH 0570/1752] Remove extra spaces --- src/Makefile | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3444c1a1f7a..d48bb396fa6 100644 --- a/src/Makefile +++ b/src/Makefile @@ -18,9 +18,9 @@ OPTIMIZATION?=-O2 DEPENDENCY_TARGETS=hiredis linenoise lua # Default settings -STD= -std=c99 -pedantic -WARN= -Wall -OPT= $(OPTIMIZATION) +STD=-std=c99 -pedantic +WARN=-Wall +OPT=$(OPTIMIZATION) # Default allocator ifeq ($(uname_S),Linux) @@ -85,8 +85,8 @@ REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) PREFIX?=/usr/local -INSTALL_BIN= $(PREFIX)/bin -INSTALL= cp -pf +INSTALL_BIN=$(PREFIX)/bin +INSTALL=cp -pf CCCOLOR="\033[34m" LINKCOLOR="\033[34;1m" @@ -101,17 +101,17 @@ QUIET_LINK = @printf ' %b %b\n' $(LINKCOLOR)LINK$(ENDCOLOR) $(BINCOLOR)$@$(EN QUIET_INSTALL = @printf ' %b %b\n' $(LINKCOLOR)INSTALL$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) 1>&2; endif -REDIS_SERVER_NAME= redis-server -REDIS_SENTINEL_NAME= redis-sentinel -REDIS_SERVER_OBJ= adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o -REDIS_CLI_NAME= redis-cli -REDIS_CLI_OBJ= anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o -REDIS_BENCHMARK_NAME= redis-benchmark -REDIS_BENCHMARK_OBJ= ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o -REDIS_CHECK_DUMP_NAME= redis-check-dump -REDIS_CHECK_DUMP_OBJ= redis-check-dump.o lzf_c.o lzf_d.o crc64.o -REDIS_CHECK_AOF_NAME= redis-check-aof -REDIS_CHECK_AOF_OBJ= redis-check-aof.o +REDIS_SERVER_NAME=redis-server +REDIS_SENTINEL_NAME=redis-sentinel +REDIS_SERVER_OBJ=adlist.o ae.o anet.o dict.o redis.o sds.o zmalloc.o lzf_c.o lzf_d.o pqsort.o zipmap.o sha1.o ziplist.o release.o networking.o util.o object.o db.o replication.o rdb.o t_string.o t_list.o t_set.o t_zset.o t_hash.o config.o aof.o pubsub.o multi.o debug.o sort.o intset.o syncio.o migrate.o endianconv.o slowlog.o scripting.o bio.o rio.o rand.o memtest.o crc64.o bitops.o sentinel.o notify.o setproctitle.o +REDIS_CLI_NAME=redis-cli +REDIS_CLI_OBJ=anet.o sds.o adlist.o redis-cli.o zmalloc.o release.o anet.o ae.o crc64.o +REDIS_BENCHMARK_NAME=redis-benchmark +REDIS_BENCHMARK_OBJ=ae.o anet.o redis-benchmark.o sds.o adlist.o zmalloc.o redis-benchmark.o +REDIS_CHECK_DUMP_NAME=redis-check-dump +REDIS_CHECK_DUMP_OBJ=redis-check-dump.o lzf_c.o lzf_d.o crc64.o +REDIS_CHECK_AOF_NAME=redis-check-aof +REDIS_CHECK_AOF_OBJ=redis-check-aof.o all: $(REDIS_SERVER_NAME) $(REDIS_SENTINEL_NAME) $(REDIS_CLI_NAME) $(REDIS_BENCHMARK_NAME) $(REDIS_CHECK_DUMP_NAME) $(REDIS_CHECK_AOF_NAME) @echo "" From 2094eb36e737f747eeafcefbc119d6571f3c45de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sun, 17 Mar 2013 17:49:57 +1100 Subject: [PATCH 0571/1752] Only pass -rdynamic as linker option --- src/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index d48bb396fa6..1c271d86416 100644 --- a/src/Makefile +++ b/src/Makefile @@ -58,7 +58,6 @@ else ifeq ($(uname_S),Darwin) else FINAL_LDFLAGS+= -rdynamic FINAL_LIBS+= -pthread - DEBUG+= -rdynamic endif # Include paths to dependencies From 034d94193d4bfec0c5471f80a23b53291e6cc2dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sun, 17 Mar 2013 18:03:14 +1100 Subject: [PATCH 0572/1752] Silence mkdir output --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 1c271d86416..5cedbdf2a57 100644 --- a/src/Makefile +++ b/src/Makefile @@ -231,7 +231,7 @@ src/help.h: @../utils/generate-command-help.rb > help.h install: all - mkdir -p $(INSTALL_BIN) + @mkdir -p $(INSTALL_BIN) $(REDIS_INSTALL) $(REDIS_SERVER_NAME) $(INSTALL_BIN) $(REDIS_INSTALL) $(REDIS_BENCHMARK_NAME) $(INSTALL_BIN) $(REDIS_INSTALL) $(REDIS_CLI_NAME) $(INSTALL_BIN) From 0bc7dbda1c6d2b2e52af0606e4d135524562e43b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Wed, 20 Mar 2013 22:05:59 +1100 Subject: [PATCH 0573/1752] use `install` as default installer (except on SunOS) --- src/Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Makefile b/src/Makefile index 5cedbdf2a57..c37549dfe6d 100644 --- a/src/Makefile +++ b/src/Makefile @@ -22,6 +22,10 @@ STD=-std=c99 -pedantic WARN=-Wall OPT=$(OPTIMIZATION) +PREFIX?=/usr/local +INSTALL_BIN=$(PREFIX)/bin +INSTALL=install + # Default allocator ifeq ($(uname_S),Linux) MALLOC=jemalloc @@ -51,6 +55,7 @@ FINAL_LIBS=-lm DEBUG=-g -ggdb ifeq ($(uname_S),SunOS) + INSTALL=cp -pf FINAL_CFLAGS+= -D__EXTENSIONS__ -D_XPG6 FINAL_LIBS+= -ldl -lnsl -lsocket -lpthread else ifeq ($(uname_S),Darwin) @@ -83,10 +88,6 @@ REDIS_CC=$(QUIET_CC)$(CC) $(FINAL_CFLAGS) REDIS_LD=$(QUIET_LINK)$(CC) $(FINAL_LDFLAGS) REDIS_INSTALL=$(QUIET_INSTALL)$(INSTALL) -PREFIX?=/usr/local -INSTALL_BIN=$(PREFIX)/bin -INSTALL=cp -pf - CCCOLOR="\033[34m" LINKCOLOR="\033[34;1m" SRCCOLOR="\033[33m" From d6b0c18c510aee0036c494d228233d144c7f5d25 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 2 Apr 2013 14:05:50 +0200 Subject: [PATCH 0574/1752] Throttle BGSAVE attempt on saving error. When a BGSAVE fails, Redis used to flood itself trying to BGSAVE at every next cron call, that is either 10 or 100 times per second depending on configuration and server version. This commit does not allow a new automatic BGSAVE attempt to be performed before a few seconds delay (currently 5). This avoids both the auto-flood problem and filling the disk with logs at a serious rate. The five seconds limit, considering a log entry of 200 bytes, will use less than 4 MB of disk space per day that is reasonable, the sysadmin should notice before of catastrofic events especially since by default Redis will stop serving write queries after the first failed BGSAVE. This fixes issue #849 --- src/rdb.c | 1 + src/redis.c | 13 +++++++++++-- src/redis.h | 2 ++ 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 72d36eacd24..3118e320f53 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -722,6 +722,7 @@ int rdbSaveBackground(char *filename) { if (server.rdb_child_pid != -1) return REDIS_ERR; server.dirty_before_bgsave = server.dirty; + server.lastbgsave_try = time(NULL); start = ustime(); if ((childpid = fork()) == 0) { diff --git a/src/redis.c b/src/redis.c index 07e215c69bc..67740c0b651 100644 --- a/src/redis.c +++ b/src/redis.c @@ -998,8 +998,16 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { for (j = 0; j < server.saveparamslen; j++) { struct saveparam *sp = server.saveparams+j; + /* Save if we reached the given amount of changes, + * the given amount of seconds, and if the latest bgsave was + * successful or if, in case of an error, at least + * REDIS_BGSAVE_RETRY_DELAY seconds already elapsed. */ if (server.dirty >= sp->changes && - server.unixtime-server.lastsave > sp->seconds) { + server.unixtime-server.lastsave > sp->seconds && + (server.unixtime-server.lastbgsave_try > + REDIS_BGSAVE_RETRY_DELAY || + server.lastbgsave_status == REDIS_OK)) + { redisLog(REDIS_NOTICE,"%d changes in %d seconds. Saving...", sp->changes, (int)sp->seconds); rdbSaveBackground(server.rdb_filename); @@ -1400,7 +1408,8 @@ void initServer() { server.aof_child_pid = -1; aofRewriteBufferReset(); server.aof_buf = sdsempty(); - server.lastsave = time(NULL); + server.lastsave = time(NULL); /* At startup we consider the DB saved. */ + server.lastbgsave_try = 0; /* At startup we never tried to BGSAVE. */ server.rdb_save_time_last = -1; server.rdb_save_time_start = -1; server.dirty = 0; diff --git a/src/redis.h b/src/redis.h index 5b4edf41dd1..4ace7eb37e4 100644 --- a/src/redis.h +++ b/src/redis.h @@ -97,6 +97,7 @@ #define REDIS_DEFAULT_REPL_BACKLOG_SIZE (1024*1024) /* 1mb */ #define REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT (60*60) /* 1 hour */ #define REDIS_REPL_BACKLOG_MIN_SIZE (1024*16) /* 16k */ +#define REDIS_BGSAVE_RETRY_DELAY 5 /* Wait a few secs before trying again. */ /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ @@ -616,6 +617,7 @@ struct redisServer { int rdb_compression; /* Use compression in RDB? */ int rdb_checksum; /* Use RDB checksum? */ time_t lastsave; /* Unix time of last successful save */ + time_t lastbgsave_try; /* Unix time of last attempted bgsave */ time_t rdb_save_time_last; /* Time used by last RDB save run. */ time_t rdb_save_time_start; /* Current RDB save start time. */ int lastbgsave_status; /* REDIS_OK or REDIS_ERR */ From d83d72f380179a89bc08b49eabccae95c56ec71a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 3 Apr 2013 12:41:14 +0200 Subject: [PATCH 0575/1752] Make rio.c comment 80-columns friendly. --- src/rio.h | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/rio.h b/src/rio.h index 43eae0cadb6..ba17469f832 100644 --- a/src/rio.h +++ b/src/rio.h @@ -43,10 +43,11 @@ struct _rio { size_t (*read)(struct _rio *, void *buf, size_t len); size_t (*write)(struct _rio *, const void *buf, size_t len); off_t (*tell)(struct _rio *); - /* The update_cksum method if not NULL is used to compute the checksum of all the - * data that was read or written so far. The method should be designed so that - * can be called with the current checksum, and the buf and len fields pointing - * to the new block of data to add to the checksum computation. */ + /* The update_cksum method if not NULL is used to compute the checksum of + * all the data that was read or written so far. The method should be + * designed so that can be called with the current checksum, and the buf + * and len fields pointing to the new block of data to add to the checksum + * computation. */ void (*update_cksum)(struct _rio *, const void *buf, size_t len); /* The current checksum */ From 8d81e68d18d9a513a8d657f566bd6f986a634f85 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 4 Apr 2013 14:30:05 +0200 Subject: [PATCH 0576/1752] Test: remove useless statements and comments from test default config. --- tests/assets/default.conf | 325 +------------------------------------- 1 file changed, 1 insertion(+), 324 deletions(-) diff --git a/tests/assets/default.conf b/tests/assets/default.conf index d6da94e0d3d..9f95700d1c8 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -1,347 +1,24 @@ -# Redis configuration file example +# Redis configuration for testing. -# Note on units: when memory size is needed, it is possible to specifiy -# it in the usual form of 1k 5GB 4M and so forth: -# -# 1k => 1000 bytes -# 1kb => 1024 bytes -# 1m => 1000000 bytes -# 1mb => 1024*1024 bytes -# 1g => 1000000000 bytes -# 1gb => 1024*1024*1024 bytes -# -# units are case insensitive so 1GB 1Gb 1gB are all the same. - -# Enable keyspace events notification for testing so that we cover more code. notify-keyspace-events KEA - -# By default Redis does not run as a daemon. Use 'yes' if you need it. -# Note that Redis will write a pid file in /var/run/redis.pid when daemonized. daemonize no - -# When running daemonized, Redis writes a pid file in /var/run/redis.pid by -# default. You can specify a custom pid file location here. pidfile /var/run/redis.pid - -# Accept connections on the specified port, default is 6379. port 6379 - -# If you want you can bind a single interface, if the bind option is not -# specified all the interfaces will listen for incoming connections. -# -# bind 127.0.0.1 - -# Specify the path for the unix socket that will be used to listen for -# incoming connections. There is no default, so Redis will not listen -# on a unix socket when not specified. -# -# unixsocket /tmp/redis.sock - -# Close the connection after a client is idle for N seconds (0 to disable) timeout 0 - -# Set server verbosity to 'debug' -# it can be one of: -# debug (a lot of information, useful for development/testing) -# verbose (many rarely useful info, but not a mess like the debug level) -# notice (moderately verbose, what you want in production probably) -# warning (only very important / critical messages are logged) loglevel verbose - -# Specify the log file name. Also 'stdout' can be used to force -# Redis to log on the standard output. Note that if you use standard -# output for logging but daemonize, logs will be sent to /dev/null logfile stdout - -# To enable logging to the system logger, just set 'syslog-enabled' to yes, -# and optionally update the other syslog parameters to suit your needs. -# syslog-enabled no - -# Specify the syslog identity. -# syslog-ident redis - -# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. -# syslog-facility local0 - -# Set the number of databases. The default database is DB 0, you can select -# a different one on a per-connection basis using SELECT where -# dbid is a number between 0 and 'databases'-1 databases 16 -################################ SNAPSHOTTING ################################# -# -# Save the DB on disk: -# -# save -# -# Will save the DB if both the given number of seconds and the given -# number of write operations against the DB occurred. -# -# In the example below the behaviour will be to save: -# after 900 sec (15 min) if at least 1 key changed -# after 300 sec (5 min) if at least 10 keys changed -# after 60 sec if at least 10000 keys changed -# -# Note: you can disable saving at all commenting all the "save" lines. - save 900 1 save 300 10 save 60 10000 -# Compress string objects using LZF when dump .rdb databases? -# For default that's set to 'yes' as it's almost always a win. -# If you want to save some CPU in the saving child set it to 'no' but -# the dataset will likely be bigger if you have compressible values or keys. rdbcompression yes - -# The filename where to dump the DB dbfilename dump.rdb - -# The working directory. -# -# The DB will be written inside this directory, with the filename specified -# above using the 'dbfilename' configuration directive. -# -# Also the Append Only File will be created inside this directory. -# -# Note that you must specify a directory here, not a file name. dir ./ -################################# REPLICATION ################################# - -# Master-Slave replication. Use slaveof to make a Redis instance a copy of -# another Redis server. Note that the configuration is local to the slave -# so for example it is possible to configure the slave to save the DB with a -# different interval, or to listen to another port, and so on. -# -# slaveof - -# If the master is password protected (using the "requirepass" configuration -# directive below) it is possible to tell the slave to authenticate before -# starting the replication synchronization process, otherwise the master will -# refuse the slave request. -# -# masterauth - -# When a slave lost the connection with the master, or when the replication -# is still in progress, the slave can act in two different ways: -# -# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will -# still reply to client requests, possibly with out of data data, or the -# data set may just be empty if this is the first synchronization. -# -# 2) if slave-serve-stale data is set to 'no' the slave will reply with -# an error "SYNC with master in progress" to all the kind of commands -# but to INFO and SLAVEOF. -# slave-serve-stale-data yes - -################################## SECURITY ################################### - -# Require clients to issue AUTH before processing any other -# commands. This might be useful in environments in which you do not trust -# others with access to the host running redis-server. -# -# This should stay commented out for backward compatibility and because most -# people do not need auth (e.g. they run their own servers). -# -# Warning: since Redis is pretty fast an outside user can try up to -# 150k passwords per second against a good box. This means that you should -# use a very strong password otherwise it will be very easy to break. -# -# requirepass foobared - -# Command renaming. -# -# It is possilbe to change the name of dangerous commands in a shared -# environment. For instance the CONFIG command may be renamed into something -# of hard to guess so that it will be still available for internal-use -# tools but not available for general clients. -# -# Example: -# -# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 -# -# It is also possilbe to completely kill a command renaming it into -# an empty string: -# -# rename-command CONFIG "" - -################################### LIMITS #################################### - -# Set the max number of connected clients at the same time. By default there -# is no limit, and it's up to the number of file descriptors the Redis process -# is able to open. The special value '0' means no limits. -# Once the limit is reached Redis will close all the new connections sending -# an error 'max number of clients reached'. -# -# maxclients 128 - -# Don't use more memory than the specified amount of bytes. -# When the memory limit is reached Redis will try to remove keys with an -# EXPIRE set. It will try to start freeing keys that are going to expire -# in little time and preserve keys with a longer time to live. -# Redis will also try to remove objects from free lists if possible. -# -# If all this fails, Redis will start to reply with errors to commands -# that will use more memory, like SET, LPUSH, and so on, and will continue -# to reply to most read-only commands like GET. -# -# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a -# 'state' server or cache, not as a real DB. When Redis is used as a real -# database the memory usage will grow over the weeks, it will be obvious if -# it is going to use too much memory in the long run, and you'll have the time -# to upgrade. With maxmemory after the limit is reached you'll start to get -# errors for write operations, and this may even lead to DB inconsistency. -# -# maxmemory - -# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory -# is reached? You can select among five behavior: -# -# volatile-lru -> remove the key with an expire set using an LRU algorithm -# allkeys-lru -> remove any key accordingly to the LRU algorithm -# volatile-random -> remove a random key with an expire set -# allkeys->random -> remove a random key, any key -# volatile-ttl -> remove the key with the nearest expire time (minor TTL) -# noeviction -> don't expire at all, just return an error on write operations -# -# Note: with all the kind of policies, Redis will return an error on write -# operations, when there are not suitable keys for eviction. -# -# At the date of writing this commands are: set setnx setex append -# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd -# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby -# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby -# getset mset msetnx exec sort -# -# The default is: -# -# maxmemory-policy volatile-lru - -# LRU and minimal TTL algorithms are not precise algorithms but approximated -# algorithms (in order to save memory), so you can select as well the sample -# size to check. For instance for default Redis will check three keys and -# pick the one that was used less recently, you can change the sample size -# using the following configuration directive. -# -# maxmemory-samples 3 - -############################## APPEND ONLY MODE ############################### - -# By default Redis asynchronously dumps the dataset on disk. If you can live -# with the idea that the latest records will be lost if something like a crash -# happens this is the preferred way to run Redis. If instead you care a lot -# about your data and don't want to that a single record can get lost you should -# enable the append only mode: when this mode is enabled Redis will append -# every write operation received in the file appendonly.aof. This file will -# be read on startup in order to rebuild the full dataset in memory. -# -# Note that you can have both the async dumps and the append only file if you -# like (you have to comment the "save" statements above to disable the dumps). -# Still if append only mode is enabled Redis will load the data from the -# log file at startup ignoring the dump.rdb file. -# -# IMPORTANT: Check the BGREWRITEAOF to check how to rewrite the append -# log file in background when it gets too big. - appendonly no - -# The name of the append only file (default: "appendonly.aof") -# appendfilename appendonly.aof - -# The fsync() call tells the Operating System to actually write data on disk -# instead to wait for more data in the output buffer. Some OS will really flush -# data on disk, some other OS will just try to do it ASAP. -# -# Redis supports three different modes: -# -# no: don't fsync, just let the OS flush the data when it wants. Faster. -# always: fsync after every write to the append only log . Slow, Safest. -# everysec: fsync only if one second passed since the last fsync. Compromise. -# -# The default is "everysec" that's usually the right compromise between -# speed and data safety. It's up to you to understand if you can relax this to -# "no" that will will let the operating system flush the output buffer when -# it wants, for better performances (but if you can live with the idea of -# some data loss consider the default persistence mode that's snapshotting), -# or on the contrary, use "always" that's very slow but a bit safer than -# everysec. -# -# If unsure, use "everysec". - -# appendfsync always appendfsync everysec -# appendfsync no - -# When the AOF fsync policy is set to always or everysec, and a background -# saving process (a background save or AOF log background rewriting) is -# performing a lot of I/O against the disk, in some Linux configurations -# Redis may block too long on the fsync() call. Note that there is no fix for -# this currently, as even performing fsync in a different thread will block -# our synchronous write(2) call. -# -# In order to mitigate this problem it's possible to use the following option -# that will prevent fsync() from being called in the main process while a -# BGSAVE or BGREWRITEAOF is in progress. -# -# This means that while another child is saving the durability of Redis is -# the same as "appendfsync none", that in pratical terms means that it is -# possible to lost up to 30 seconds of log in the worst scenario (with the -# default Linux settings). -# -# If you have latency problems turn this to "yes". Otherwise leave it as -# "no" that is the safest pick from the point of view of durability. no-appendfsync-on-rewrite no - -############################### ADVANCED CONFIG ############################### - -# Hashes are encoded in a special way (much more memory efficient) when they -# have at max a given numer of elements, and the biggest element does not -# exceed a given threshold. You can configure this limits with the following -# configuration directives. -hash-max-ziplist-entries 64 -hash-max-ziplist-value 512 - -# Similarly to hashes, small lists are also encoded in a special way in order -# to save a lot of space. The special representation is only used when -# you are under the following limits: -list-max-ziplist-entries 512 -list-max-ziplist-value 64 - -# Sets have a special encoding in just one case: when a set is composed -# of just strings that happens to be integers in radix 10 in the range -# of 64 bit signed integers. -# The following configuration setting sets the limit in the size of the -# set in order to use this special memory saving encoding. -set-max-intset-entries 512 - -# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in -# order to help rehashing the main Redis hash table (the one mapping top-level -# keys to values). The hash table implementation redis uses (see dict.c) -# performs a lazy rehashing: the more operation you run into an hash table -# that is rhashing, the more rehashing "steps" are performed, so if the -# server is idle the rehashing is never complete and some more memory is used -# by the hash table. -# -# The default is to use this millisecond 10 times every second in order to -# active rehashing the main dictionaries, freeing memory when possible. -# -# If unsure: -# use "activerehashing no" if you have hard latency requirements and it is -# not a good thing in your environment that Redis can reply form time to time -# to queries with 2 milliseconds delay. -# -# use "activerehashing yes" if you don't have such hard requirements but -# want to free memory asap when possible. activerehashing yes - -################################## INCLUDES ################################### - -# Include one or more other config files here. This is useful if you -# have a standard template that goes to all redis server but also need -# to customize a few per-server settings. Include files can include -# other files, so use this wisely. -# -# include /path/to/local.conf -# include /path/to/other.conf From 0d7b10ee6ef790b15af70c6e9b649196f12bb9dc Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Apr 2013 13:11:41 +0200 Subject: [PATCH 0577/1752] redis-cli: --latency-history mode implemented. --- src/redis-cli.c | 62 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index 3953c10f120..d467bb4270e 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -72,6 +72,7 @@ static struct config { int monitor_mode; int pubsub_mode; int latency_mode; + int latency_history; int cluster_mode; int cluster_reissue_command; int slave_mode; @@ -700,6 +701,9 @@ static int parseOptions(int argc, char **argv) { config.output = OUTPUT_CSV; } else if (!strcmp(argv[i],"--latency")) { config.latency_mode = 1; + } else if (!strcmp(argv[i],"--latency-history")) { + config.latency_mode = 1; + config.latency_history = 1; } else if (!strcmp(argv[i],"--slave")) { config.slave_mode = 1; } else if (!strcmp(argv[i],"--stat")) { @@ -753,26 +757,29 @@ static void usage() { "redis-cli %s\n" "\n" "Usage: redis-cli [OPTIONS] [cmd [arg [arg ...]]]\n" -" -h Server hostname (default: 127.0.0.1)\n" -" -p Server port (default: 6379)\n" -" -s Server socket (overrides hostname and port)\n" -" -a Password to use when connecting to the server\n" -" -r Execute specified command N times\n" -" -i When -r is used, waits seconds per command.\n" -" It is possible to specify sub-second times like -i 0.1\n" -" -n Database number\n" -" -x Read last argument from STDIN\n" -" -d Multi-bulk delimiter in for raw formatting (default: \\n)\n" -" -c Enable cluster mode (follow -ASK and -MOVED redirections)\n" -" --raw Use raw formatting for replies (default when STDOUT is not a tty)\n" -" --latency Enter a special mode continuously sampling latency\n" -" --slave Simulate a slave showing commands received from the master\n" -" --rdb Transfer an RDB dump from remote server to local file.\n" -" --pipe Transfer raw Redis protocol from stdin to server\n" -" --bigkeys Sample Redis keys looking for big keys\n" -" --eval Send an EVAL command using the Lua script at \n" -" --help Output this help and exit\n" -" --version Output version and exit\n" +" -h Server hostname (default: 127.0.0.1)\n" +" -p Server port (default: 6379)\n" +" -s Server socket (overrides hostname and port)\n" +" -a Password to use when connecting to the server\n" +" -r Execute specified command N times\n" +" -i When -r is used, waits seconds per command.\n" +" It is possible to specify sub-second times like -i 0.1\n" +" -n Database number\n" +" -x Read last argument from STDIN\n" +" -d Multi-bulk delimiter in for raw formatting (default: \\n)\n" +" -c Enable cluster mode (follow -ASK and -MOVED redirections)\n" +" --raw Use raw formatting for replies (default when STDOUT is\n" +" not a tty)\n" +" --latency Enter a special mode continuously sampling latency\n" +" --latency-history Like --latency but tracking latency changes over time.\n" +" Default time interval is 15 sec. Change it using -i.\n" +" --slave Simulate a slave showing commands received from the master\n" +" --rdb Transfer an RDB dump from remote server to local file.\n" +" --pipe Transfer raw Redis protocol from stdin to server\n" +" --bigkeys Sample Redis keys looking for big keys\n" +" --eval Send an EVAL command using the Lua script at \n" +" --help Output this help and exit\n" +" --version Output version and exit\n" "\n" "Examples:\n" " cat /etc/passwd | redis-cli -x set mypasswd\n" @@ -943,10 +950,16 @@ static int evalMode(int argc, char **argv) { return cliSendCommand(argc+3-got_comma, argv2, config.repeat); } +#define LATENCY_SAMPLE_RATE 10 /* milliseconds. */ +#define LATENCY_HISTORY_DEFAULT_INTERVAL 15000 /* milliseconds. */ static void latencyMode(void) { redisReply *reply; long long start, latency, min = 0, max = 0, tot = 0, count = 0; + long long history_interval = + config.interval ? config.interval/1000 : + LATENCY_HISTORY_DEFAULT_INTERVAL; double avg; + long long history_start = mstime(); if (!context) exit(1); while(1) { @@ -971,7 +984,13 @@ static void latencyMode(void) { printf("\x1b[0G\x1b[2Kmin: %lld, max: %lld, avg: %.2f (%lld samples)", min, max, avg, count); fflush(stdout); - usleep(10000); + if (config.latency_history && mstime()-history_start > history_interval) + { + printf(" -- %.2f seconds range\n", (float)(mstime()-history_start)/1000); + history_start = mstime(); + min = max = tot = count = 0; + } + usleep(LATENCY_SAMPLE_RATE * 1000); } } @@ -1452,6 +1471,7 @@ int main(int argc, char **argv) { config.monitor_mode = 0; config.pubsub_mode = 0; config.latency_mode = 0; + config.latency_history = 0; config.cluster_mode = 0; config.slave_mode = 0; config.getrdb_mode = 0; From 42ba5d3788770b1f9d9348ffde6a8fa3917177c0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Apr 2013 13:15:20 +0200 Subject: [PATCH 0578/1752] redis-cli: raise error on bad command line switch. Previously redis-cli never tried to raise an error when an unrecognized switch was encountered, as everything after the initial options is to be transmitted to the server. However this is too liberal, as there are no commands starting with "-". So the new behavior is to produce an error if there is an unrecognized switch starting with "-". This should not break past redis-cli usages but should prevent broken options to be silently discarded. As far the first token not starting with "-" is encountered, all the rest is considered to be part of the command, so you cna still use strings starting with "-" as values, like in: redis-cli --port 6380 set foo --my-value --- src/redis-cli.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index d467bb4270e..f12059ca4bd 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -728,7 +728,15 @@ static int parseOptions(int argc, char **argv) { sdsfree(version); exit(0); } else { - break; + if (argv[i][0] == '-') { + fprintf(stderr, + "Unrecognized option or bad number of args for: '%s'\n", + argv[i]); + exit(1); + } else { + /* Likely the command name, stop here. */ + break; + } } } return i; From 0e3baa52417c7142c0f6cdd97978a12ea0f18453 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 18 Apr 2013 16:21:32 +0200 Subject: [PATCH 0579/1752] Redis/Jemalloc Gitignore were too aggressive. Redis gitignore was too aggressive since simply broken. Jemalloc gitignore was too agressive because it is conceived to just keep the files that allow to generate all the rest in development environments (so for instance the "configure" file is excluded). --- .gitignore | 9 +++++++-- deps/jemalloc/.gitignore | 5 ----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 89afeb0f79b..cf904522d4c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,13 @@ .*.swp *.o -*.rdb *.log -redis-* +dump.rdb +redis-benchmark +redis-check-aof +redis-check-dump +redis-cli +redis-sentinel +redis-server doc-tools release misc/* diff --git a/deps/jemalloc/.gitignore b/deps/jemalloc/.gitignore index 6607a5fd17f..0406413a0eb 100644 --- a/deps/jemalloc/.gitignore +++ b/deps/jemalloc/.gitignore @@ -2,24 +2,19 @@ /config.stamp /config.log /config.status -/configure /doc/html.xsl /doc/manpages.xsl /doc/jemalloc.xml -/doc/jemalloc.html -/doc/jemalloc.3 /lib/ /Makefile /include/jemalloc/internal/jemalloc_internal\.h /include/jemalloc/internal/size_classes\.h /include/jemalloc/jemalloc\.h /include/jemalloc/jemalloc_defs\.h -/test/jemalloc_test\.h /src/*.[od] /test/*.[od] /test/*.out /test/[a-zA-Z_]* !test/*.c !test/*.exp -/VERSION /bin/jemalloc.sh From 3a865829a735bc6627b48a22c02aaa7afb12dfb4 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 19 Apr 2013 15:11:34 +0200 Subject: [PATCH 0580/1752] More explicit panic message on out of memory. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 67740c0b651..52bd0c0eb4e 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2674,7 +2674,7 @@ void loadDataFromDisk(void) { void redisOutOfMemoryHandler(size_t allocation_size) { redisLog(REDIS_WARNING,"Out Of Memory allocating %zu bytes!", allocation_size); - redisPanic("OOM"); + redisPanic("Redis aborting for OUT OF MEMORY"); } void redisSetProcTitle(char *title) { From a500c120ba48ab5cf49d9b71e99cd44106b3d087 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 22 Apr 2013 11:25:44 +0200 Subject: [PATCH 0581/1752] Test: split conceptually unrelated comments in RDB test. --- tests/integration/rdb.tcl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index 9322ef2af03..fdc9e42d4a4 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -63,8 +63,10 @@ start_server_and_kill_it [list "dir" $server_path] { } } -# Fix permissions of the RDB file, but corrupt its CRC64 checksum. +# Fix permissions of the RDB file. file attributes [file join $server_path dump.rdb] -permissions 0666 + +# Corrupt its CRC64 checksum. set filesize [file size [file join $server_path dump.rdb]] set fd [open [file join $server_path dump.rdb] r+] fconfigure $fd -translation binary From d0c9a2a76723e6882d2eadd4992ba9c9d63c7a9e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 19 Apr 2013 16:34:07 +0200 Subject: [PATCH 0582/1752] Sentinel: turn old master into a slave when it comes back. --- src/sentinel.c | 42 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index fc857344cc0..1932c92fe3b 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -74,6 +74,8 @@ typedef struct sentinelAddr { #define SRI_RECONF_DONE (1<<13) /* Slave synchronized with new master. */ #define SRI_FORCE_FAILOVER (1<<14) /* Force failover with master up. */ #define SRI_SCRIPT_KILL_SENT (1<<15) /* SCRIPT KILL already sent on -BUSY */ +#define SRI_DEMOTE (1<<16) /* If the instance claims to be a master, demote + it into a slave sending SLAVEOF. */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 @@ -1401,7 +1403,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * otherwise add it. */ if (sentinelRedisInstanceLookupSlave(ri,ip,atoi(port)) == NULL) { if ((slave = createSentinelRedisInstance(NULL,SRI_SLAVE,ip, - atoi(port), ri->quorum,ri)) != NULL) + atoi(port), ri->quorum, ri)) != NULL) { sentinelEvent(REDIS_NOTICE,"+slave",slave,"%@"); } @@ -1467,7 +1469,19 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Act if a slave turned into a master. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { - if (!(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && + if (ri->flags & SRI_DEMOTE) { + int retval; + + /* Old master returned back? Turn it into a slave ASAP. + * We'll clear this flag only when we have the acknowledge + * that it's a slave again. */ + retval = redisAsyncCommand(ri->cc, + sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %d", + ri->master->addr->ip, + ri->master->addr->port); + if (retval == REDIS_OK) + sentinelEvent(REDIS_NOTICE,"+demote-old-slave",ri,"%@"); + } else if (!(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && (runid_changed || first_runid)) { /* If a slave turned into master but: @@ -1564,6 +1578,12 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->failover_state_change_time = mstime(); } } + + /* Detect if the old master was demoted as slave. */ + if (role == SRI_SLAVE && ri->flags & SRI_DEMOTE) { + sentinelEvent(REDIS_NOTICE,"+slave",ri,"%@"); + ri->flags &= ~SRI_DEMOTE; + } } void sentinelInfoReplyCallback(redisAsyncContext *c, void *reply, void *privdata) { @@ -1835,6 +1855,7 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { if (ri->flags & SRI_RECONF_SENT) flags = sdscat(flags,"reconf_sent,"); if (ri->flags & SRI_RECONF_INPROG) flags = sdscat(flags,"reconf_inprog,"); if (ri->flags & SRI_RECONF_DONE) flags = sdscat(flags,"reconf_done,"); + if (ri->flags & SRI_DEMOTE) flags = sdscat(flags,"demote,"); if (sdslen(flags) != 0) flags = sdsrange(flags,0,-2); /* remove last "," */ addReplyBulkCString(c,flags); @@ -2592,6 +2613,7 @@ sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master) { mstime_t info_validity_time = mstime()-SENTINEL_INFO_VALIDITY_TIME; if (slave->flags & (SRI_S_DOWN|SRI_O_DOWN|SRI_DISCONNECTED)) continue; + if (slave->flags & SRI_DEMOTE) continue; /* Old master not yet ready. */ if (slave->last_avail_time < info_validity_time) continue; if (slave->slave_priority == 0) continue; @@ -2837,12 +2859,28 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) { void sentinelFailoverSwitchToPromotedSlave(sentinelRedisInstance *master) { sentinelRedisInstance *ref = master->promoted_slave ? master->promoted_slave : master; + sds old_master_ip; + int old_master_port; sentinelEvent(REDIS_WARNING,"+switch-master",master,"%s %s %d %s %d", master->name, master->addr->ip, master->addr->port, ref->addr->ip, ref->addr->port); + old_master_ip = sdsdup(master->addr->ip); + old_master_port = master->addr->port; sentinelResetMasterAndChangeAddress(master,ref->addr->ip,ref->addr->port); + /* If this is a real switch, that is, we have master->promoted_slave not + * NULL, then we want to add the old master as a slave of the new master, + * but flagging it with SRI_DEMOTE so that we know we'll need to send + * SLAVEOF once the old master is reachable again. */ + if (master != ref) { + /* Add the new slave, but don't generate a Sentinel event as it will + * happen later when finally the instance will claim to be a slave + * in the INFO output. */ + createSentinelRedisInstance(NULL,SRI_SLAVE|SRI_DEMOTE, + old_master_ip, old_master_port, master->quorum, master); + } + sdsfree(old_master_ip); } void sentinelFailoverStateMachine(sentinelRedisInstance *ri) { From e426ea52459756f6c0d29f3d88ed00938a7d2886 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 23 Apr 2013 14:08:42 +0200 Subject: [PATCH 0583/1752] Test: fix RDB test checking file permissions. When the test is executed using the root account, setting the permission to 222 does not work as expected, as root can read files with 222 permission. Now we skip the test if root is detected. This fixes issue #1034 and the duplicated #1040 issue. Thanks to Jan-Erik Rediger (@badboy on Github) for finding a way to reproduce the issue. --- tests/integration/rdb.tcl | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tests/integration/rdb.tcl b/tests/integration/rdb.tcl index fdc9e42d4a4..71876a6edc3 100644 --- a/tests/integration/rdb.tcl +++ b/tests/integration/rdb.tcl @@ -53,13 +53,24 @@ proc start_server_and_kill_it {overrides code} { # Make the RDB file unreadable file attributes [file join $server_path dump.rdb] -permissions 0222 +# Detect root account (it is able to read the file even with 002 perm) +set isroot 0 +catch { + open [file join $server_path dump.rdb] + set isroot 1 +} + # Now make sure the server aborted with an error -start_server_and_kill_it [list "dir" $server_path] { - wait_for_condition 50 100 { - [string match {*Fatal error loading*} \ - [exec tail -n1 < [dict get $srv stdout]]] - } else { - fail "Server started even if RDB was unreadable!" +if {!$isroot} { + start_server_and_kill_it [list "dir" $server_path] { + test {Server should not start if RDB file can't be open} { + wait_for_condition 50 100 { + [string match {*Fatal error loading*} \ + [exec tail -n1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB was unreadable!" + } + } } } @@ -76,10 +87,12 @@ close $fd # Now make sure the server aborted with an error start_server_and_kill_it [list "dir" $server_path] { - wait_for_condition 50 100 { - [string match {*RDB checksum*} \ - [exec tail -n1 < [dict get $srv stdout]]] - } else { - fail "Server started even if RDB was corrupted!" + test {Server should not start if RDB is corrupted} { + wait_for_condition 50 100 { + [string match {*RDB checksum*} \ + [exec tail -n1 < [dict get $srv stdout]]] + } else { + fail "Server started even if RDB was corrupted!" + } } } From 5276996cf1e2b521c80444e77b27abeb361871c9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 3 Apr 2013 15:31:40 +0200 Subject: [PATCH 0584/1752] rio.c: added ability to fdatasync() from time to time while writing. --- src/rio.c | 31 ++++++++++++++++++++++++++++++- src/rio.h | 3 +++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/rio.c b/src/rio.c index d87d62fd791..b2f46a08bc1 100644 --- a/src/rio.c +++ b/src/rio.c @@ -48,9 +48,12 @@ #include "fmacros.h" #include #include +#include #include "rio.h" #include "util.h" #include "crc64.h" +#include "config.h" +#include "redis.h" /* Returns 1 or 0 for success/failure. */ static size_t rioBufferWrite(rio *r, const void *buf, size_t len) { @@ -75,7 +78,18 @@ static off_t rioBufferTell(rio *r) { /* Returns 1 or 0 for success/failure. */ static size_t rioFileWrite(rio *r, const void *buf, size_t len) { - return fwrite(buf,len,1,r->io.file.fp); + size_t retval; + + retval = fwrite(buf,len,1,r->io.file.fp); + r->io.file.buffered += len; + + if (r->io.file.autosync && + r->io.file.buffered >= r->io.file.autosync) + { + aof_fsync(fileno(r->io.file.fp)); + r->io.file.buffered = 0; + } + return retval; } /* Returns 1 or 0 for success/failure. */ @@ -109,6 +123,8 @@ static const rio rioFileIO = { void rioInitWithFile(rio *r, FILE *fp) { *r = rioFileIO; r->io.file.fp = fp; + r->io.file.buffered = 0; + r->io.file.autosync = 0; } void rioInitWithBuffer(rio *r, sds s) { @@ -123,6 +139,19 @@ void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len) { r->cksum = crc64(r->cksum,buf,len); } +/* Set the file-based rio object to auto-fsync every 'bytes' file written. + * By default this is set to zero that means no automatic file sync is + * performed. + * + * This feature is useful in a few contexts since when we rely on OS write + * buffers sometimes the OS buffers way too much, resulting in too many + * disk I/O concentrated in very little time. When we fsync in an explicit + * way instead the I/O pressure is more distributed across time. */ +void rioSetAutoSync(rio *r, off_t bytes) { + redisAssert(r->read == rioFileIO.read); + r->io.file.autosync = bytes; +} + /* ------------------------------ Higher level interface --------------------------- * The following higher level functions use lower level rio.c functions to help * generating the Redis protocol for the Append Only File. */ diff --git a/src/rio.h b/src/rio.h index ba17469f832..3cab66af0f9 100644 --- a/src/rio.h +++ b/src/rio.h @@ -61,6 +61,8 @@ struct _rio { } buffer; struct { FILE *fp; + off_t buffered; /* Bytes written since last fsync. */ + off_t autosync; /* fsync after 'autosync' bytes written. */ } file; } io; }; @@ -97,5 +99,6 @@ size_t rioWriteBulkLongLong(rio *r, long long l); size_t rioWriteBulkDouble(rio *r, double d); void rioGenericUpdateChecksum(rio *r, const void *buf, size_t len); +void rioSetAutoSync(rio *r, off_t bytes); #endif From 9ca306d874a02409cdef6ff873e684a0e5cc3078 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 3 Apr 2013 18:55:38 +0200 Subject: [PATCH 0585/1752] AOF: sync data on disk every 32MB when rewriting. This prevents the kernel from putting too much stuff in the output buffers, doing too heavy I/O all at once. So the goal of this commit is to split the disk pressure due to the AOF rewrite process into smaller spikes. Please see issue #1019 for more information. --- src/aof.c | 1 + src/redis.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/aof.c b/src/aof.c index c629bed4744..ac50d572d70 100644 --- a/src/aof.c +++ b/src/aof.c @@ -855,6 +855,7 @@ int rewriteAppendOnlyFile(char *filename) { } rioInitWithFile(&aof,fp); + rioSetAutoSync(&aof,REDIS_AOF_AUTOSYNC_BYTES); for (j = 0; j < server.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; redisDb *db = server.db+j; diff --git a/src/redis.h b/src/redis.h index 4ace7eb37e4..76e6172e679 100644 --- a/src/redis.h +++ b/src/redis.h @@ -106,6 +106,7 @@ #define REDIS_INLINE_MAX_SIZE (1024*64) /* Max size of inline reads */ #define REDIS_MBULK_BIG_ARG (1024*32) #define REDIS_LONGSTR_SIZE 21 /* Bytes needed for long -> str */ +#define REDIS_AOF_AUTOSYNC_BYTES (1024*1024*32) /* fdatasync every 32MB */ /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ From b06f13e7b753aae8ad9bdeea1121a7c1d9e9b523 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Apr 2013 10:57:07 +0200 Subject: [PATCH 0586/1752] Config option to turn AOF rewrite incremental fsync on/off. --- redis.conf | 6 ++++++ src/aof.c | 3 ++- src/config.c | 13 +++++++++++++ src/redis.c | 1 + src/redis.h | 1 + 5 files changed, 23 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index f135463cd7a..581dbf971b9 100644 --- a/redis.conf +++ b/redis.conf @@ -648,6 +648,12 @@ client-output-buffer-limit pubsub 32mb 8mb 60 # 100 only in environments where very low latency is required. hz 10 +# When a child rewrites the AOF file, if the following option is enabled +# the file will be fsync-ed every 32 MB of data generated. This is useful +# in order to commit the file to the disk more incrementally and avoid +# big latency spikes. +aof-rewrite-incremental-fsync yes + ################################## INCLUDES ################################### # Include one or more other config files here. This is useful if you diff --git a/src/aof.c b/src/aof.c index ac50d572d70..56a985e727e 100644 --- a/src/aof.c +++ b/src/aof.c @@ -855,7 +855,8 @@ int rewriteAppendOnlyFile(char *filename) { } rioInitWithFile(&aof,fp); - rioSetAutoSync(&aof,REDIS_AOF_AUTOSYNC_BYTES); + if (server.aof_rewrite_incremental_fsync) + rioSetAutoSync(&aof,REDIS_AOF_AUTOSYNC_BYTES); for (j = 0; j < server.dbnum; j++) { char selectcmd[] = "*2\r\n$6\r\nSELECT\r\n"; redisDb *db = server.db+j; diff --git a/src/config.c b/src/config.c index c430cfe8ff2..227d30013c9 100644 --- a/src/config.c +++ b/src/config.c @@ -330,6 +330,12 @@ void loadServerConfigFromString(char *config) { argc == 2) { server.aof_rewrite_min_size = memtoll(argv[1],NULL); + } else if (!strcasecmp(argv[0],"aof-rewrite-incremental-fsync") && + argc == 2) + { + if ((server.aof_rewrite_incremental_fsync = yesnotoi(argv[1])) == -1) { + err = "argument must be 'yes' or 'no'"; goto loaderr; + } } else if (!strcasecmp(argv[0],"requirepass") && argc == 2) { if (strlen(argv[1]) > REDIS_AUTHPASS_MAX_LEN) { err = "Password is longer than REDIS_AUTHPASS_MAX_LEN"; @@ -587,6 +593,11 @@ void configSetCommand(redisClient *c) { } else if (!strcasecmp(c->argv[2]->ptr,"auto-aof-rewrite-min-size")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.aof_rewrite_min_size = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"aof-rewrite-incremental-fsync")) { + int yn = yesnotoi(o->ptr); + + if (yn == -1) goto badfmt; + server.aof_rewrite_incremental_fsync = yn; } else if (!strcasecmp(c->argv[2]->ptr,"save")) { int vlen, j; sds *v = sdssplitlen(o->ptr,sdslen(o->ptr)," ",1,&vlen); @@ -876,6 +887,8 @@ void configGetCommand(redisClient *c) { config_get_bool_field("activerehashing", server.activerehashing); config_get_bool_field("repl-disable-tcp-nodelay", server.repl_disable_tcp_nodelay); + config_get_bool_field("aof-rewrite-incremental-fsync", + server.aof_rewrite_incremental_fsync); /* Everything we can't handle with macros follows. */ diff --git a/src/redis.c b/src/redis.c index 52bd0c0eb4e..f7b04f08f05 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1200,6 +1200,7 @@ void initServerConfig() { server.aof_fd = -1; server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; + server.aof_rewrite_incremental_fsync = 1; server.pidfile = zstrdup("/var/run/redis.pid"); server.rdb_filename = zstrdup("dump.rdb"); server.aof_filename = zstrdup("appendonly.aof"); diff --git a/src/redis.h b/src/redis.h index 76e6172e679..a05a3ad3647 100644 --- a/src/redis.h +++ b/src/redis.h @@ -608,6 +608,7 @@ struct redisServer { time_t aof_rewrite_time_start; /* Current AOF rewrite start time. */ int aof_lastbgrewrite_status; /* REDIS_OK or REDIS_ERR */ unsigned long aof_delayed_fsync; /* delayed AOF fsync() counter */ + int aof_rewrite_incremental_fsync;/* fsync incrementally while rewriting? */ /* RDB persistence */ long long dirty; /* Changes to DB from the last save */ long long dirty_before_bgsave; /* Used to restore dirty on failed BGSAVE */ From d2ff5ed6039aaa73c8880138a5e6121e52b2673b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Apr 2013 11:30:17 +0200 Subject: [PATCH 0587/1752] Sentinel: always redirect on master->slave transition. Sentinel redirected to the master if the instance changed runid or it was the first time we got INFO, and a role change was detected from master to slave. While this is a good idea in case of slave->master, since otherwise we could detect a failover without good reasons just after a reboot with a slave with a wrong configuration, in the case of master->slave transition is much better to always perform the redirection for the following reasons: 1) A Sentinel may go down for some time. When it is back online there is no other way to understand there was a failover. 2) Pointing clients to a slave seems to be always the wrong thing to do. 3) There is no good rationale about handling things differently once an instance is rebooted (runid change) in that case. --- src/sentinel.c | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 1932c92fe3b..6e592ae1ea3 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1452,19 +1452,15 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { if (sentinel.tilt) return; /* Act if a master turned into a slave. */ - if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) { - if ((first_runid || runid_changed) && ri->slave_master_host) { - /* If it is the first time we receive INFO from it, but it's - * a slave while it was configured as a master, we want to monitor - * its master instead. */ - sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, - "%s %s %d %s %d", - ri->name, ri->addr->ip, ri->addr->port, - ri->slave_master_host, ri->slave_master_port); - sentinelResetMasterAndChangeAddress(ri,ri->slave_master_host, - ri->slave_master_port); - return; - } + if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE && ri->slave_master_host) + { + sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, + "%s %s %d %s %d", + ri->name, ri->addr->ip, ri->addr->port, + ri->slave_master_host, ri->slave_master_port); + sentinelResetMasterAndChangeAddress(ri,ri->slave_master_host, + ri->slave_master_port); + return; } /* Act if a slave turned into a master. */ From 2b427a89fb403d4a1bd50d34088e7cb58667c216 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 29 Apr 2013 12:00:54 +0200 Subject: [PATCH 0588/1752] Lua updated to version 5.1.5. --- deps/lua/COPYRIGHT | 2 +- deps/lua/Makefile | 2 +- deps/lua/doc/amazon.gif | Bin 797 -> 0 bytes deps/lua/doc/contents.html | 160 ++++++++++++++++++------------------- deps/lua/doc/lua.css | 48 ++++++++++- deps/lua/doc/manual.css | 15 +++- deps/lua/doc/manual.html | 31 +++---- deps/lua/doc/readme.html | 4 +- deps/lua/etc/lua.pc | 2 +- deps/lua/src/Makefile | 2 +- deps/lua/src/lbaselib.c | 2 +- deps/lua/src/lcode.c | 10 +-- deps/lua/src/ldblib.c | 3 +- deps/lua/src/ldo.c | 3 +- deps/lua/src/lgc.c | 3 +- deps/lua/src/liolib.c | 7 +- deps/lua/src/llex.c | 6 +- deps/lua/src/loadlib.c | 4 +- deps/lua/src/lparser.c | 4 +- deps/lua/src/lstrlib.c | 6 +- deps/lua/src/lua.h | 8 +- deps/lua/src/lvm.c | 8 +- 22 files changed, 194 insertions(+), 136 deletions(-) delete mode 100644 deps/lua/doc/amazon.gif diff --git a/deps/lua/COPYRIGHT b/deps/lua/COPYRIGHT index 3a53e741e03..a86026803a0 100644 --- a/deps/lua/COPYRIGHT +++ b/deps/lua/COPYRIGHT @@ -9,7 +9,7 @@ For details and rationale, see http://www.lua.org/license.html . =============================================================================== -Copyright (C) 1994-2008 Lua.org, PUC-Rio. +Copyright (C) 1994-2012 Lua.org, PUC-Rio. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/deps/lua/Makefile b/deps/lua/Makefile index 6e78f66fa5b..209a1324418 100644 --- a/deps/lua/Makefile +++ b/deps/lua/Makefile @@ -48,7 +48,7 @@ TO_MAN= lua.1 luac.1 # Lua version and release. V= 5.1 -R= 5.1.4 +R= 5.1.5 all: $(PLAT) diff --git a/deps/lua/doc/amazon.gif b/deps/lua/doc/amazon.gif deleted file mode 100644 index f2586d5765361bb8a33a72401449f3abdefe4e16..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 797 zcmV+&1LFKgNk%w1VOjtj0K^&q@9)h50%r{k3;+NBK0Y<5t#HiDnQw1j_x9wkua=aQ zf8O4{J3ExYz@^dAvgzr__o-L@ehvGQF7LT?@$1I#;Ij3_g2%_+^3u}mmw|9dEECm0Kc!X!@|#@3&?4Fr)=|m57&HqxlCDW&_KZ+1;`8vSj33YC_e@b zLW(B90kQ_n9Ds8N3z4jU4HiAyWz9`JYknO6suY&Mn*c`+`V13=EEIuKH*uwDKrGlz zo{tDTNaHR5yf%>5cyj}g-^zXgiM@QV=g%Gh303A)7B84EpbPvE-6X=O$DmxJ`b(9|4E82-m%Ds!U< zI=HL8TMx@{~>s4%C(a0wK8Wjd@n+|5vxOvSr(vuAh- z5-e0mzlK5u3lKm6pnB`22G3HYpfaDDC`;MbMg*4 b)M@9Pc;4WH3w-wJ=bwNED(Iksh5!INDePJ9 diff --git a/deps/lua/doc/contents.html b/deps/lua/doc/contents.html index 8e58e18ca9b..3d83da98a6d 100644 --- a/deps/lua/doc/contents.html +++ b/deps/lua/doc/contents.html @@ -3,11 +3,10 @@ Lua 5.1 Reference Manual - contents - + @@ -20,7 +19,13 @@

Lua 5.1 Reference Manual

-This is an online version of +

+The reference manual is the official definition of the Lua language. +For a complete introduction to Lua programming, see the book +Programming in Lua. + +

+This manual is also available as a book:

@@ -29,128 +34,119 @@


by R. Ierusalimschy, L. H. de Figueiredo, W. Celes
Lua.org, August 2006
ISBN 85-903798-3-3 -
 -[Buy from Amazon]

-

-Buy a copy of this book and +

+Buy a copy +of this book and help to support the Lua project. -

-The reference manual is the official definition of the Lua language. -For a complete introduction to Lua programming, see the book -Programming in Lua.

- start · contents · index · -português -· -español +other versions

-Copyright © 2006-2008 Lua.org, PUC-Rio. +Copyright © 2006–2012 Lua.org, PUC-Rio. Freely available under the terms of the -Lua license. +Lua license. -

Contents

Index

@@ -160,6 +156,8 @@

Index

Lua functions

_G
_VERSION
+

+ assert
collectgarbage
dofile
@@ -487,12 +485,12 @@

auxiliary library

- + Last update: -Sat Jan 19 13:24:29 BRST 2008 +Mon Feb 13 18:53:32 BRST 2012 diff --git a/deps/lua/doc/lua.css b/deps/lua/doc/lua.css index 039cf11698c..7fafbb1bb63 100644 --- a/deps/lua/doc/lua.css +++ b/deps/lua/doc/lua.css @@ -1,17 +1,37 @@ body { color: #000000 ; background-color: #FFFFFF ; - font-family: sans-serif ; + font-family: Helvetica, Arial, sans-serif ; text-align: justify ; - margin-right: 20px ; - margin-left: 20px ; + margin-right: 30px ; + margin-left: 30px ; } h1, h2, h3, h4 { + font-family: Verdana, Geneva, sans-serif ; font-weight: normal ; font-style: italic ; } +h2 { + padding-top: 0.4em ; + padding-bottom: 0.4em ; + padding-left: 30px ; + padding-right: 30px ; + margin-left: -30px ; + background-color: #E0E0FF ; +} + +h3 { + padding-left: 0.5em ; + border-left: solid #E0E0FF 1em ; +} + +table h3 { + padding-left: 0px ; + border-left: none ; +} + a:link { color: #000080 ; background-color: inherit ; @@ -39,3 +59,25 @@ hr { background-color: #a0a0a0 ; } +:target { + background-color: #F8F8F8 ; + padding: 8px ; + border: solid #a0a0a0 2px ; +} + +.footer { + color: gray ; + font-size: small ; +} + +input[type=text] { + border: solid #a0a0a0 2px ; + border-radius: 2em ; + -moz-border-radius: 2em ; + background-image: url('images/search.png') ; + background-repeat: no-repeat; + background-position: 4px center ; + padding-left: 20px ; + height: 2em ; +} + diff --git a/deps/lua/doc/manual.css b/deps/lua/doc/manual.css index eed5afd9eef..b49b362937a 100644 --- a/deps/lua/doc/manual.css +++ b/deps/lua/doc/manual.css @@ -1,13 +1,24 @@ h3 code { font-family: inherit ; + font-size: inherit ; } -pre { - font-size: 105% ; +pre, code { + font-size: 12pt ; } span.apii { float: right ; font-family: inherit ; + font-style: normal ; + font-size: small ; + color: gray ; } +p+h1, ul+h1 { + padding-top: 0.4em ; + padding-bottom: 0.4em ; + padding-left: 30px ; + margin-left: -30px ; + background-color: #E0E0FF ; +} diff --git a/deps/lua/doc/manual.html b/deps/lua/doc/manual.html index f46f17c8ec1..4e41683d028 100644 --- a/deps/lua/doc/manual.html +++ b/deps/lua/doc/manual.html @@ -19,9 +19,9 @@

by Roberto Ierusalimschy, Luiz Henrique de Figueiredo, Waldemar Celes

-Copyright © 2006-2008 Lua.org, PUC-Rio. +Copyright © 2006–2012 Lua.org, PUC-Rio. Freely available under the terms of the -Lua license. +Lua license.

@@ -29,11 +29,13 @@

contents · index +· +other versions

- + @@ -5645,7 +5647,7 @@

5 - Standard Libraries

    -
  • basic library,
    • which includes the coroutine sub-library; +
  • basic library, which includes the coroutine sub-library;
  • package library;
    • @@ -5709,7 +5711,7 @@

    5.1 - Basic Functions

    -

    collectgarbage (opt [, arg])

    +

    collectgarbage ([opt [, arg]])

    @@ -5718,6 +5720,11 @@

    5.1 - Basic Functions

      +
    • "collect": +performs a full garbage-collection cycle. +This is the default option. +
      • +
    • "stop": stops the garbage collector.
      • @@ -5726,10 +5733,6 @@

      5.1 - Basic Functions

      restarts the garbage collector. -
    • "collect": -performs a full garbage-collection cycle. -
      • -
    • "count": returns the total memory in use by Lua (in Kbytes).
      • @@ -5760,7 +5763,7 @@

      5.1 - Basic Functions

      -

      dofile (filename)

      +

      dofile ([filename])

      Opens the named file and executes its contents as a Lua chunk. When called without arguments, dofile executes the contents of the standard input (stdin). @@ -8421,7 +8424,7 @@

      5.9 - The Debug Library

      -

      debug.traceback ([thread,] [message] [, level])

      +

      debug.traceback ([thread,] [message [, level]])

      @@ -8789,12 +8792,12 @@

      8 - The Complete Syntax of Lua

      - + Last update: -Mon Aug 18 13:25:46 BRT 2008 +Mon Feb 13 18:54:19 BRST 2012 diff --git a/deps/lua/doc/readme.html b/deps/lua/doc/readme.html index 38be6dbbfc7..3ed6a81895b 100644 --- a/deps/lua/doc/readme.html +++ b/deps/lua/doc/readme.html @@ -12,7 +12,7 @@

      Documentation

      -This is the documentation included in the source distribution of Lua 5.1.4. +This is the documentation included in the source distribution of Lua 5.1.5.
      • Reference manual @@ -33,7 +33,7 @@

        Last update: -Tue Aug 12 14:46:07 BRT 2008 +Fri Feb 3 09:44:42 BRST 2012 diff --git a/deps/lua/etc/lua.pc b/deps/lua/etc/lua.pc index f52f55b0123..07e2852b0a7 100644 --- a/deps/lua/etc/lua.pc +++ b/deps/lua/etc/lua.pc @@ -5,7 +5,7 @@ # grep '^V=' ../Makefile V= 5.1 # grep '^R=' ../Makefile -R= 5.1.4 +R= 5.1.5 # grep '^INSTALL_.*=' ../Makefile | sed 's/INSTALL_TOP/prefix/' prefix= /usr/local diff --git a/deps/lua/src/Makefile b/deps/lua/src/Makefile index f5c45fcd3e8..34b0c361734 100644 --- a/deps/lua/src/Makefile +++ b/deps/lua/src/Makefile @@ -48,7 +48,7 @@ o: $(ALL_O) a: $(ALL_A) $(LUA_A): $(CORE_O) $(LIB_O) - $(AR) $@ $? + $(AR) $@ $(CORE_O) $(LIB_O) # DLL needs all object files $(RANLIB) $@ $(LUA_T): $(LUA_O) $(LUA_A) diff --git a/deps/lua/src/lbaselib.c b/deps/lua/src/lbaselib.c index 2a4c079d3b0..2ab550bd48d 100644 --- a/deps/lua/src/lbaselib.c +++ b/deps/lua/src/lbaselib.c @@ -631,7 +631,7 @@ static void base_open (lua_State *L) { luaL_register(L, "_G", base_funcs); lua_pushliteral(L, LUA_VERSION); lua_setglobal(L, "_VERSION"); /* set global _VERSION */ - /* `ipairs' and `pairs' need auxliliary functions as upvalues */ + /* `ipairs' and `pairs' need auxiliary functions as upvalues */ auxopen(L, "ipairs", luaB_ipairs, ipairsaux); auxopen(L, "pairs", luaB_pairs, luaB_next); /* `newproxy' needs a weaktable as upvalue */ diff --git a/deps/lua/src/lcode.c b/deps/lua/src/lcode.c index cff626b7fa6..679cb9cfd98 100644 --- a/deps/lua/src/lcode.c +++ b/deps/lua/src/lcode.c @@ -1,5 +1,5 @@ /* -** $Id: lcode.c,v 2.25.1.3 2007/12/28 15:32:23 roberto Exp $ +** $Id: lcode.c,v 2.25.1.5 2011/01/31 14:53:16 roberto Exp $ ** Code generator for Lua ** See Copyright Notice in lua.h */ @@ -544,10 +544,6 @@ void luaK_goiftrue (FuncState *fs, expdesc *e) { pc = NO_JUMP; /* always true; do nothing */ break; } - case VFALSE: { - pc = luaK_jump(fs); /* always jump */ - break; - } case VJMP: { invertjump(fs, e); pc = e->u.s.info; @@ -572,10 +568,6 @@ static void luaK_goiffalse (FuncState *fs, expdesc *e) { pc = NO_JUMP; /* always false; do nothing */ break; } - case VTRUE: { - pc = luaK_jump(fs); /* always jump */ - break; - } case VJMP: { pc = e->u.s.info; break; diff --git a/deps/lua/src/ldblib.c b/deps/lua/src/ldblib.c index 67de1222a94..2027eda5983 100644 --- a/deps/lua/src/ldblib.c +++ b/deps/lua/src/ldblib.c @@ -1,5 +1,5 @@ /* -** $Id: ldblib.c,v 1.104.1.3 2008/01/21 13:11:21 roberto Exp $ +** $Id: ldblib.c,v 1.104.1.4 2009/08/04 18:50:18 roberto Exp $ ** Interface from Lua to its debug API ** See Copyright Notice in lua.h */ @@ -45,6 +45,7 @@ static int db_setmetatable (lua_State *L) { static int db_getfenv (lua_State *L) { + luaL_checkany(L, 1); lua_getfenv(L, 1); return 1; } diff --git a/deps/lua/src/ldo.c b/deps/lua/src/ldo.c index 8de05f728e7..d1bf786cb72 100644 --- a/deps/lua/src/ldo.c +++ b/deps/lua/src/ldo.c @@ -1,5 +1,5 @@ /* -** $Id: ldo.c,v 2.38.1.3 2008/01/18 22:31:22 roberto Exp $ +** $Id: ldo.c,v 2.38.1.4 2012/01/18 02:27:10 roberto Exp $ ** Stack and Call structure of Lua ** See Copyright Notice in lua.h */ @@ -217,6 +217,7 @@ static StkId adjust_varargs (lua_State *L, Proto *p, int actual) { int nvar = actual - nfixargs; /* number of extra arguments */ lua_assert(p->is_vararg & VARARG_HASARG); luaC_checkGC(L); + luaD_checkstack(L, p->maxstacksize); htab = luaH_new(L, nvar, 1); /* create `arg' table */ for (i=0; itop - nvar + i); diff --git a/deps/lua/src/lgc.c b/deps/lua/src/lgc.c index d9e0b78294e..e909c79a969 100644 --- a/deps/lua/src/lgc.c +++ b/deps/lua/src/lgc.c @@ -1,5 +1,5 @@ /* -** $Id: lgc.c,v 2.38.1.1 2007/12/27 13:02:25 roberto Exp $ +** $Id: lgc.c,v 2.38.1.2 2011/03/18 18:05:38 roberto Exp $ ** Garbage Collector ** See Copyright Notice in lua.h */ @@ -627,7 +627,6 @@ void luaC_step (lua_State *L) { } } else { - lua_assert(g->totalbytes >= g->estimate); setthreshold(g); } } diff --git a/deps/lua/src/liolib.c b/deps/lua/src/liolib.c index e79ed1cb2e2..649f9a59515 100644 --- a/deps/lua/src/liolib.c +++ b/deps/lua/src/liolib.c @@ -1,5 +1,5 @@ /* -** $Id: liolib.c,v 2.73.1.3 2008/01/18 17:47:43 roberto Exp $ +** $Id: liolib.c,v 2.73.1.4 2010/05/14 15:33:51 roberto Exp $ ** Standard I/O (and system) library ** See Copyright Notice in lua.h */ @@ -276,7 +276,10 @@ static int read_number (lua_State *L, FILE *f) { lua_pushnumber(L, d); return 1; } - else return 0; /* read fails */ + else { + lua_pushnil(L); /* "result" to be removed */ + return 0; /* read fails */ + } } diff --git a/deps/lua/src/llex.c b/deps/lua/src/llex.c index 6dc319358c0..88c6790c076 100644 --- a/deps/lua/src/llex.c +++ b/deps/lua/src/llex.c @@ -1,5 +1,5 @@ /* -** $Id: llex.c,v 2.20.1.1 2007/12/27 13:02:25 roberto Exp $ +** $Id: llex.c,v 2.20.1.2 2009/11/23 14:58:22 roberto Exp $ ** Lexical Analyzer ** See Copyright Notice in lua.h */ @@ -118,8 +118,10 @@ TString *luaX_newstring (LexState *ls, const char *str, size_t l) { lua_State *L = ls->L; TString *ts = luaS_newlstr(L, str, l); TValue *o = luaH_setstr(L, ls->fs->h, ts); /* entry for `str' */ - if (ttisnil(o)) + if (ttisnil(o)) { setbvalue(o, 1); /* make sure `str' will not be collected */ + luaC_checkGC(L); + } return ts; } diff --git a/deps/lua/src/loadlib.c b/deps/lua/src/loadlib.c index 0d401eba1cf..6158c5353df 100644 --- a/deps/lua/src/loadlib.c +++ b/deps/lua/src/loadlib.c @@ -1,5 +1,5 @@ /* -** $Id: loadlib.c,v 1.52.1.3 2008/08/06 13:29:28 roberto Exp $ +** $Id: loadlib.c,v 1.52.1.4 2009/09/09 13:17:16 roberto Exp $ ** Dynamic library loader for Lua ** See Copyright Notice in lua.h ** @@ -639,7 +639,7 @@ LUALIB_API int luaopen_package (lua_State *L) { lua_pushvalue(L, -1); lua_replace(L, LUA_ENVIRONINDEX); /* create `loaders' table */ - lua_createtable(L, 0, sizeof(loaders)/sizeof(loaders[0]) - 1); + lua_createtable(L, sizeof(loaders)/sizeof(loaders[0]) - 1, 0); /* fill it with pre-defined loaders */ for (i=0; loaders[i] != NULL; i++) { lua_pushcfunction(L, loaders[i]); diff --git a/deps/lua/src/lparser.c b/deps/lua/src/lparser.c index 1e2a9a88b79..dda7488dcad 100644 --- a/deps/lua/src/lparser.c +++ b/deps/lua/src/lparser.c @@ -1,5 +1,5 @@ /* -** $Id: lparser.c,v 2.42.1.3 2007/12/28 15:32:23 roberto Exp $ +** $Id: lparser.c,v 2.42.1.4 2011/10/21 19:31:42 roberto Exp $ ** Lua Parser ** See Copyright Notice in lua.h */ @@ -374,9 +374,9 @@ static void close_func (LexState *ls) { lua_assert(luaG_checkcode(f)); lua_assert(fs->bl == NULL); ls->fs = fs->prev; - L->top -= 2; /* remove table and prototype from the stack */ /* last token read was anchored in defunct function; must reanchor it */ if (fs) anchor_token(ls); + L->top -= 2; /* remove table and prototype from the stack */ } diff --git a/deps/lua/src/lstrlib.c b/deps/lua/src/lstrlib.c index 1b4763d4ee1..7a03489bebd 100644 --- a/deps/lua/src/lstrlib.c +++ b/deps/lua/src/lstrlib.c @@ -1,5 +1,5 @@ /* -** $Id: lstrlib.c,v 1.132.1.4 2008/07/11 17:27:21 roberto Exp $ +** $Id: lstrlib.c,v 1.132.1.5 2010/05/14 15:34:19 roberto Exp $ ** Standard library for string operations and pattern-matching ** See Copyright Notice in lua.h */ @@ -754,6 +754,7 @@ static void addintlen (char *form) { static int str_format (lua_State *L) { + int top = lua_gettop(L); int arg = 1; size_t sfl; const char *strfrmt = luaL_checklstring(L, arg, &sfl); @@ -768,7 +769,8 @@ static int str_format (lua_State *L) { else { /* format item */ char form[MAX_FORMAT]; /* to store the format (`%...') */ char buff[MAX_ITEM]; /* to store the formatted item */ - arg++; + if (++arg > top) + luaL_argerror(L, arg, "no value"); strfrmt = scanformat(L, strfrmt, form); switch (*strfrmt++) { case 'c': { diff --git a/deps/lua/src/lua.h b/deps/lua/src/lua.h index e4bdfd3b94f..a4b73e743ed 100644 --- a/deps/lua/src/lua.h +++ b/deps/lua/src/lua.h @@ -1,5 +1,5 @@ /* -** $Id: lua.h,v 1.218.1.5 2008/08/06 13:30:12 roberto Exp $ +** $Id: lua.h,v 1.218.1.7 2012/01/13 20:36:20 roberto Exp $ ** Lua - An Extensible Extension Language ** Lua.org, PUC-Rio, Brazil (http://www.lua.org) ** See Copyright Notice at the end of this file @@ -17,9 +17,9 @@ #define LUA_VERSION "Lua 5.1" -#define LUA_RELEASE "Lua 5.1.4" +#define LUA_RELEASE "Lua 5.1.5" #define LUA_VERSION_NUM 501 -#define LUA_COPYRIGHT "Copyright (C) 1994-2008 Lua.org, PUC-Rio" +#define LUA_COPYRIGHT "Copyright (C) 1994-2012 Lua.org, PUC-Rio" #define LUA_AUTHORS "R. Ierusalimschy, L. H. de Figueiredo & W. Celes" @@ -362,7 +362,7 @@ struct lua_Debug { /****************************************************************************** -* Copyright (C) 1994-2008 Lua.org, PUC-Rio. All rights reserved. +* Copyright (C) 1994-2012 Lua.org, PUC-Rio. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining * a copy of this software and associated documentation files (the diff --git a/deps/lua/src/lvm.c b/deps/lua/src/lvm.c index ee3256ab94d..e0a0cd85219 100644 --- a/deps/lua/src/lvm.c +++ b/deps/lua/src/lvm.c @@ -1,5 +1,5 @@ /* -** $Id: lvm.c,v 2.63.1.3 2007/12/28 15:32:23 roberto Exp $ +** $Id: lvm.c,v 2.63.1.5 2011/08/17 20:43:11 roberto Exp $ ** Lua virtual machine ** See Copyright Notice in lua.h */ @@ -133,6 +133,7 @@ void luaV_gettable (lua_State *L, const TValue *t, TValue *key, StkId val) { void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { int loop; + TValue temp; for (loop = 0; loop < MAXTAGLOOP; loop++) { const TValue *tm; if (ttistable(t)) { /* `t' is a table? */ @@ -141,6 +142,7 @@ void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { if (!ttisnil(oldval) || /* result is no nil? */ (tm = fasttm(L, h->metatable, TM_NEWINDEX)) == NULL) { /* or no TM? */ setobj2t(L, oldval, val); + h->flags = 0; luaC_barriert(L, h, val); return; } @@ -152,7 +154,9 @@ void luaV_settable (lua_State *L, const TValue *t, TValue *key, StkId val) { callTM(L, tm, t, key, val); return; } - t = tm; /* else repeat with `tm' */ + /* else repeat with `tm' */ + setobj(L, &temp, tm); /* avoid pointing inside table (may rehash) */ + t = &temp; } luaG_runerror(L, "loop in settable"); } From 70845320ccf551687e09bcc632dd72ff509b7881 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 26 Apr 2013 11:11:21 +0200 Subject: [PATCH 0589/1752] Sentinel: only demote old master into slave under certain conditions. We used to always turn a master into a slave if the DEMOTE flag was set, as this was a resurrecting master instance. However the following race condition is possible for a Sentinel that got partitioned or internal issues (tilt mode), and was not able to refresh the state in the meantime: 1) Sentinel X is running, master is instance "A". 3) "A" fails, sentinels will promote slave "B" as master. 2) Sentinel X goes down because of a network partition. 4) "A" returns available, Sentinels will demote it as a slave. 5) "B" fails, other Sentinels will promote slave "A" as master. 6) At this point Sentinel X comes back. When "X" comes back he thinks that: "B" is the master. "A" is the slave to demote. We want to avoid that Sentinel "X" will demote "A" into a slave. We also want that Sentinel "X" will detect that the conditions changed and will reconfigure itself to monitor the right master. There are two main ways for the Sentinel to reconfigure itself after this event: 1) If "B" is reachable and already configured as a slave by other sentinels, "X" will perform a redirection to "A". 2) If there are not the conditions to demote "A", the fact that "A" reports to be a master will trigger a failover detection in "X", that will end into a reconfiguraiton to monitor "A". However if the Sentinel was not reachable, its state may not be updated, so in case it titled, or was partiitoned from the master instance of the slave to demote, the new implementation waits some time (enough to guarantee we can detect the new INFO, and new DOWN conditions). If after some time still there are not the right condiitons to demote the instance, the DEMOTE flag is cleared. --- src/sentinel.c | 57 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 46 insertions(+), 11 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 6e592ae1ea3..959f26e352e 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -405,7 +405,7 @@ void initSentinel(void) { /* Initialize various data structures. */ sentinel.masters = dictCreate(&instancesDictType,NULL); sentinel.tilt = 0; - sentinel.tilt_start_time = mstime(); + sentinel.tilt_start_time = 0; sentinel.previous_time = mstime(); sentinel.running_scripts = 0; sentinel.scripts_queue = listCreate(); @@ -1132,7 +1132,6 @@ int sentinelResetMastersByPattern(char *pattern, int flags) { * TODO: make this reset so that original sentinels are re-added with * same ip / port / runid. */ - int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, int port) { sentinelAddr *oldaddr, *newaddr; @@ -1141,12 +1140,26 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, sentinelResetMaster(master,SENTINEL_NO_FLAGS); oldaddr = master->addr; master->addr = newaddr; + master->o_down_since_time = 0; + master->s_down_since_time = 0; + /* Release the old address at the end so we are safe even if the function * gets the master->addr->ip and master->addr->port as arguments. */ releaseSentinelAddr(oldaddr); return REDIS_OK; } +/* Return non-zero if there was no SDOWN or ODOWN error associated to this + * instance in the latest 'ms' milliseconds. */ +int sentinelRedisInstanceNoDownFor(sentinelRedisInstance *ri, mstime_t ms) { + mstime_t most_recent; + + most_recent = ri->s_down_since_time; + if (ri->o_down_since_time > most_recent) + most_recent = ri->o_down_since_time; + return most_recent == 0 || (mstime() - most_recent) > ms; +} + /* ============================ Config handling ============================= */ char *sentinelHandleConfiguration(char **argv, int argc) { sentinelRedisInstance *ri; @@ -1466,17 +1479,37 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Act if a slave turned into a master. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { if (ri->flags & SRI_DEMOTE) { - int retval; + /* If this sentinel was partitioned from the slave's master, + * or tilted recently, wait some time before to act, + * so that DOWN and roles info will be refreshed. */ + if (!sentinelRedisInstanceNoDownFor(ri->master, + SENTINEL_INFO_PERIOD*2)) + return; + if (mstime()-sentinel.tilt_start_time < + SENTINEL_TILT_PERIOD+ri->master->down_after_period*2) + return; - /* Old master returned back? Turn it into a slave ASAP. + /* Old master returned back? Turn it into a slave ASAP if: + * * We'll clear this flag only when we have the acknowledge * that it's a slave again. */ - retval = redisAsyncCommand(ri->cc, - sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %d", - ri->master->addr->ip, - ri->master->addr->port); - if (retval == REDIS_OK) - sentinelEvent(REDIS_NOTICE,"+demote-old-slave",ri,"%@"); + if (ri->master->flags & SRI_MASTER && + (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && + (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) + { + int retval; + retval = redisAsyncCommand(ri->cc, + sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %d", + ri->master->addr->ip, + ri->master->addr->port); + if (retval == REDIS_OK) + sentinelEvent(REDIS_NOTICE,"+demote-old-slave",ri,"%@"); + } else { + /* Otherwise if there are not the conditions to demote, we + * no longer trust the DEMOTE flag and remove it. */ + ri->flags &= ~SRI_DEMOTE; + sentinelEvent(REDIS_NOTICE,"-demote-flag-cleared",ri,"%@"); + } } else if (!(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && (runid_changed || first_runid)) { @@ -1533,6 +1566,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->master->failover_state_change_time = mstime(); ri->master->promoted_slave = ri; ri->flags |= SRI_PROMOTED; + ri->flags &= ~SRI_DEMOTE; sentinelCallClientReconfScript(ri->master,SENTINEL_OBSERVER, "start", ri->master->addr,ri->addr); /* We are an observer, so we can only assume that the leader @@ -1575,7 +1609,8 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { } } - /* Detect if the old master was demoted as slave. */ + /* Detect if the old master was demoted as slave and generate the + * +slave event. */ if (role == SRI_SLAVE && ri->flags & SRI_DEMOTE) { sentinelEvent(REDIS_NOTICE,"+slave",ri,"%@"); ri->flags &= ~SRI_DEMOTE; From 4028a777b67a77bf43da6a192784810ef5909be1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 30 Apr 2013 15:08:22 +0200 Subject: [PATCH 0590/1752] Sentinel: more sensible delay in master demote after tilt. --- src/sentinel.c | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 959f26e352e..2711ca304ed 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1481,18 +1481,20 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { if (ri->flags & SRI_DEMOTE) { /* If this sentinel was partitioned from the slave's master, * or tilted recently, wait some time before to act, - * so that DOWN and roles info will be refreshed. */ - if (!sentinelRedisInstanceNoDownFor(ri->master, - SENTINEL_INFO_PERIOD*2)) - return; - if (mstime()-sentinel.tilt_start_time < - SENTINEL_TILT_PERIOD+ri->master->down_after_period*2) + * so that DOWN and roles INFO will be refreshed. */ + mstime_t wait_time = SENTINEL_INFO_PERIOD*2 + + ri->master->down_after_period*2; + + if (!sentinelRedisInstanceNoDownFor(ri->master,wait_time) || + (mstime()-sentinel.tilt_start_time) < wait_time) return; - /* Old master returned back? Turn it into a slave ASAP if: + /* Old master returned back? Turn it into a slave ASAP if + * we can reach what we believe is the new master now, and + * have a recent role information for it. * - * We'll clear this flag only when we have the acknowledge - * that it's a slave again. */ + * Note: we'll clear the DEMOTE flag only when we have the + * acknowledge that it's a slave again. */ if (ri->master->flags & SRI_MASTER && (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) From e7bcec829ce7f78aa74863f756ea6ff8fcd5396e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 30 Apr 2013 12:24:46 +0200 Subject: [PATCH 0591/1752] Sentinel: changes to tilt mode. Tilt mode was too aggressive (not processing INFO output), this resulted in a few problems: 1) Redirections were not followed when in tilt mode. This opened a window to misinform clients about the current master when a Sentinel was in tilt mode and a fail over happened during the time it was not able to update the state. 2) It was possible for a Sentinel exiting tilt mode to detect a false fail over start, if a slave rebooted with a wrong configuration about at the same time. This used to happen since in tilt mode we lose the information that the runid changed (reboot). Now instead the Sentinel in tilt mode will still remove the instance from the list of slaves if it changes state AND runid at the same time. Both are edge conditions but the changes should overall improve the reliability of Sentinel. --- src/sentinel.c | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 2711ca304ed..a4db9408eca 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1461,10 +1461,13 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->info_refresh = mstime(); sdsfreesplitres(lines,numlines); - /* ---------------------------- Acting half ----------------------------- */ - if (sentinel.tilt) return; + /* ---------------------------- Acting half ----------------------------- + * Some things will not happen if sentinel.tilt is true, but some will + * still be processed. */ - /* Act if a master turned into a slave. */ + /* When what we believe is our master, turned into a slave, the wiser + * thing we can do is to follow the events and redirect to the new + * master, always. */ if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE && ri->slave_master_host) { sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, @@ -1473,12 +1476,12 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->slave_master_host, ri->slave_master_port); sentinelResetMasterAndChangeAddress(ri,ri->slave_master_host, ri->slave_master_port); - return; + return; /* Don't process anything after this event. */ } - /* Act if a slave turned into a master. */ + /* Handle slave -> master role switch. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { - if (ri->flags & SRI_DEMOTE) { + if (!sentinel.tilt && ri->flags & SRI_DEMOTE) { /* If this sentinel was partitioned from the slave's master, * or tilted recently, wait some time before to act, * so that DOWN and roles INFO will be refreshed. */ @@ -1513,22 +1516,25 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { sentinelEvent(REDIS_NOTICE,"-demote-flag-cleared",ri,"%@"); } } else if (!(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && - (runid_changed || first_runid)) + (runid_changed || first_runid)) { /* If a slave turned into master but: * * 1) Failover not in progress. - * 2) RunID hs changed, or its the first time we see an INFO output. + * 2) RunID has changed or its the first time we see an INFO output. * * We assume this is a reboot with a wrong configuration. - * Log the event and remove the slave. */ + * Log the event and remove the slave. Note that this is processed + * in tilt mode as well, otherwise we lose the information that the + * runid changed (reboot?) and when the tilt mode ends a fake + * failover will be detected. */ int retval; sentinelEvent(REDIS_WARNING,"-slave-restart-as-master",ri,"%@ #removing it from the attached slaves"); retval = dictDelete(ri->master->slaves,ri->name); redisAssert(retval == REDIS_OK); return; - } else if (ri->flags & SRI_PROMOTED) { + } else if (!sentinel.tilt && ri->flags & SRI_PROMOTED) { /* If this is a promoted slave we can change state to the * failover state machine. */ if ((ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && @@ -1544,11 +1550,12 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { sentinelCallClientReconfScript(ri->master,SENTINEL_LEADER, "start",ri->master->addr,ri->addr); } - } else if (!(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) || - ((ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && - (ri->master->flags & SRI_I_AM_THE_LEADER) && - ri->master->failover_state == - SENTINEL_FAILOVER_STATE_WAIT_START)) + } else if (!sentinel.tilt && ( + !(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) || + ((ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && + (ri->master->flags & SRI_I_AM_THE_LEADER) && + ri->master->failover_state == + SENTINEL_FAILOVER_STATE_WAIT_START))) { /* No failover in progress? Then it is the start of a failover * and we are an observer. @@ -1580,6 +1587,10 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { } } + /* None of the following conditions are processed when in tilt mode, so + * return asap. */ + if (sentinel.tilt) return; + /* Detect if the slave that is in the process of being reconfigured * changed state. */ if ((ri->flags & SRI_SLAVE) && role == SRI_SLAVE && From a94db7908a86b1520fd2003ba1ff9b7f64758d20 Mon Sep 17 00:00:00 2001 From: charsyam Date: Tue, 30 Apr 2013 12:07:58 +0900 Subject: [PATCH 0592/1752] Fix AOF bug: expire could be removed from key on AOF rewrite. There was a race condition in the AOF rewrite code that, with bad enough timing, could cause a volatile key just about to expire to be turned into a non-volatile key. The bug was never reported to cause actualy issues, but was found analytically by an user in the Redis mailing list: https://groups.google.com/forum/?fromgroups=#!topic/redis-db/Kvh2FAGK4Uk This commit fixes issue #1079. --- src/aof.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/aof.c b/src/aof.c index 56a985e727e..9e602ce0178 100644 --- a/src/aof.c +++ b/src/aof.c @@ -884,6 +884,9 @@ int rewriteAppendOnlyFile(char *filename) { expiretime = getExpire(db,&key); + /* If this key is already expired skip it */ + if (expiretime != -1 && expiretime < now) continue; + /* Save the key and associated value */ if (o->type == REDIS_STRING) { /* Emit a SET command */ @@ -906,8 +909,6 @@ int rewriteAppendOnlyFile(char *filename) { /* Save the expire time */ if (expiretime != -1) { char cmd[]="*3\r\n$9\r\nPEXPIREAT\r\n"; - /* If this key is already expired skip it */ - if (expiretime < now) continue; if (rioWrite(&aof,cmd,sizeof(cmd)-1) == 0) goto werr; if (rioWriteBulkObject(&aof,&key) == 0) goto werr; if (rioWriteBulkLongLong(&aof,expiretime) == 0) goto werr; From b60465eb788ca0ac2bdaf5ae6df6f2401348f805 Mon Sep 17 00:00:00 2001 From: Michel Martens Date: Thu, 25 Apr 2013 11:06:10 -0300 Subject: [PATCH 0593/1752] Reset masterauth if an empty string is configured. --- src/config.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 227d30013c9..2dcfa7cd7c6 100644 --- a/src/config.c +++ b/src/config.c @@ -514,7 +514,11 @@ void configSetCommand(redisClient *c) { server.requirepass = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } else if (!strcasecmp(c->argv[2]->ptr,"masterauth")) { zfree(server.masterauth); - server.masterauth = zstrdup(o->ptr); + if (sdslen(o->ptr)) { + server.masterauth = zstrdup(o->ptr); + } else { + server.masterauth = NULL; + } } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; From b05d2516fef38e01bac297d42e446cc36fa96dd4 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 2 May 2013 17:20:49 +0200 Subject: [PATCH 0594/1752] CONFIG SET server.masterauth aesthetic change. This is just to make the code exactly like the above instance used for requirepass. No actual change nor the original code violated the Redis coding style. --- src/config.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/config.c b/src/config.c index 2dcfa7cd7c6..06441210fa3 100644 --- a/src/config.c +++ b/src/config.c @@ -514,11 +514,7 @@ void configSetCommand(redisClient *c) { server.requirepass = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } else if (!strcasecmp(c->argv[2]->ptr,"masterauth")) { zfree(server.masterauth); - if (sdslen(o->ptr)) { - server.masterauth = zstrdup(o->ptr); - } else { - server.masterauth = NULL; - } + server.masterauth = ((char*)o->ptr)[0] ? zstrdup(o->ptr) : NULL; } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; From 85671d0e3b96943717c130f5589202821b2cf5e6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 May 2013 09:48:38 +0200 Subject: [PATCH 0595/1752] Added more info about 32 bit builds into README. --- README | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README b/README index 51152342a91..329eb1cb3fa 100644 --- a/README +++ b/README @@ -26,9 +26,20 @@ After building Redis is a good idea to test it, using: % make test -NOTE: if after building Redis with a 32 bit target you need to rebuild it - with a 64 bit target you need to perform a "make clean" in the root - directory of the Redis distribution. +Fixing problems building 32 bit binaries +--------- + +If after building Redis with a 32 bit target you need to rebuild it +with a 64 bit target, or the other way around, you need to perform a +"make distclean" in the root directory of the Redis distribution. + +In case of build errors when trying to build a 32 bit binary of Redis, try +the following steps: + +* Install the packages libc6-dev-i386 (also try g++-multilib). +* Try using the following command line instead of "make 32bit": + + make CFLAGS="-m32 -march=native" LDFLAGS="-m32" Allocator --------- From 3d93dffd5d7faa13667fd5fcf9208939315005a6 Mon Sep 17 00:00:00 2001 From: Jiahao Huang Date: Tue, 7 May 2013 19:55:57 +0800 Subject: [PATCH 0596/1752] in 32bit machine, popcount don't work with a input string length up to 512 MB, bitcount commant may return negtive integer with string length more than 256 MB --- src/bitops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index f2d03f9b0f1..47f768c31d5 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -58,8 +58,8 @@ static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to * work with a input string length up to 512 MB. */ -long popcount(void *s, long count) { - long bits = 0; +size_t popcount(void *s, long count) { + size_t bits = 0; unsigned char *p; uint32_t *p4 = s; static const unsigned char bitsinbyte[256] = {0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,1,2,2,3,2,3,3,4,2,3,3,4,3,4,4,5,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,2,3,3,4,3,4,4,5,3,4,4,5,4,5,5,6,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,3,4,4,5,4,5,5,6,4,5,5,6,5,6,6,7,4,5,5,6,5,6,6,7,5,6,6,7,6,7,7,8}; From 2865bb7f1f719b3e23da9a637e74ce66d7d29bb3 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 May 2013 11:58:26 +0200 Subject: [PATCH 0597/1752] Test: various issues with the replication-4.tcl test fixed. The test actually worked, but vars for master and slave were inverted and sometimes used incorrectly. --- tests/integration/replication-4.tcl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 0a0dcdc2bdd..29ab48fc269 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -10,21 +10,22 @@ proc stop_bg_complex_data {handle} { start_server {tags {"repl"}} { start_server {} { - set master [srv 0 client] - set master_host [srv 0 host] - set master_port [srv 0 port] + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] test {First server should have role slave after SLAVEOF} { - r -1 slaveof [srv 0 host] [srv 0 port] + $slave slaveof $master_host $master_port after 1000 - s -1 role + s 0 role } {slave} test {Test replication with parallel clients writing in differnet DBs} { - lappend slave [srv 0 client] after 5000 stop_bg_complex_data $load_handle0 stop_bg_complex_data $load_handle1 @@ -37,7 +38,7 @@ start_server {tags {"repl"}} { } assert {[$master dbsize] > 0} - if {[r debug digest] ne [r -1 debug digest]} { + if {[$master debug digest] ne [$slave debug digest]} { set csv1 [csvdump r] set csv2 [csvdump {r -1}] set fd [open /tmp/repldump1.txt w] From c847c73e78443177ff4c520123205d76090bd552 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 May 2013 13:01:42 +0200 Subject: [PATCH 0598/1752] Test: check that replication partial sync works if we break the link. The test checks both successful syncs and unsuccessful ones by changing the backlog size. --- tests/integration/replication-psync.tcl | 79 +++++++++++++++++++++++++ tests/test_helper.tcl | 1 + 2 files changed, 80 insertions(+) create mode 100644 tests/integration/replication-psync.tcl diff --git a/tests/integration/replication-psync.tcl b/tests/integration/replication-psync.tcl new file mode 100644 index 00000000000..b49ab2519f9 --- /dev/null +++ b/tests/integration/replication-psync.tcl @@ -0,0 +1,79 @@ +proc start_bg_complex_data {host port db ops} { + set tclsh [info nameofexecutable] + exec $tclsh tests/helpers/bg_complex_data.tcl $host $port $db $ops & +} + +proc stop_bg_complex_data {handle} { + catch {exec /bin/kill -9 $handle} +} + +proc test_psync {backlog cond} { + start_server {tags {"repl"}} { + start_server {} { + + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + $master config set repl-backlog-size $backlog + + set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] + set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] + set load_handle2 [start_bg_complex_data $master_host $master_port 12 100000] + + test {First server should have role slave after SLAVEOF} { + $slave slaveof $master_host $master_port + after 1000 + s 0 role + } {slave} + + test "Test replication partial resync with backlog $backlog" { + # Now while the clients are writing data, break the maste-slave + # link multiple times. + for {set j 0} {$j < 100} {incr j} { + after 100 ;# 100 times 100 milliseconds = 10 seconds total test + # catch {puts "MASTER [$master dbsize] keys, SLAVE [$slave dbsize] keys"} + + if {($j % 20) == 0} { + catch { + $slave client kill $master_host:$master_port + } + } + } + stop_bg_complex_data $load_handle0 + stop_bg_complex_data $load_handle1 + stop_bg_complex_data $load_handle2 + set retry 10 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} + + if {[$master debug digest] ne [$slave debug digest]} { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + } + assert_equal [r debug digest] [r -1 debug digest] + eval $cond + } + } + } +} + +test_psync 1000000 { + assert {[s -1 sync_partial_ok] > 0} +} +test_psync 100 { + assert {[s -1 sync_partial_err] > 0} +} diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 6aeff6ac26f..8d34a078307 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -32,6 +32,7 @@ set ::all_tests { integration/replication-2 integration/replication-3 integration/replication-4 + integration/replication-psync integration/aof integration/rdb integration/convert-zipmap-hash-on-load From cb55ad991b54031fbfeb627b38d5b4d726895a0a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 May 2013 15:24:54 +0200 Subject: [PATCH 0599/1752] Version bumped to 2.7.1 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 3d211db5a16..4844e7017fa 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.0" +#define REDIS_VERSION "2.7.1" From 41babac68d548583e71fa3001dadc868e367132a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 May 2013 12:52:04 +0200 Subject: [PATCH 0600/1752] Test: more PSYNC tests (backlog TTL). --- tests/integration/replication-psync.tcl | 48 +++++++++++++++++++------ 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/tests/integration/replication-psync.tcl b/tests/integration/replication-psync.tcl index b49ab2519f9..97d74db3755 100644 --- a/tests/integration/replication-psync.tcl +++ b/tests/integration/replication-psync.tcl @@ -7,7 +7,13 @@ proc stop_bg_complex_data {handle} { catch {exec /bin/kill -9 $handle} } -proc test_psync {backlog cond} { +# Creates a master-slave pair and breaks the link continuously to force +# partial resyncs attempts, all this while flooding the master with +# write queries. +# +# You can specifiy backlog size, ttl, delay before reconnection, test duration +# in seconds, and an additional condition to verify at the end. +proc test_psync {descr duration backlog_size backlog_ttl delay cond} { start_server {tags {"repl"}} { start_server {} { @@ -16,7 +22,8 @@ proc test_psync {backlog cond} { set master_port [srv -1 port] set slave [srv 0 client] - $master config set repl-backlog-size $backlog + $master config set repl-backlog-size $backlog_size + $master config set repl-backlog-ttl $backlog_ttl set load_handle0 [start_bg_complex_data $master_host $master_port 9 100000] set load_handle1 [start_bg_complex_data $master_host $master_port 11 100000] @@ -24,20 +31,30 @@ proc test_psync {backlog cond} { test {First server should have role slave after SLAVEOF} { $slave slaveof $master_host $master_port - after 1000 - s 0 role - } {slave} + wait_for_condition 50 100 { + [s 0 role] eq {slave} + } else { + fail "Replication not started." + } + } - test "Test replication partial resync with backlog $backlog" { + test "Test replication partial resync: $descr" { # Now while the clients are writing data, break the maste-slave # link multiple times. - for {set j 0} {$j < 100} {incr j} { - after 100 ;# 100 times 100 milliseconds = 10 seconds total test + for {set j 0} {$j < $duration*10} {incr j} { + after 100 # catch {puts "MASTER [$master dbsize] keys, SLAVE [$slave dbsize] keys"} if {($j % 20) == 0} { catch { - $slave client kill $master_host:$master_port + if {$delay} { + $slave multi + $slave client kill $master_host:$master_port + $slave debug sleep $delay + $slave exec + } else { + $slave client kill $master_host:$master_port + } } } } @@ -71,9 +88,18 @@ proc test_psync {backlog cond} { } } -test_psync 1000000 { +test_psync {ok psync} 6 1000000 3600 0 { assert {[s -1 sync_partial_ok] > 0} } -test_psync 100 { + +test_psync {no backlog} 6 100 3600 0 { + assert {[s -1 sync_partial_err] > 0} +} + +test_psync {ok after delay} 3 100000000 3600 3 { + assert {[s -1 sync_partial_ok] > 0} +} + +test_psync {backlog expired} 3 100000000 1 3 { assert {[s -1 sync_partial_err] > 0} } From d8dd70c2dd9775f96048e11e8251151b33878f67 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 May 2013 11:22:50 +0200 Subject: [PATCH 0601/1752] redis-cli: help.h updated. --- src/help.h | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/help.h b/src/help.h index 1378b8b85d2..60a5726bfbe 100644 --- a/src/help.h +++ b/src/help.h @@ -69,6 +69,11 @@ struct commandHelp { "Pop a value from a list, push it to another list and return it; or block until one is available", 2, "2.2.0" }, + { "CLIENT GETNAME", + "-", + "Get the current connection name", + 9, + "2.6.9" }, { "CLIENT KILL", "ip:port", "Kill the connection of a client", @@ -79,6 +84,11 @@ struct commandHelp { "Get the list of client connections", 9, "2.4.0" }, + { "CLIENT SETNAME", + "connection-name", + "Set the current connection name", + 9, + "2.6.9" }, { "CONFIG GET", "parameter", "Get the value of a configuration parameter", @@ -280,7 +290,7 @@ struct commandHelp { 1, "2.6.0" }, { "INFO", - "-", + "[section]", "Get information and statistics about the server", 9, "1.0.0" }, @@ -525,7 +535,7 @@ struct commandHelp { 8, "1.0.0" }, { "SET", - "key value", + "key value [EX seconds] [PX milliseconds] [NX|XX]", "Set the string value of a key", 1, "1.0.0" }, @@ -665,7 +675,7 @@ struct commandHelp { 7, "2.2.0" }, { "ZADD", - "key score member [score] [member]", + "key score member [score member ...]", "Add one or more members to a sorted set, or update its score if it already exists", 4, "1.2.0" }, From 7a5d3d91a10aa4639d2b4e4f0fb3d5aced801f59 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 May 2013 16:57:59 +0200 Subject: [PATCH 0602/1752] Obtain absoute path of configuration file, expose it in INFO. --- src/redis.c | 60 +++++++++++++++++++++++++++++++++++++++++++++++++++-- src/redis.h | 1 + 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index f7b04f08f05..75617d6ae1c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1163,6 +1163,7 @@ void createSharedObjects(void) { void initServerConfig() { getRandomHexChars(server.runid,REDIS_RUN_ID_SIZE); + server.configfile = NULL; server.hz = REDIS_DEFAULT_HZ; server.runid[REDIS_RUN_ID_SIZE] = '\0'; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; @@ -2006,7 +2007,8 @@ sds genRedisInfoString(char *section) { "uptime_in_seconds:%ld\r\n" "uptime_in_days:%ld\r\n" "hz:%d\r\n" - "lru_clock:%ld\r\n", + "lru_clock:%ld\r\n" + "config_file:%s\r\n", REDIS_VERSION, redisGitSHA1(), strtol(redisGitDirty(),NULL,10) > 0, @@ -2026,7 +2028,8 @@ sds genRedisInfoString(char *section) { uptime, uptime/(3600*24), server.hz, - (unsigned long) server.lruclock); + (unsigned long) server.lruclock, + server.configfile ? server.configfile : ""); } /* Clients */ @@ -2689,6 +2692,58 @@ void redisSetProcTitle(char *title) { #endif } +/* Given the filename, return the absolute path as an SDS string, or NULL + * if it fails for some reason. Note that "filename" may be an absolute path + * already, this will be detected and handled correctly. + * + * The function does not try to normalize everything, but only the obvious + * case of one or more "../" appearning at the start of "filename" + * relative path. */ +sds getAbsolutePath(char *filename) { + char cwd[1024]; + sds abspath; + sds relpath = sdsnew(filename); + + relpath = sdstrim(relpath," \r\n\t"); + if (relpath[0] == '/') return relpath; /* Path is already absolute. */ + + /* If path is relative, join cwd and relative path. */ + if (getcwd(cwd,sizeof(cwd)) == NULL) { + sdsfree(relpath); + return NULL; + } + abspath = sdsnew(cwd); + if (sdslen(abspath) && abspath[sdslen(abspath)-1] != '/') + abspath = sdscat(abspath,"/"); + + /* At this point we have the current path always ending with "/", and + * the trimmed relative path. Try to normalize the obvious case of + * trailing ../ elements at the start of the path. + * + * For every "../" we find in the filename, we remove it and also remove + * the last element of the cwd, unless the current cwd is "/". */ + while (sdslen(relpath) >= 3 && + relpath[0] == '.' && relpath[1] == '.' && relpath[2] == '/') + { + relpath = sdsrange(relpath,3,-1); + if (sdslen(abspath) > 1) { + char *p = abspath + sdslen(abspath)-2; + int trimlen = 1; + + while(*p != '/') { + p--; + trimlen++; + } + abspath = sdsrange(abspath,0,-(trimlen+1)); + } + } + + /* Finally glue the two parts together. */ + abspath = sdscatsds(abspath,relpath); + sdsfree(relpath); + return abspath; +} + int main(int argc, char **argv) { struct timeval tv; @@ -2756,6 +2811,7 @@ int main(int argc, char **argv) { resetServerSaveParams(); loadServerConfig(configfile,options); sdsfree(options); + if (configfile) server.configfile = getAbsolutePath(configfile); } else { redisLog(REDIS_WARNING, "Warning: no config file specified, using the default config. In order to specify a config file use %s /path/to/%s.conf", argv[0], server.sentinel_mode ? "sentinel" : "redis"); } diff --git a/src/redis.h b/src/redis.h index a05a3ad3647..23773ca493b 100644 --- a/src/redis.h +++ b/src/redis.h @@ -519,6 +519,7 @@ typedef struct redisOpArray { struct redisServer { /* General */ + char *configfile; /* Absolute config file path, or NULL */ int hz; /* serverCron() calls frequency in hertz */ redisDb *db; dict *commands; /* Command table */ From 973f793b04fc92ceb939adcda509c7e4665036c8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 May 2013 00:15:18 +0200 Subject: [PATCH 0603/1752] CONFIG REWRITE: Initial support code and design. --- src/config.c | 423 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/redis.c | 4 +- src/redis.h | 3 + 3 files changed, 427 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index 06441210fa3..582479b6f41 100644 --- a/src/config.c +++ b/src/config.c @@ -495,7 +495,7 @@ void loadServerConfig(char *filename, char *options) { } /*----------------------------------------------------------------------------- - * CONFIG command for remote configuration + * CONFIG SET implementation *----------------------------------------------------------------------------*/ void configSetCommand(redisClient *c) { @@ -791,6 +791,10 @@ void configSetCommand(redisClient *c) { (char*)c->argv[2]->ptr); } +/*----------------------------------------------------------------------------- + * CONFIG GET implementation + *----------------------------------------------------------------------------*/ + #define config_get_string_field(_name,_var) do { \ if (stringmatch(pattern,_name,0)) { \ addReplyBulkCString(c,_name); \ @@ -1015,6 +1019,412 @@ void configGetCommand(redisClient *c) { setDeferredMultiBulkLength(c,replylen,matches*2); } +/*----------------------------------------------------------------------------- + * CONFIG REWRITE implementation + *----------------------------------------------------------------------------*/ + +/* IGNORE: + * + * rename-command + * include + * + * Special handling: + * + * notify-keyspace-events + * client-output-buffer-limit + * save + * appendonly + * appendfsync + * dir + * maxmemory-policy + * loglevel + * unixsocketperm + * slaveof + * + * Type of config directives: + * + * CUSTOM + * VERBATIM + * YESNO + * L + * LL + * + */ + +/* We use the following dictionary type to store where a configuration + * option is mentioned in the old configuration file, so it's + * like "maxmemory" -> list of line numbers (first line is zero). */ +unsigned int dictSdsHash(const void *key); +int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2); +void dictSdsDestructor(void *privdata, void *val); +void dictListDestructor(void *privdata, void *val); + +dictType optionToLineDictType = { + dictSdsHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + dictListDestructor /* val destructor */ +}; + +/* The config rewrite state. */ +struct rewriteConfigState { + dict *option_to_line; /* Option -> list of config file lines map */ + int numlines; /* Number of lines in current config */ + sds *lines; /* Current lines as an array of sds strings */ + int has_tail; /* True if we already added directives that were + not present in the original config file. */ +}; + +/* Append the new line to the current configuration state. */ +void rewriteConfigAppendLine(struct rewriteConfigState *state, sds line) { + state->lines = zrealloc(state->lines, sizeof(char*) * (state->numlines+1)); + state->lines[state->numlines++] = line; +} + +/* Populate the option -> list of line numbers map. */ +void rewriteConfigAddLineNumberToOption(struct rewriteConfigState *state, sds option, int linenum) { + list *l = dictFetchValue(state->option_to_line,option); + + if (l == NULL) { + l = listCreate(); + dictAdd(state->option_to_line,sdsdup(option),l); + } + listAddNodeTail(l,(void*)(long)linenum); +} + +/* Read the old file, split it into lines to populate a newly created + * config rewrite state, and return it to the caller. + * + * If it is impossible to read the old file, NULL is returned. + * If the old file does not exist at all, an empty state is returned. */ +struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { + FILE *fp = fopen(path,"r"); + struct rewriteConfigState *state = zmalloc(sizeof(*state)); + char buf[REDIS_CONFIGLINE_MAX+1]; + int linenum = -1; + + if (fp == NULL && errno != ENOENT) return NULL; + + state->option_to_line = dictCreate(&optionToLineDictType,NULL); + state->numlines = 0; + state->lines = NULL; + state->has_tail = 0; + if (fp == NULL) return state; + + /* Read the old file line by line, populate the state. */ + while(fgets(buf,REDIS_CONFIGLINE_MAX+1,fp) != NULL) { + int argc; + sds *argv; + sds line = sdstrim(sdsnew(buf),"\r\n\t "); + + linenum++; /* Zero based, so we init at -1 */ + + /* Handle comments and empty lines. */ + if (line[0] == '#' || line[0] == '\0') { + rewriteConfigAppendLine(state,line); + continue; + } + + /* Not a comment, split into arguments. */ + argv = sdssplitargs(line,&argc); + if (argv == NULL) { + /* Apparently the line is unparsable for some reason, for + * instance it may have unbalanced quotes. Load it as a + * comment. */ + sds aux = sdsnew("# ??? "); + aux = sdscatsds(aux,line); + sdsfree(line); + rewriteConfigAppendLine(state,aux); + continue; + } + + sdstolower(argv[0]); /* We only want lowercase config directives. */ + + /* Now we populate the state according to the content of this line. + * Append the line and populate the option -> line numbers map. */ + rewriteConfigAppendLine(state,line); + rewriteConfigAddLineNumberToOption(state,argv[0],linenum); + + sdsfreesplitres(argv,argc); + } + fclose(fp); + return state; +} + +/* Rewrite the specified configuration option with the new "line". + * It progressively uses lines of the file that were already used for the same + * configuraiton option in the old version of the file, removing that line from + * the map of options -> line numbers. + * + * If there are lines associated with a given configuration option and + * "force" is non-zero, the line is appended to the configuration file. + * Usually "force" is true when an option has not its default value, so it + * must be rewritten even if not present previously. + * + * The first time a line is appended into a configuration file, a comment + * is added to show that starting from that point the config file was generated + * by CONFIG REWRITE. + * + * "line" is either used, or freed, so the caller does not need to free it + * in any way. */ +void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sds line, int force) { + sds o = sdsnew(option); + list *l = dictFetchValue(state->option_to_line,o); + + if (!l && !force) { + /* Option not used previously, and we are not forced to use it. */ + sdsfree(line); + sdsfree(o); + return; + } + + if (l) { + listNode *ln = listFirst(l); + int linenum = (long) ln->value; + + /* There are still lines in the old configuration file we can reuse + * for this option. Replace the line with the new one. */ + listDelNode(l,ln); + if (listLength(l) == 0) dictDelete(state->option_to_line,o); + sdsfree(state->lines[linenum]); + state->lines[linenum] = line; + } else { + /* Append a new line. */ + if (!state->has_tail) { + rewriteConfigAppendLine(state, + sdsnew("# Generated by CONFIG REWRITE")); + state->has_tail = 1; + } + rewriteConfigAppendLine(state,line); + } + sdsfree(o); +} + +/* Rewrite a simple "option-name " configuration option. */ +void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { + int force = value != defvalue; + /* TODO: check if we can write it using MB, GB, or other suffixes. */ + sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); + + rewriteConfigRewriteLine(state,option,line,force); +} + +void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { + int force = value != defvalue; + sds line = sdscatprintf(sdsempty(),"%s %s",option, + value ? "yes" : "no"); + + rewriteConfigRewriteLine(state,option,line,force); +} + +void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, char *value, char *defvalue) { + int force = 1; + sds line; + + /* String options set to NULL need to be not present at all in the + * configuration file to be set to NULL again at the next reboot. */ + if (value == NULL) return; + + /* Compare the strings as sds strings to have a binary safe comparison. */ + if (defvalue && strcmp(value,defvalue) == 0) force = 0; + + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, value, strlen(value)); + + rewriteConfigRewriteLine(state,option,line,force); +} + +void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { + int force = value != defvalue; + sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); + + rewriteConfigRewriteLine(state,option,line,force); +} + +void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { + int force = value != defvalue; + sds line = sdscatprintf(sdsempty(),"%s %o",option,value); + + rewriteConfigRewriteLine(state,option,line,force); +} + +void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, ...) { + va_list ap; + char *enum_name, *matching_name; + int enum_val, def_val, force; + sds line; + + va_start(ap, value); + while(1) { + enum_name = va_arg(ap,char*); + enum_val = va_arg(ap,int); + if (enum_name == NULL) { + def_val = enum_val; + break; + } + if (value == enum_val) matching_name = enum_name; + } + va_end(ap); + + force = value != def_val; + line = sdscatprintf(sdsempty(),"%s %s",option,matching_name); + rewriteConfigRewriteLine(state,option,line,force); +} + +void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { +} + +void rewriteConfigSaveOption(struct rewriteConfigState *state) { +} + +void rewriteConfigDirOption(struct rewriteConfigState *state) { +} + +void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { +} + +void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { +} + +void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { +} + +void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state) { +} + +sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { + sds content = sdsempty(); + int j; + + for (j = 0; j < state->numlines; j++) { + content = sdscatsds(content,state->lines[j]); + content = sdscatlen(content,"\n",1); + } + return content; +} + +void rewriteConfigReleaseState(struct rewriteConfigState *state) { + sdsfreesplitres(state->lines,state->numlines); + dictRelease(state->option_to_line); + zfree(state); +} + +/* Rewrite the configuration file at "path". + * If the configuration file already exists, we try at best to retain comments + * and overall structure. + * + * Configuration parameters that are at their default value, unless already + * explicitly included in the old configuration file, are not rewritten. + * + * On error -1 is returned and errno is set accordingly, otherwise 0. */ +int rewriteConfig(char *path) { + struct rewriteConfigState *state; + sds newcontent; + + /* Step 1: read the old config into our rewrite state. */ + if ((state = rewriteConfigReadOldFile(path)) == NULL) return -1; + + /* Step 2: rewrite every single option, replacing or appending it inside + * the rewrite state. */ + + /* TODO: Turn every default into a define, use it also in + * initServerConfig(). */ + rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); + rewriteConfigStringOption(state,"pidfile",server.pidfile,REDIS_DEFAULT_PID_FILE); + rewriteConfigNumericalOption(state,"port",server.port,REDIS_SERVERPORT); + rewriteConfigStringOption(state,"bindaddr",server.bindaddr,NULL); + rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); + rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,0); + rewriteConfigNumericalOption(state,"timeout",server.maxidletime,0); + rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,0); + rewriteConfigEnumOption(state,"loglevel",server.verbosity, + "debug", REDIS_DEBUG, + "verbose", REDIS_VERBOSE, + "notice", REDIS_NOTICE, + "warning", REDIS_WARNING, + NULL, REDIS_NOTICE); + rewriteConfigStringOption(state,"logfile",server.logfile,"stdout"); + rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,0); + rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,REDIS_DEFAULT_SYSLOG_IDENT); + rewriteConfigSyslogfacilityOption(state); + rewriteConfigSaveOption(state); + rewriteConfigNumericalOption(state,"databases",server.dbnum,REDIS_DEFAULT_DBNUM); + rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,1); + rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,1); + rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,1); + rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,"dump.rdb"); + rewriteConfigDirOption(state); + rewriteConfigSlaveofOption(state); + rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); + rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,1); + rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,1); + rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,REDIS_REPL_PING_SLAVE_PERIOD); + rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,REDIS_REPL_TIMEOUT); + rewriteConfigNumericalOption(state,"repl-backlog-size",server.repl_backlog_size,REDIS_DEFAULT_REPL_BACKLOG_SIZE); + rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT); + rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,REDIS_DEFAULT_SLAVE_PRIORITY); + rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); + rewriteConfigNumericalOption(state,"maxclients",server.maxclients,REDIS_MAX_CLIENTS); + rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,0); + rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy, + "volatile-lru", REDIS_MAXMEMORY_VOLATILE_LRU, + "allkeys-lru", REDIS_MAXMEMORY_ALLKEYS_LRU, + "volatile-random", REDIS_MAXMEMORY_VOLATILE_RANDOM, + "allkeys-random", REDIS_MAXMEMORY_ALLKEYS_RANDOM, + "volatile-ttl", REDIS_MAXMEMORY_VOLATILE_TTL, + "noeviction", REDIS_MAXMEMORY_NO_EVICTION, + NULL, REDIS_MAXMEMORY_VOLATILE_LRU); + rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,3); + rewriteConfigAppendonlyOption(state); + rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync, + "eveysec", AOF_FSYNC_EVERYSEC, + "always", AOF_FSYNC_ALWAYS, + "no", AOF_FSYNC_NO, + NULL, AOF_FSYNC_EVERYSEC); + rewriteConfigYesNoOption(state,"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite,0); + rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,REDIS_AOF_REWRITE_PERC); + rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,REDIS_AOF_REWRITE_MIN_SIZE); + rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,REDIS_LUA_TIME_LIMIT); + rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); + rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,REDIS_DEFAULT_CLUSTER_CONFIG_FILE); + rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,REDIS_CLUSTER_DEFAULT_NODE_TIMEOUT); + rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,REDIS_SLOWLOG_LOG_SLOWER_THAN); + rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,REDIS_SLOWLOG_MAX_LEN); + rewriteConfigNotifykeyspaceeventsOption(state); + rewriteConfigNumericalOption(state,"hash-max-ziplist-entries",server.hash_max_ziplist_entries,REDIS_HASH_MAX_ZIPLIST_ENTRIES); + rewriteConfigNumericalOption(state,"hash-max-ziplist-value",server.hash_max_ziplist_value,REDIS_HASH_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"list-max-ziplist-entries",server.list_max_ziplist_entries,REDIS_LIST_MAX_ZIPLIST_ENTRIES); + rewriteConfigNumericalOption(state,"list-max-ziplist-value",server.list_max_ziplist_value,REDIS_LIST_MAX_ZIPLIST_VALUE); + rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES); + rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES); + rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE); + rewriteConfigYesNoOption(state,"active-rehashing",server.activerehashing,1); + rewriteConfigClientoutputbufferlimitOption(state); + rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ); + rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,1); + + /* Step 3: remove all the orphaned lines in the old file, that is, lines + * that were used by a config option and are no longer used, like in case + * of multiple "save" options or duplicated options. */ +// rewriteConfigRemoveOrphaned(state); + + /* Step 4: generate a new configuration file from the modified state + * and write it into the original file. */ + newcontent = rewriteConfigGetContentFromState(state); + printf("%s\n", newcontent); + + sdsfree(newcontent); + rewriteConfigReleaseState(state); + return 0; +} + +/*----------------------------------------------------------------------------- + * CONFIG command entry point + *----------------------------------------------------------------------------*/ + void configCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"set")) { if (c->argc != 4) goto badarity; @@ -1034,6 +1444,17 @@ void configCommand(redisClient *c) { server.aof_delayed_fsync = 0; resetCommandTableStats(); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"rewrite")) { + if (c->argc != 2) goto badarity; + if (server.configfile == NULL) { + addReplyError(c,"The server is running without a config file"); + return; + } + if (rewriteConfig(server.configfile) == -1) { + addReplyErrorFormat(c,"Rewriting config file: %s", strerror(errno)); + } else { + addReply(c,shared.ok); + } } else { addReplyError(c, "CONFIG subcommand must be one of GET, SET, RESETSTAT"); diff --git a/src/redis.c b/src/redis.c index 75617d6ae1c..9802c4efb31 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1183,7 +1183,7 @@ void initServerConfig() { server.loading = 0; server.logfile = NULL; /* NULL = log on standard output */ server.syslog_enabled = 0; - server.syslog_ident = zstrdup("redis"); + server.syslog_ident = zstrdup(REDIS_DEFAULT_SYSLOG_IDENT); server.syslog_facility = LOG_LOCAL0; server.daemonize = 0; server.aof_state = REDIS_AOF_OFF; @@ -1202,7 +1202,7 @@ void initServerConfig() { server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; server.aof_rewrite_incremental_fsync = 1; - server.pidfile = zstrdup("/var/run/redis.pid"); + server.pidfile = zstrdup(REDIS_DEFAULT_PID_FILE); server.rdb_filename = zstrdup("dump.rdb"); server.aof_filename = zstrdup("appendonly.aof"); server.requirepass = NULL; diff --git a/src/redis.h b/src/redis.h index 23773ca493b..90a4c084a36 100644 --- a/src/redis.h +++ b/src/redis.h @@ -98,6 +98,9 @@ #define REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT (60*60) /* 1 hour */ #define REDIS_REPL_BACKLOG_MIN_SIZE (1024*16) /* 16k */ #define REDIS_BGSAVE_RETRY_DELAY 5 /* Wait a few secs before trying again. */ +#define REDIS_DEFAULT_PID_FILE "/var/run/redis.pid" +#define REDIS_DEFAULT_SYSLOG_IDENT "redis" +#define REDIS_DEFAULT_CLUSTER_CONFIG_FILE "nodes.conf" /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ From 91ef10ef45b49c478a41b1eb755272f2deaf5044 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 May 2013 11:11:35 +0200 Subject: [PATCH 0604/1752] CONFIG REWRITE: support for save and syslog-facility. --- src/config.c | 55 ++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 15 deletions(-) diff --git a/src/config.c b/src/config.c index 582479b6f41..4e8ac002173 100644 --- a/src/config.c +++ b/src/config.c @@ -31,6 +31,22 @@ #include "redis.h" +static struct { + const char *name; + const int value; +} validSyslogFacilities[] = { + {"user", LOG_USER}, + {"local0", LOG_LOCAL0}, + {"local1", LOG_LOCAL1}, + {"local2", LOG_LOCAL2}, + {"local3", LOG_LOCAL3}, + {"local4", LOG_LOCAL4}, + {"local5", LOG_LOCAL5}, + {"local6", LOG_LOCAL6}, + {"local7", LOG_LOCAL7}, + {NULL, 0} +}; + /*----------------------------------------------------------------------------- * Config file parsing *----------------------------------------------------------------------------*/ @@ -164,21 +180,6 @@ void loadServerConfigFromString(char *config) { if (server.syslog_ident) zfree(server.syslog_ident); server.syslog_ident = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"syslog-facility") && argc == 2) { - struct { - const char *name; - const int value; - } validSyslogFacilities[] = { - {"user", LOG_USER}, - {"local0", LOG_LOCAL0}, - {"local1", LOG_LOCAL1}, - {"local2", LOG_LOCAL2}, - {"local3", LOG_LOCAL3}, - {"local4", LOG_LOCAL4}, - {"local5", LOG_LOCAL5}, - {"local6", LOG_LOCAL6}, - {"local7", LOG_LOCAL7}, - {NULL, 0} - }; int i; for (i = 0; validSyslogFacilities[i].name; i++) { @@ -1275,9 +1276,33 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int } void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { + int value = server.syslog_facility, j; + int force = value != LOG_LOCAL0; + char *name, *option = "syslog-facility"; + sds line; + + for (j = 0; validSyslogFacilities[j].name; j++) { + if (validSyslogFacilities[j].value == value) { + name = (char*) validSyslogFacilities[j].name; + break; + } + } + line = sdscatprintf(sdsempty(),"%s %s",option,name); + rewriteConfigRewriteLine(state,option,line,force); } void rewriteConfigSaveOption(struct rewriteConfigState *state) { + int j; + sds line; + + /* Note that if there are no save parameters at all, all the current + * config line with "save" will be detected as orphaned and deleted, + * resulting into no RDB persistence as expected. */ + for (j = 0; j < server.saveparamslen; j++) { + line = sdscatprintf(sdsempty(),"save %ld %d", + server.saveparams[j].seconds, server.saveparams[j].changes); + rewriteConfigRewriteLine(state,"save",line,1); + } } void rewriteConfigDirOption(struct rewriteConfigState *state) { From f62851aea919ac644f714d984230614a94f12176 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 May 2013 11:26:43 +0200 Subject: [PATCH 0605/1752] CONFIG REWRITE: support for dir and slaveof. --- src/config.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/config.c b/src/config.c index 4e8ac002173..9ea61acdc67 100644 --- a/src/config.c +++ b/src/config.c @@ -1306,9 +1306,21 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { } void rewriteConfigDirOption(struct rewriteConfigState *state) { + char cwd[1024]; + + if (getcwd(cwd,sizeof(cwd)) == NULL) return; /* no rewrite on error. */ + rewriteConfigStringOption(state,"dir",cwd,NULL); } void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { + sds line; + + /* If this is a master, we want all the slaveof config options + * in the file to be removed. */ + if (server.masterhost == NULL) return; + line = sdscatprintf(sdsempty(),"slaveof %s %d", + server.masterhost, server.masterport); + rewriteConfigRewriteLine(state,"slaveof",line,1); } void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { From cb8433f3133ca0b95575382c0b46e9e7d804decf Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 May 2013 18:34:18 +0200 Subject: [PATCH 0606/1752] CONFIG REWRITE: support for client-output-buffer-limit. --- src/config.c | 46 ++++++++++++++++++++++++++++++++++++++++++++-- src/redis.c | 13 ++++--------- src/redis.h | 2 ++ 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/config.c b/src/config.c index 9ea61acdc67..ef3d7157420 100644 --- a/src/config.c +++ b/src/config.c @@ -47,6 +47,12 @@ static struct { {NULL, 0} }; +clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_LIMIT_NUM_CLASSES] = { + {0, 0, 0}, /* normal */ + {1024*1024*256, 1024*1024*64, 60}, /* slave */ + {1024*1024*32, 1024*1024*8, 60} /* pubsub */ +}; + /*----------------------------------------------------------------------------- * Config file parsing *----------------------------------------------------------------------------*/ @@ -1313,23 +1319,59 @@ void rewriteConfigDirOption(struct rewriteConfigState *state) { } void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { + char *option = "slaveof"; sds line; /* If this is a master, we want all the slaveof config options * in the file to be removed. */ if (server.masterhost == NULL) return; - line = sdscatprintf(sdsempty(),"slaveof %s %d", + line = sdscatprintf(sdsempty(),"%s %s %d", option, server.masterhost, server.masterport); - rewriteConfigRewriteLine(state,"slaveof",line,1); + rewriteConfigRewriteLine(state,option,line,1); } void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { + int force = server.aof_state != REDIS_AOF_OFF; + char *option = "appendonly"; + sds line; + + line = sdscatprintf(sdsempty(),"%s %s", option, + (server.aof_state == REDIS_AOF_OFF) ? "no" : "yes"); + rewriteConfigRewriteLine(state,option,line,force); } void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { + int force = server.notify_keyspace_events != 0; + char *option = "notify-keyspace-events"; + sds line, flags; + + flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); + line = sdscatprintf(sdsempty(),"%s %s", option, flags); + sdsfree(flags); + rewriteConfigRewriteLine(state,option,line,force); } void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state) { + int j; + char *option = "client-output-buffer-limit"; + + for (j = 0; j < REDIS_CLIENT_LIMIT_NUM_CLASSES; j++) { + int force = (server.client_obuf_limits[j].hard_limit_bytes != + clientBufferLimitsDefaults[j].hard_limit_bytes) || + (server.client_obuf_limits[j].soft_limit_bytes != + clientBufferLimitsDefaults[j].soft_limit_bytes) || + (server.client_obuf_limits[j].soft_limit_seconds != + clientBufferLimitsDefaults[j].soft_limit_seconds); + sds line; + + line = sdscatprintf(sdsempty(),"%s %s %llu %llu %ld", + option, + getClientLimitClassName(j), + server.client_obuf_limits[j].hard_limit_bytes, + server.client_obuf_limits[j].soft_limit_bytes, + (long) server.client_obuf_limits[j].soft_limit_seconds); + rewriteConfigRewriteLine(state,option,line,force); + } } sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { diff --git a/src/redis.c b/src/redis.c index 9802c4efb31..a9565e17650 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1162,6 +1162,8 @@ void createSharedObjects(void) { } void initServerConfig() { + int j; + getRandomHexChars(server.runid,REDIS_RUN_ID_SIZE); server.configfile = NULL; server.hz = REDIS_DEFAULT_HZ; @@ -1263,15 +1265,8 @@ void initServerConfig() { server.repl_no_slaves_since = time(NULL); /* Client output buffer limits */ - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].hard_limit_bytes = 0; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].soft_limit_bytes = 0; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_NORMAL].soft_limit_seconds = 0; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_SLAVE].hard_limit_bytes = 1024*1024*256; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_SLAVE].soft_limit_bytes = 1024*1024*64; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_SLAVE].soft_limit_seconds = 60; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_PUBSUB].hard_limit_bytes = 1024*1024*32; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_PUBSUB].soft_limit_bytes = 1024*1024*8; - server.client_obuf_limits[REDIS_CLIENT_LIMIT_CLASS_PUBSUB].soft_limit_seconds = 60; + for (j = 0; j < REDIS_CLIENT_LIMIT_NUM_CLASSES; j++) + server.client_obuf_limits[j] = clientBufferLimitsDefaults[j]; /* Double constants initialization */ R_Zero = 0.0; diff --git a/src/redis.h b/src/redis.h index 90a4c084a36..1b076a11d4b 100644 --- a/src/redis.h +++ b/src/redis.h @@ -492,6 +492,8 @@ typedef struct clientBufferLimitsConfig { time_t soft_limit_seconds; } clientBufferLimitsConfig; +extern clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_LIMIT_NUM_CLASSES]; + /* The redisOp structure defines a Redis Operation, that is an instance of * a command with an argument vector, database ID, propagation target * (REDIS_PROPAGATE_*), and command pointer. From 8cf0b666068fab532414bb60e809a5f04064aff2 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 May 2013 10:22:55 +0200 Subject: [PATCH 0607/1752] CONFIG REWRITE: strip multiple empty lines. --- src/config.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index ef3d7157420..bfca82c31a7 100644 --- a/src/config.c +++ b/src/config.c @@ -1374,11 +1374,20 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state } } +/* Glue together the configuration lines in the current configuration + * rewrite state into a single string, stripping multiple empty lines. */ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { sds content = sdsempty(); - int j; + int j, was_empty = 0; for (j = 0; j < state->numlines; j++) { + /* Every cluster of empty lines is turned into a single empty line. */ + if (sdslen(state->lines[j]) == 0) { + if (was_empty) continue; + was_empty = 1; + } else { + was_empty = 0; + } content = sdscatsds(content,state->lines[j]); content = sdscatlen(content,"\n",1); } From a461376ca9a28e7e2ef27674e9f7d2fcc9da38d7 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 May 2013 11:17:18 +0200 Subject: [PATCH 0608/1752] CONFIG REWRITE: remove orphaned lines. --- src/config.c | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index bfca82c31a7..3c6a8998028 100644 --- a/src/config.c +++ b/src/config.c @@ -1218,6 +1218,7 @@ void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, lo rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a yes/no option. */ void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %s",option, @@ -1226,6 +1227,7 @@ void rewriteConfigYesNoOption(struct rewriteConfigState *state, char *option, in rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a string option. */ void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, char *value, char *defvalue) { int force = 1; sds line; @@ -1244,6 +1246,7 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, c rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a numerical (long long range) option. */ void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); @@ -1251,6 +1254,7 @@ void rewriteConfigNumericalOption(struct rewriteConfigState *state, char *option rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite a octal option. */ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, int value, int defvalue) { int force = value != defvalue; sds line = sdscatprintf(sdsempty(),"%s %o",option,value); @@ -1258,6 +1262,9 @@ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, in rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite an enumeration option, after the "value" every enum/value pair + * is specified, terminated by NULL. After NULL the default value is + * specified. See how the function is used for more information. */ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, ...) { va_list ap; char *enum_name, *matching_name; @@ -1281,6 +1288,7 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite the syslog-fability option. */ void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { int value = server.syslog_facility, j; int force = value != LOG_LOCAL0; @@ -1297,6 +1305,7 @@ void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite the save option. */ void rewriteConfigSaveOption(struct rewriteConfigState *state) { int j; sds line; @@ -1311,6 +1320,7 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { } } +/* Rewrite the dir option, always using absolute paths.*/ void rewriteConfigDirOption(struct rewriteConfigState *state) { char cwd[1024]; @@ -1318,6 +1328,7 @@ void rewriteConfigDirOption(struct rewriteConfigState *state) { rewriteConfigStringOption(state,"dir",cwd,NULL); } +/* Rewrite the slaveof option. */ void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { char *option = "slaveof"; sds line; @@ -1330,6 +1341,7 @@ void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,1); } +/* Rewrite the appendonly option. */ void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { int force = server.aof_state != REDIS_AOF_OFF; char *option = "appendonly"; @@ -1340,6 +1352,7 @@ void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite the notify-keyspace-events option. */ void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { int force = server.notify_keyspace_events != 0; char *option = "notify-keyspace-events"; @@ -1351,6 +1364,7 @@ void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,force); } +/* Rewrite the client-output-buffer-limit option. */ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state) { int j; char *option = "client-output-buffer-limit"; @@ -1394,12 +1408,40 @@ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { return content; } +/* Free the configuration rewrite state. */ void rewriteConfigReleaseState(struct rewriteConfigState *state) { sdsfreesplitres(state->lines,state->numlines); dictRelease(state->option_to_line); zfree(state); } +/* At the end of the rewrite process the state contains the remaining + * map between "option name" => "lines in the original config file". + * Lines used by the rewrite process were removed by the function + * rewriteConfigRewriteLine(), all the other lines are "orphaned" and + * should be replaced by empty lines. + * + * This function does just this, iterating all the option names and + * blanking all the lines still associated. */ +void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { + dictIterator *di = dictGetIterator(state->option_to_line); + dictEntry *de; + + while((de = dictNext(di)) != NULL) { + list *l = dictGetVal(de); + + while(listLength(l)) { + listNode *ln = listFirst(l); + int linenum = (long) ln->value; + + sdsfree(state->lines[linenum]); + state->lines[linenum] = sdsempty(); + listDelNode(l,ln); + } + } + dictReleaseIterator(di); +} + /* Rewrite the configuration file at "path". * If the configuration file already exists, we try at best to retain comments * and overall structure. @@ -1497,7 +1539,7 @@ int rewriteConfig(char *path) { /* Step 3: remove all the orphaned lines in the old file, that is, lines * that were used by a config option and are no longer used, like in case * of multiple "save" options or duplicated options. */ -// rewriteConfigRemoveOrphaned(state); + rewriteConfigRemoveOrphaned(state); /* Step 4: generate a new configuration file from the modified state * and write it into the original file. */ From 8feb3d261a594d0a5b1e3953af208faaadbce8d8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 May 2013 12:32:25 +0200 Subject: [PATCH 0609/1752] CONFIG REWRITE: actually rewrite the config file, atomically. --- src/config.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/src/config.c b/src/config.c index 3c6a8998028..2196e6bec04 100644 --- a/src/config.c +++ b/src/config.c @@ -28,9 +28,11 @@ * POSSIBILITY OF SUCH DAMAGE. */ - #include "redis.h" +#include +#include + static struct { const char *name; const int value; @@ -1442,6 +1444,63 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { dictReleaseIterator(di); } +/* This function overwrites the old configuration file with the new content. + * + * 1) The old file length is obtained. + * 2) If the new content is smaller, padding is added. + * 3) A single write(2) call is used to replace the content of the file. + * 4) Later the file is truncated to the length of the new content. + * + * This way we are sure the file is left in a consistent state even if the + * process is stopped between any of the four operations. + * + * The function returns 0 on success, otherwise -1 is returned and errno + * set accordingly. */ +int rewriteConfigOverwriteFile(char *configfile, sds content) { + int retval = 0; + int fd = open(configfile,O_RDWR|O_CREAT); + int content_size = sdslen(content), padding = 0; + struct stat sb; + sds content_padded; + + /* 1) Open the old file (or create a new one if it does not + * exist), get the size. */ + if (fd == -1) return -1; /* errno set by open(). */ + if (fstat(fd,&sb) == -1) { + close(fd); + return -1; /* errno set by fstat(). */ + } + + /* 2) Pad the content at least match the old file size. */ + content_padded = sdsdup(content); + if (content_size < sb.st_size) { + /* If the old file was bigger, pad the content with + * a newline plus as many "#" chars as required. */ + padding = sb.st_size - content_size; + content_padded = sdsgrowzero(content_padded,sb.st_size); + content_padded[content_size] = '\n'; + memset(content_padded+content_size+1,'#',padding-1); + } + + /* 3) Write the new content using a single write(2). */ + if (write(fd,content_padded,strlen(content_padded)) == -1) { + retval = -1; + goto cleanup; + } + + /* 4) Truncate the file to the right length if we used padding. */ + if (padding) { + if (ftruncate(fd,content_size) == -1) { + /* Non critical error... */ + } + } + +cleanup: + sdsfree(content_padded); + close(fd); + return retval; +} + /* Rewrite the configuration file at "path". * If the configuration file already exists, we try at best to retain comments * and overall structure. @@ -1453,6 +1512,7 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { int rewriteConfig(char *path) { struct rewriteConfigState *state; sds newcontent; + int retval; /* Step 1: read the old config into our rewrite state. */ if ((state = rewriteConfigReadOldFile(path)) == NULL) return -1; @@ -1544,11 +1604,11 @@ int rewriteConfig(char *path) { /* Step 4: generate a new configuration file from the modified state * and write it into the original file. */ newcontent = rewriteConfigGetContentFromState(state); - printf("%s\n", newcontent); + retval = rewriteConfigOverwriteFile(server.configfile,newcontent); sdsfree(newcontent); rewriteConfigReleaseState(state); - return 0; + return retval; } /*----------------------------------------------------------------------------- From 417253afc3f7e922e624d1b67dd83685187c3da8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 14 May 2013 12:45:04 +0200 Subject: [PATCH 0610/1752] CONFIG REWRITE: Use sane perms when creating redis.conf from scratch. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 2196e6bec04..2aa786c0644 100644 --- a/src/config.c +++ b/src/config.c @@ -1458,7 +1458,7 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { * set accordingly. */ int rewriteConfigOverwriteFile(char *configfile, sds content) { int retval = 0; - int fd = open(configfile,O_RDWR|O_CREAT); + int fd = open(configfile,O_RDWR|O_CREAT,0644); int content_size = sdslen(content), padding = 0; struct stat sb; sds content_padded; From 180cfaae8eb53ce52fa8fda68faa338290991fd0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 10:12:29 +0200 Subject: [PATCH 0611/1752] Added a define for most configuration defaults. Also the logfile option was modified to always have an explicit value and to log to stdout when an empty string is used as log file. Previously there was special handling of the string "stdout" that set the logfile to NULL, this always required some special handling. --- redis.conf | 4 ++-- src/config.c | 46 ++++++++++++++++++++--------------------- src/redis.c | 58 ++++++++++++++++++++++++++-------------------------- src/redis.h | 20 ++++++++++++++++++ 4 files changed, 73 insertions(+), 55 deletions(-) diff --git a/redis.conf b/redis.conf index 581dbf971b9..55a4a91d268 100644 --- a/redis.conf +++ b/redis.conf @@ -63,10 +63,10 @@ tcp-keepalive 0 # warning (only very important / critical messages are logged) loglevel notice -# Specify the log file name. Also 'stdout' can be used to force +# Specify the log file name. Also the emptry string can be used to force # Redis to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null -logfile stdout +logfile "" # To enable logging to the system logger, just set 'syslog-enabled' to yes, # and optionally update the other syslog parameters to suit your needs. diff --git a/src/config.c b/src/config.c index 2aa786c0644..fe2e27128a4 100644 --- a/src/config.c +++ b/src/config.c @@ -164,12 +164,9 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"logfile") && argc == 2) { FILE *logfp; + zfree(server.logfile); server.logfile = zstrdup(argv[1]); - if (!strcasecmp(server.logfile,"stdout")) { - zfree(server.logfile); - server.logfile = NULL; - } - if (server.logfile) { + if (server.logfile[0] != '\0') { /* Test if we are able to open the file. The server will not * be able to abort just for this problem later... */ logfp = fopen(server.logfile,"a"); @@ -1527,38 +1524,39 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"port",server.port,REDIS_SERVERPORT); rewriteConfigStringOption(state,"bindaddr",server.bindaddr,NULL); rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); - rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,0); - rewriteConfigNumericalOption(state,"timeout",server.maxidletime,0); - rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,0); + rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,REDIS_DEFAULT_UNIX_SOCKET_PERM); + rewriteConfigNumericalOption(state,"timeout",server.maxidletime,REDIS_MAXIDLETIME); + rewriteConfigNumericalOption(state,"tcp-keepalive",server.tcpkeepalive,REDIS_DEFAULT_TCP_KEEPALIVE); rewriteConfigEnumOption(state,"loglevel",server.verbosity, "debug", REDIS_DEBUG, "verbose", REDIS_VERBOSE, "notice", REDIS_NOTICE, "warning", REDIS_WARNING, - NULL, REDIS_NOTICE); - rewriteConfigStringOption(state,"logfile",server.logfile,"stdout"); - rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,0); + NULL, REDIS_DEFAULT_VERBOSITY); + rewriteConfigStringOption(state,"logfile",server.logfile,REDIS_DEFAULT_LOGFILE); + rewriteConfigYesNoOption(state,"syslog-enabled",server.syslog_enabled,REDIS_DEFAULT_SYSLOG_ENABLED); rewriteConfigStringOption(state,"syslog-ident",server.syslog_ident,REDIS_DEFAULT_SYSLOG_IDENT); rewriteConfigSyslogfacilityOption(state); rewriteConfigSaveOption(state); rewriteConfigNumericalOption(state,"databases",server.dbnum,REDIS_DEFAULT_DBNUM); - rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,1); - rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,1); - rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,1); - rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,"dump.rdb"); + rewriteConfigYesNoOption(state,"stop-writes-on-bgsave-error",server.stop_writes_on_bgsave_err,REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR); + rewriteConfigYesNoOption(state,"rdbcompression",server.rdb_compression,REDIS_DEFAULT_RDB_COMPRESSION); + rewriteConfigYesNoOption(state,"rdbchecksum",server.rdb_checksum,REDIS_DEFAULT_RDB_CHECKSUM); + rewriteConfigStringOption(state,"dbfilename",server.rdb_filename,REDIS_DEFAULT_RDB_FILENAME); rewriteConfigDirOption(state); rewriteConfigSlaveofOption(state); rewriteConfigStringOption(state,"masterauth",server.masterauth,NULL); - rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,1); - rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,1); + rewriteConfigYesNoOption(state,"slave-serve-stale-data",server.repl_serve_stale_data,REDIS_DEFAULT_SLAVE_SERVE_STALE_DATA); + rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,REDIS_DEFAULT_SLAVE_READ_ONLY); rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,REDIS_REPL_PING_SLAVE_PERIOD); rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,REDIS_REPL_TIMEOUT); rewriteConfigNumericalOption(state,"repl-backlog-size",server.repl_backlog_size,REDIS_DEFAULT_REPL_BACKLOG_SIZE); rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT); + rewriteConfigBytesOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY); rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,REDIS_DEFAULT_SLAVE_PRIORITY); rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); rewriteConfigNumericalOption(state,"maxclients",server.maxclients,REDIS_MAX_CLIENTS); - rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,0); + rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,REDIS_DEFAULT_MAXMEMORY); rewriteConfigEnumOption(state,"maxmemory-policy",server.maxmemory_policy, "volatile-lru", REDIS_MAXMEMORY_VOLATILE_LRU, "allkeys-lru", REDIS_MAXMEMORY_ALLKEYS_LRU, @@ -1566,15 +1564,15 @@ int rewriteConfig(char *path) { "allkeys-random", REDIS_MAXMEMORY_ALLKEYS_RANDOM, "volatile-ttl", REDIS_MAXMEMORY_VOLATILE_TTL, "noeviction", REDIS_MAXMEMORY_NO_EVICTION, - NULL, REDIS_MAXMEMORY_VOLATILE_LRU); - rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,3); + NULL, REDIS_DEFAULT_MAXMEMORY_POLICY); + rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,REDIS_DEFAULT_MAXMEMORY_SAMPLES); rewriteConfigAppendonlyOption(state); rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync, "eveysec", AOF_FSYNC_EVERYSEC, "always", AOF_FSYNC_ALWAYS, "no", AOF_FSYNC_NO, - NULL, AOF_FSYNC_EVERYSEC); - rewriteConfigYesNoOption(state,"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite,0); + NULL, REDIS_DEFAULT_AOF_FSYNC); + rewriteConfigYesNoOption(state,"no-appendfsync-on-rewrite",server.aof_no_fsync_on_rewrite,REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE); rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,REDIS_AOF_REWRITE_PERC); rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,REDIS_AOF_REWRITE_MIN_SIZE); rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,REDIS_LUA_TIME_LIMIT); @@ -1591,10 +1589,10 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE); - rewriteConfigYesNoOption(state,"active-rehashing",server.activerehashing,1); + rewriteConfigYesNoOption(state,"active-rehashing",server.activerehashing,REDIS_DEFAULT_ACTIVE_REHASHING); rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ); - rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,1); + rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); /* Step 3: remove all the orphaned lines in the old file, that is, lines * that were used by a config option and are no longer used, like in case diff --git a/src/redis.c b/src/redis.c index a9565e17650..6a011a9e2f5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -264,11 +264,12 @@ void redisLogRaw(int level, const char *msg) { FILE *fp; char buf[64]; int rawmode = (level & REDIS_LOG_RAW); + int log_to_stdout = server.logfile[0] == '\0'; level &= 0xff; /* clear flags */ if (level < server.verbosity) return; - fp = (server.logfile == NULL) ? stdout : fopen(server.logfile,"a"); + fp = log_to_stdout ? stdout : fopen(server.logfile,"a"); if (!fp) return; if (rawmode) { @@ -284,8 +285,7 @@ void redisLogRaw(int level, const char *msg) { } fflush(fp); - if (server.logfile) fclose(fp); - + if (!log_to_stdout) fclose(fp); if (server.syslog_enabled) syslog(syslogLevelMap[level], "%s", msg); } @@ -313,13 +313,13 @@ void redisLog(int level, const char *fmt, ...) { * where we need printf-alike features are served by redisLog(). */ void redisLogFromHandler(int level, const char *msg) { int fd; + int log_to_stdout = server.logfile[0] == '\0'; char buf[64]; - if ((level&0xff) < server.verbosity || - (server.logfile == NULL && server.daemonize)) return; - fd = server.logfile ? - open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644) : - STDOUT_FILENO; + if ((level&0xff) < server.verbosity || (log_to_stdout && server.daemonize)) + return; + fd = log_to_stdout ? STDOUT_FILENO : + open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644); if (fd == -1) return; ll2string(buf,sizeof(buf),getpid()); if (write(fd,"[",1) == -1) goto err; @@ -331,7 +331,7 @@ void redisLogFromHandler(int level, const char *msg) { if (write(fd,msg,strlen(msg)) == -1) goto err; if (write(fd,"\n",1) == -1) goto err; err: - if (server.logfile) close(fd); + if (!log_to_stdout) close(fd); } /* Return the UNIX time in microseconds */ @@ -1172,25 +1172,25 @@ void initServerConfig() { server.port = REDIS_SERVERPORT; server.bindaddr = NULL; server.unixsocket = NULL; - server.unixsocketperm = 0; + server.unixsocketperm = REDIS_DEFAULT_UNIX_SOCKET_PERM; server.ipfd = -1; server.sofd = -1; server.dbnum = REDIS_DEFAULT_DBNUM; - server.verbosity = REDIS_NOTICE; + server.verbosity = REDIS_DEFAULT_VERBOSITY; server.maxidletime = REDIS_MAXIDLETIME; - server.tcpkeepalive = 0; + server.tcpkeepalive = REDIS_DEFAULT_TCP_KEEPALIVE; server.active_expire_enabled = 1; server.client_max_querybuf_len = REDIS_MAX_QUERYBUF_LEN; server.saveparams = NULL; server.loading = 0; - server.logfile = NULL; /* NULL = log on standard output */ - server.syslog_enabled = 0; + server.logfile = zstrdup(REDIS_DEFAULT_LOGFILE); + server.syslog_enabled = REDIS_DEFAULT_SYSLOG_ENABLED; server.syslog_ident = zstrdup(REDIS_DEFAULT_SYSLOG_IDENT); server.syslog_facility = LOG_LOCAL0; - server.daemonize = 0; + server.daemonize = REDIS_DEFAULT_DAEMONIZE; server.aof_state = REDIS_AOF_OFF; - server.aof_fsync = AOF_FSYNC_EVERYSEC; - server.aof_no_fsync_on_rewrite = 0; + server.aof_fsync = REDIS_DEFAULT_AOF_FSYNC; + server.aof_no_fsync_on_rewrite = REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE; server.aof_rewrite_perc = REDIS_AOF_REWRITE_PERC; server.aof_rewrite_min_size = REDIS_AOF_REWRITE_MIN_SIZE; server.aof_rewrite_base_size = 0; @@ -1203,21 +1203,21 @@ void initServerConfig() { server.aof_fd = -1; server.aof_selected_db = -1; /* Make sure the first time will not match */ server.aof_flush_postponed_start = 0; - server.aof_rewrite_incremental_fsync = 1; + server.aof_rewrite_incremental_fsync = REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC; server.pidfile = zstrdup(REDIS_DEFAULT_PID_FILE); - server.rdb_filename = zstrdup("dump.rdb"); + server.rdb_filename = zstrdup(REDIS_DEFAULT_RDB_FILENAME); server.aof_filename = zstrdup("appendonly.aof"); server.requirepass = NULL; - server.rdb_compression = 1; - server.rdb_checksum = 1; - server.stop_writes_on_bgsave_err = 1; - server.activerehashing = 1; + server.rdb_compression = REDIS_DEFAULT_RDB_COMPRESSION; + server.rdb_checksum = REDIS_DEFAULT_RDB_CHECKSUM; + server.stop_writes_on_bgsave_err = REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR; + server.activerehashing = REDIS_DEFAULT_ACTIVE_REHASHING; server.notify_keyspace_events = 0; server.maxclients = REDIS_MAX_CLIENTS; server.bpop_blocked_clients = 0; - server.maxmemory = 0; - server.maxmemory_policy = REDIS_MAXMEMORY_VOLATILE_LRU; - server.maxmemory_samples = 3; + server.maxmemory = REDIS_DEFAULT_MAXMEMORY; + server.maxmemory_policy = REDIS_DEFAULT_MAXMEMORY_POLICY; + server.maxmemory_samples = REDIS_DEFAULT_MAXMEMORY_SAMPLES; server.hash_max_ziplist_entries = REDIS_HASH_MAX_ZIPLIST_ENTRIES; server.hash_max_ziplist_value = REDIS_HASH_MAX_ZIPLIST_VALUE; server.list_max_ziplist_entries = REDIS_LIST_MAX_ZIPLIST_ENTRIES; @@ -1248,10 +1248,10 @@ void initServerConfig() { server.repl_master_initial_offset = -1; server.repl_state = REDIS_REPL_NONE; server.repl_syncio_timeout = REDIS_REPL_SYNCIO_TIMEOUT; - server.repl_serve_stale_data = 1; - server.repl_slave_ro = 1; + server.repl_serve_stale_data = REDIS_DEFAULT_SLAVE_SERVE_STALE_DATA; + server.repl_slave_ro = REDIS_DEFAULT_SLAVE_READ_ONLY; server.repl_down_since = 0; /* Never connected, repl is down since EVER. */ - server.repl_disable_tcp_nodelay = 0; + server.repl_disable_tcp_nodelay = REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY; server.slave_priority = REDIS_DEFAULT_SLAVE_PRIORITY; server.master_repl_offset = 0; diff --git a/src/redis.h b/src/redis.h index 1b076a11d4b..d8b10d012f1 100644 --- a/src/redis.h +++ b/src/redis.h @@ -101,6 +101,23 @@ #define REDIS_DEFAULT_PID_FILE "/var/run/redis.pid" #define REDIS_DEFAULT_SYSLOG_IDENT "redis" #define REDIS_DEFAULT_CLUSTER_CONFIG_FILE "nodes.conf" +#define REDIS_DEFAULT_DAEMONIZE 0 +#define REDIS_DEFAULT_UNIX_SOCKET_PERM 0 +#define REDIS_DEFAULT_TCP_KEEPALIVE 0 +#define REDIS_DEFAULT_LOGFILE "" +#define REDIS_DEFAULT_SYSLOG_ENABLED 0 +#define REDIS_DEFAULT_STOP_WRITES_ON_BGSAVE_ERROR 1 +#define REDIS_DEFAULT_RDB_COMPRESSION 1 +#define REDIS_DEFAULT_RDB_CHECKSUM 1 +#define REDIS_DEFAULT_RDB_FILENAME "dump.rdb" +#define REDIS_DEFAULT_SLAVE_SERVE_STALE_DATA 1 +#define REDIS_DEFAULT_SLAVE_READ_ONLY 1 +#define REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY 0 +#define REDIS_DEFAULT_MAXMEMORY 0 +#define REDIS_DEFAULT_MAXMEMORY_SAMPLES 3 +#define REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE 0 +#define REDIS_DEFAULT_ACTIVE_REHASHING 1 +#define REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ @@ -243,6 +260,7 @@ #define REDIS_NOTICE 2 #define REDIS_WARNING 3 #define REDIS_LOG_RAW (1<<10) /* Modifier to log without timestamp */ +#define REDIS_DEFAULT_VERBOSITY REDIS_NOTICE /* Anti-warning macro... */ #define REDIS_NOTUSED(V) ((void) V) @@ -254,6 +272,7 @@ #define AOF_FSYNC_NO 0 #define AOF_FSYNC_ALWAYS 1 #define AOF_FSYNC_EVERYSEC 2 +#define REDIS_DEFAULT_AOF_FSYNC AOF_FSYNC_EVERYSEC /* Zip structure related defaults */ #define REDIS_HASH_MAX_ZIPLIST_ENTRIES 512 @@ -276,6 +295,7 @@ #define REDIS_MAXMEMORY_ALLKEYS_LRU 3 #define REDIS_MAXMEMORY_ALLKEYS_RANDOM 4 #define REDIS_MAXMEMORY_NO_EVICTION 5 +#define REDIS_DEFAULT_MAXMEMORY_POLICY REDIS_MAXMEMORY_VOLATILE_LRU /* Scripting */ #define REDIS_LUA_TIME_LIMIT 5000 /* milliseconds */ From e8cc95930002731b1fe15b13f2a15262c8a689b6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 10:14:36 +0200 Subject: [PATCH 0612/1752] Test default config modified to set logfile to empty string. Required because of recent changes in the way logfile is set to standard output. --- tests/assets/default.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/assets/default.conf b/tests/assets/default.conf index 9f95700d1c8..17d21b07c31 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -6,7 +6,7 @@ pidfile /var/run/redis.pid port 6379 timeout 0 loglevel verbose -logfile stdout +logfile '' databases 16 save 900 1 From c95a60960be50b3dad2512fc85b5e561dd4e73f4 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:04:53 +0200 Subject: [PATCH 0613/1752] CONFIG REWRITE: repl-disable-tcp-nodelay is a boolean option. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index fe2e27128a4..450e52d1d21 100644 --- a/src/config.c +++ b/src/config.c @@ -1552,7 +1552,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,REDIS_REPL_TIMEOUT); rewriteConfigNumericalOption(state,"repl-backlog-size",server.repl_backlog_size,REDIS_DEFAULT_REPL_BACKLOG_SIZE); rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT); - rewriteConfigBytesOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY); + rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY); rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,REDIS_DEFAULT_SLAVE_PRIORITY); rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); rewriteConfigNumericalOption(state,"maxclients",server.maxclients,REDIS_MAX_CLIENTS); From 2a29cf430a155ac3dbdc964e84e30298f21db3e9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:06:56 +0200 Subject: [PATCH 0614/1752] CONFIG REWRITE: fixed typo in AOF fsync policy. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 450e52d1d21..f0169805bff 100644 --- a/src/config.c +++ b/src/config.c @@ -1568,7 +1568,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,REDIS_DEFAULT_MAXMEMORY_SAMPLES); rewriteConfigAppendonlyOption(state); rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync, - "eveysec", AOF_FSYNC_EVERYSEC, + "everysec", AOF_FSYNC_EVERYSEC, "always", AOF_FSYNC_ALWAYS, "no", AOF_FSYNC_NO, NULL, REDIS_DEFAULT_AOF_FSYNC); From b4ce02988ccdf4414f1d4eb7ab4005f714414c53 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:09:19 +0200 Subject: [PATCH 0615/1752] CONFIG REWRITE: "active-rehashing" -> "activerehashing". --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index f0169805bff..a9fd392cc34 100644 --- a/src/config.c +++ b/src/config.c @@ -1589,7 +1589,7 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"set-max-intset-entries",server.set_max_intset_entries,REDIS_SET_MAX_INTSET_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-entries",server.zset_max_ziplist_entries,REDIS_ZSET_MAX_ZIPLIST_ENTRIES); rewriteConfigNumericalOption(state,"zset-max-ziplist-value",server.zset_max_ziplist_value,REDIS_ZSET_MAX_ZIPLIST_VALUE); - rewriteConfigYesNoOption(state,"active-rehashing",server.activerehashing,REDIS_DEFAULT_ACTIVE_REHASHING); + rewriteConfigYesNoOption(state,"activerehashing",server.activerehashing,REDIS_DEFAULT_ACTIVE_REHASHING); rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ); rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); From 901ceab20b9a39cb2196f5404f5b89a2af328ef8 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:15:18 +0200 Subject: [PATCH 0616/1752] CONFIG REWRITE: correctly escape the notify-keyspace-events option. --- src/config.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index a9fd392cc34..e3cb80118b2 100644 --- a/src/config.c +++ b/src/config.c @@ -1358,7 +1358,9 @@ void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { sds line, flags; flags = keyspaceEventsFlagsToString(server.notify_keyspace_events); - line = sdscatprintf(sdsempty(),"%s %s", option, flags); + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatrepr(line, flags, sdslen(flags)); sdsfree(flags); rewriteConfigRewriteLine(state,option,line,force); } From 2173f30836a632f1aac94e4babab74f86855288f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:33:02 +0200 Subject: [PATCH 0617/1752] CONFIG REWRITE: when rewriting amount of bytes use GB, MB, KB if possible. --- src/config.c | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/config.c b/src/config.c index e3cb80118b2..05db7d45f2a 100644 --- a/src/config.c +++ b/src/config.c @@ -1208,13 +1208,34 @@ void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sd sdsfree(o); } +/* Write the long long 'bytes' value as a string in a way that is parsable + * inside redis.conf. If possible uses the GB, MB, KB notation. */ +int rewriteConfigFormatMemory(char *buf, size_t len, long long bytes) { + int gb = 1024*1024*1024; + int mb = 1024*1024; + int kb = 1024; + + if (bytes && (bytes % gb) == 0) { + return snprintf(buf,len,"%lldgb",bytes/gb); + } else if (bytes && (bytes % mb) == 0) { + return snprintf(buf,len,"%lldmb",bytes/mb); + } else if (bytes && (bytes % kb) == 0) { + return snprintf(buf,len,"%lldkb",bytes/kb); + } else { + return snprintf(buf,len,"%lld",bytes); + } +} + /* Rewrite a simple "option-name " configuration option. */ void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, long long value, long long defvalue) { + char buf[64]; int force = value != defvalue; - /* TODO: check if we can write it using MB, GB, or other suffixes. */ - sds line = sdscatprintf(sdsempty(),"%s %lld",option,value); + sds line; + rewriteConfigFormatMemory(buf,sizeof(buf),value); + line = sdscatprintf(sdsempty(),"%s %s",option,buf); rewriteConfigRewriteLine(state,option,line,force); + } /* Rewrite a yes/no option. */ @@ -1378,12 +1399,15 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state (server.client_obuf_limits[j].soft_limit_seconds != clientBufferLimitsDefaults[j].soft_limit_seconds); sds line; + char hard[64], soft[64]; + + rewriteConfigFormatMemory(hard,sizeof(hard), + server.client_obuf_limits[j].hard_limit_bytes); + rewriteConfigFormatMemory(soft,sizeof(soft), + server.client_obuf_limits[j].soft_limit_bytes); - line = sdscatprintf(sdsempty(),"%s %s %llu %llu %ld", - option, - getClientLimitClassName(j), - server.client_obuf_limits[j].hard_limit_bytes, - server.client_obuf_limits[j].soft_limit_bytes, + line = sdscatprintf(sdsempty(),"%s %s %s %s %ld", + option, getClientLimitClassName(j), hard, soft, (long) server.client_obuf_limits[j].soft_limit_seconds); rewriteConfigRewriteLine(state,option,line,force); } From 7d338fd982650926f228c681de815940b538142f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:38:43 +0200 Subject: [PATCH 0618/1752] CONFIG REWRITE: bindaddr -> bind. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 05db7d45f2a..12d27186a2e 100644 --- a/src/config.c +++ b/src/config.c @@ -1548,7 +1548,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); rewriteConfigStringOption(state,"pidfile",server.pidfile,REDIS_DEFAULT_PID_FILE); rewriteConfigNumericalOption(state,"port",server.port,REDIS_SERVERPORT); - rewriteConfigStringOption(state,"bindaddr",server.bindaddr,NULL); + rewriteConfigStringOption(state,"bind",server.bindaddr,NULL); rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,REDIS_DEFAULT_UNIX_SOCKET_PERM); rewriteConfigNumericalOption(state,"timeout",server.maxidletime,REDIS_MAXIDLETIME); From 604efb02ab6f331b219e3f74d8fad8ba6caafa00 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:39:29 +0200 Subject: [PATCH 0619/1752] CONFIG REWRITE: backlog size is a bytes option. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 12d27186a2e..dc974620682 100644 --- a/src/config.c +++ b/src/config.c @@ -1576,7 +1576,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"slave-read-only",server.repl_slave_ro,REDIS_DEFAULT_SLAVE_READ_ONLY); rewriteConfigNumericalOption(state,"repl-ping-slave-period",server.repl_ping_slave_period,REDIS_REPL_PING_SLAVE_PERIOD); rewriteConfigNumericalOption(state,"repl-timeout",server.repl_timeout,REDIS_REPL_TIMEOUT); - rewriteConfigNumericalOption(state,"repl-backlog-size",server.repl_backlog_size,REDIS_DEFAULT_REPL_BACKLOG_SIZE); + rewriteConfigBytesOption(state,"repl-backlog-size",server.repl_backlog_size,REDIS_DEFAULT_REPL_BACKLOG_SIZE); rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT); rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY); rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,REDIS_DEFAULT_SLAVE_PRIORITY); From 414d5db85caf3f37c121697085dcb3b2bcec38b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 11:55:14 +0200 Subject: [PATCH 0620/1752] Use memtoll() when parsing the backlog-size option. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index dc974620682..a4e2912a6a1 100644 --- a/src/config.c +++ b/src/config.c @@ -256,7 +256,7 @@ void loadServerConfigFromString(char *config) { err = "argument must be 'yes' or 'no'"; goto loaderr; } } else if (!strcasecmp(argv[0],"repl-backlog-size") && argc == 2) { - long long size = strtoll(argv[1],NULL,10); + long long size = memtoll(argv[1],NULL); if (size <= 0) { err = "repl-backlog-size must be 1 or greater."; goto loaderr; From 059018c4cd7f19b016cc2b2069c91599f47a8965 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 15 May 2013 12:02:44 +0200 Subject: [PATCH 0621/1752] CONFIG REWRITE: remove cluster stuff for 2.8 back port. --- src/config.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/config.c b/src/config.c index a4e2912a6a1..a08a5a563aa 100644 --- a/src/config.c +++ b/src/config.c @@ -1602,9 +1602,6 @@ int rewriteConfig(char *path) { rewriteConfigNumericalOption(state,"auto-aof-rewrite-percentage",server.aof_rewrite_perc,REDIS_AOF_REWRITE_PERC); rewriteConfigBytesOption(state,"auto-aof-rewrite-min-size",server.aof_rewrite_min_size,REDIS_AOF_REWRITE_MIN_SIZE); rewriteConfigNumericalOption(state,"lua-time-limit",server.lua_time_limit,REDIS_LUA_TIME_LIMIT); - rewriteConfigYesNoOption(state,"cluster-enabled",server.cluster_enabled,0); - rewriteConfigStringOption(state,"cluster-config-file",server.cluster_configfile,REDIS_DEFAULT_CLUSTER_CONFIG_FILE); - rewriteConfigNumericalOption(state,"cluster-node-timeout",server.cluster_node_timeout,REDIS_CLUSTER_DEFAULT_NODE_TIMEOUT); rewriteConfigNumericalOption(state,"slowlog-log-slower-than",server.slowlog_log_slower_than,REDIS_SLOWLOG_LOG_SLOWER_THAN); rewriteConfigNumericalOption(state,"slowlog-max-len",server.slowlog_max_len,REDIS_SLOWLOG_MAX_LEN); rewriteConfigNotifykeyspaceeventsOption(state); From 0b398b6ffa8b0d99169b7ad4f955983509e787bf Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 18 May 2013 16:21:52 +0200 Subject: [PATCH 0622/1752] Don't stop reading redis.conf if line has no args. Should be "continue" and was "return". This fixes issue #1110 --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index a08a5a563aa..b69f2e133d7 100644 --- a/src/config.c +++ b/src/config.c @@ -105,7 +105,7 @@ void loadServerConfigFromString(char *config) { /* Skip this line if the resulting command vector is empty. */ if (argc == 0) { sdsfreesplitres(argv,argc); - return; + continue; } sdstolower(argv[0]); From 302b0a5c7b26e9e4f1bb1561b4719c4b56044a91 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 24 May 2013 18:41:43 +0200 Subject: [PATCH 0623/1752] Top comment for prepareClientToWrite() clarified. We don't write the output buffer to the client socket for slaves only if the slave is not online. --- src/networking.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 694d953e4b4..f421f348ea3 100644 --- a/src/networking.c +++ b/src/networking.c @@ -117,8 +117,8 @@ redisClient *createClient(int fd) { * loop so that when the socket is writable new data gets written. * * If the client should not receive new data, because it is a fake client - * or a slave, or because the setup of the write handler failed, the function - * returns REDIS_ERR. + * or a slave not yet online, or because the setup of the write handler + * failed, the function returns REDIS_ERR. * * Typically gets called every time a reply is built, before adding more * data to the clients output buffers. If the function returns REDIS_ERR no From 7ae29d5d9eb0daa10f15f5afa1c8e45a28a783b8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 24 May 2013 18:58:57 +0200 Subject: [PATCH 0624/1752] Replication: don't even queue replies to master commands. When master send commands, there is no need for the slave to reply. Redis used to queue the reply in the output buffer and discard the reply later, this is a waste of work and it is not clear why it was this way (I sincerely don't remember). This commit changes it in order to don't queue the reply at all. All tests passing. --- src/networking.c | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/src/networking.c b/src/networking.c index f421f348ea3..7349d4bea99 100644 --- a/src/networking.c +++ b/src/networking.c @@ -116,15 +116,15 @@ redisClient *createClient(int fd) { * returns REDIS_OK, and make sure to install the write handler in our event * loop so that when the socket is writable new data gets written. * - * If the client should not receive new data, because it is a fake client - * or a slave not yet online, or because the setup of the write handler + * If the client should not receive new data, because it is a fake client, + * a master, a slave not yet online, or because the setup of the write handler * failed, the function returns REDIS_ERR. * * Typically gets called every time a reply is built, before adding more * data to the clients output buffers. If the function returns REDIS_ERR no * data should be appended to the output buffers. */ int prepareClientToWrite(redisClient *c) { - if (c->flags & REDIS_LUA_CLIENT) return REDIS_OK; + if (c->flags & (REDIS_LUA_CLIENT|REDIS_MASTER)) return REDIS_OK; if (c->fd <= 0) return REDIS_ERR; /* Fake client */ if (c->bufpos == 0 && listLength(c->reply) == 0 && (c->replstate == REDIS_REPL_NONE || @@ -739,13 +739,8 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { while(c->bufpos > 0 || listLength(c->reply)) { if (c->bufpos > 0) { - if (c->flags & REDIS_MASTER) { - /* Don't reply to a master */ - nwritten = c->bufpos - c->sentlen; - } else { - nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); - if (nwritten <= 0) break; - } + nwritten = write(fd,c->buf+c->sentlen,c->bufpos-c->sentlen); + if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; @@ -765,13 +760,8 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { continue; } - if (c->flags & REDIS_MASTER) { - /* Don't reply to a master */ - nwritten = objlen - c->sentlen; - } else { - nwritten = write(fd, ((char*)o->ptr)+c->sentlen,objlen-c->sentlen); - if (nwritten <= 0) break; - } + nwritten = write(fd, ((char*)o->ptr)+c->sentlen,objlen-c->sentlen); + if (nwritten <= 0) break; c->sentlen += nwritten; totwritten += nwritten; From 0387f86947c8ad91ed8ee3847e1c7c512b386e58 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 25 May 2013 01:00:07 +0200 Subject: [PATCH 0625/1752] Fixed a bug in no queueing replies to master. --- src/networking.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 7349d4bea99..97394b40c3f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -124,7 +124,8 @@ redisClient *createClient(int fd) { * data to the clients output buffers. If the function returns REDIS_ERR no * data should be appended to the output buffers. */ int prepareClientToWrite(redisClient *c) { - if (c->flags & (REDIS_LUA_CLIENT|REDIS_MASTER)) return REDIS_OK; + if (c->flags & REDIS_LUA_CLIENT) return REDIS_OK; + if (c->flags & REDIS_MASTER) return REDIS_ERR; if (c->fd <= 0) return REDIS_ERR; /* Fake client */ if (c->bufpos == 0 && listLength(c->reply) == 0 && (c->replstate == REDIS_REPL_NONE || From 1e77b77de442358425df9ef7ab23181814dfc04e Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 25 May 2013 00:37:56 +0200 Subject: [PATCH 0626/1752] REPLCONF ACK command. This special command is used by the slave to inform the master the amount of replication stream it currently consumed. it does not return anything so that we not need to consume additional bandwidth needed by the master to reply something. The master can do a number of things knowing the amount of stream processed, such as understanding the "lag" in bytes of the slave, verify if a given command was already processed by the slave, and so forth. --- src/networking.c | 2 ++ src/redis.h | 2 ++ src/replication.c | 13 +++++++++++++ 3 files changed, 17 insertions(+) diff --git a/src/networking.c b/src/networking.c index 97394b40c3f..0f1ee684f01 100644 --- a/src/networking.c +++ b/src/networking.c @@ -88,6 +88,8 @@ redisClient *createClient(int fd) { c->authenticated = 0; c->replstate = REDIS_REPL_NONE; c->reploff = 0; + c->repl_ack_off = 0; + c->repl_ack_time = 0; c->slave_listening_port = 0; c->reply = listCreate(); c->reply_bytes = 0; diff --git a/src/redis.h b/src/redis.h index d8b10d012f1..512065228ac 100644 --- a/src/redis.h +++ b/src/redis.h @@ -449,6 +449,8 @@ typedef struct redisClient { long repldboff; /* replication DB file offset */ off_t repldbsize; /* replication DB file size */ long long reploff; /* replication offset if this is our master */ + long long repl_ack_off; /* replication ack offset, if this is a slave */ + long long repl_ack_time;/* replication ack time, if this is a slave */ char replrunid[REDIS_RUN_ID_SIZE+1]; /* master run id if this is a master */ int slave_listening_port; /* As configured with: SLAVECONF listening-port */ multiState mstate; /* MULTI/EXEC state */ diff --git a/src/replication.c b/src/replication.c index 6583f322a86..7532d8f8c87 100644 --- a/src/replication.c +++ b/src/replication.c @@ -588,6 +588,19 @@ void replconfCommand(redisClient *c) { &port,NULL) != REDIS_OK)) return; c->slave_listening_port = port; + } else if (!strcasecmp(c->argv[j]->ptr,"ack")) { + /* REPLCONF ACK is used by slave to inform the master the amount + * of replication stream that it processed so far. It is an + * internal only command that normal clients should never use. */ + long long offset; + + if (!(c->flags & REDIS_SLAVE)) return; + if ((getLongLongFromObject(c->argv[j+1], &offset) != REDIS_OK)) + return; + if (offset > c->repl_ack_off) + c->repl_ack_off = offset; + c->repl_ack_time = server.unixtime; + /* Note: this command does not reply anything! */ } else { addReplyErrorFormat(c,"Unrecognized REPLCONF option: %s", (char*)c->argv[j]->ptr); From 146f1d7d86173a82ecf33b80221c726aa35ab45e Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 25 May 2013 00:54:00 +0200 Subject: [PATCH 0627/1752] Replication: send REPLCONF ACK to master. --- src/networking.c | 3 ++- src/redis.c | 14 ++++++++++++-- src/redis.h | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 0f1ee684f01..b1ee2df84cd 100644 --- a/src/networking.c +++ b/src/networking.c @@ -127,7 +127,8 @@ redisClient *createClient(int fd) { * data should be appended to the output buffers. */ int prepareClientToWrite(redisClient *c) { if (c->flags & REDIS_LUA_CLIENT) return REDIS_OK; - if (c->flags & REDIS_MASTER) return REDIS_ERR; + if ((c->flags & REDIS_MASTER) && + !(c->flags & REDIS_MASTER_FORCE_REPLY)) return REDIS_ERR; if (c->fd <= 0) return REDIS_ERR; /* Fake client */ if (c->bufpos == 0 && listLength(c->reply) == 0 && (c->replstate == REDIS_REPL_NONE || diff --git a/src/redis.c b/src/redis.c index 6a011a9e2f5..ebf8c7ec401 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1798,6 +1798,15 @@ int processCommand(redisClient *c) { call(c,REDIS_CALL_FULL); if (listLength(server.ready_keys)) handleClientsBlockedOnLists(); + /* Acknowledge the master about the execution of this command. */ + if (c->flags & REDIS_MASTER) { + c->flags |= REDIS_MASTER_FORCE_REPLY; + addReplyMultiBulkLen(c,3); + addReplyBulkCString(c,"REPLCONF"); + addReplyBulkCString(c,"ACK"); + addReplyBulkLongLong(c,c->reploff); + c->flags &= ~REDIS_MASTER_FORCE_REPLY; + } } return REDIS_OK; } @@ -2264,8 +2273,9 @@ sds genRedisInfoString(char *section) { break; } if (state == NULL) continue; - info = sdscatprintf(info,"slave%d:%s,%d,%s\r\n", - slaveid,ip,slave->slave_listening_port,state); + info = sdscatprintf(info,"slave%d:%s,%d,%s,%lld\r\n", + slaveid,ip,slave->slave_listening_port,state, + slave->repl_ack_off); slaveid++; } } diff --git a/src/redis.h b/src/redis.h index 512065228ac..ba09516e37c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -212,6 +212,7 @@ #define REDIS_CLOSE_ASAP (1<<10)/* Close this client ASAP */ #define REDIS_UNIX_SOCKET (1<<11) /* Client connected via Unix domain socket */ #define REDIS_DIRTY_EXEC (1<<12) /* EXEC will fail for errors while queueing */ +#define REDIS_MASTER_FORCE_REPLY (1<<13) /* Queue replies even if is master */ /* Client request types */ #define REDIS_REQ_INLINE 1 From 0000d5334df14cc281231b79a1d5f0e17d343c7a Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 25 May 2013 01:22:27 +0200 Subject: [PATCH 0628/1752] Make sure that REPLCONF ACK really has no return value. --- src/replication.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/replication.c b/src/replication.c index 7532d8f8c87..7fd76af5f8b 100644 --- a/src/replication.c +++ b/src/replication.c @@ -601,6 +601,7 @@ void replconfCommand(redisClient *c) { c->repl_ack_off = offset; c->repl_ack_time = server.unixtime; /* Note: this command does not reply anything! */ + return; } else { addReplyErrorFormat(c,"Unrecognized REPLCONF option: %s", (char*)c->argv[j]->ptr); From a74f8fe1ad90b1f5f0a856857740c71b125b96a0 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 May 2013 10:41:53 +0200 Subject: [PATCH 0629/1752] Don't ACK the master after every command. Sending an ACK is now moved into the replicationSendAck() function. --- src/redis.c | 9 --------- src/replication.c | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/redis.c b/src/redis.c index ebf8c7ec401..3bf05065e4a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1798,15 +1798,6 @@ int processCommand(redisClient *c) { call(c,REDIS_CALL_FULL); if (listLength(server.ready_keys)) handleClientsBlockedOnLists(); - /* Acknowledge the master about the execution of this command. */ - if (c->flags & REDIS_MASTER) { - c->flags |= REDIS_MASTER_FORCE_REPLY; - addReplyMultiBulkLen(c,3); - addReplyBulkCString(c,"REPLCONF"); - addReplyBulkCString(c,"ACK"); - addReplyBulkLongLong(c,c->reploff); - c->flags &= ~REDIS_MASTER_FORCE_REPLY; - } } return REDIS_OK; } diff --git a/src/replication.c b/src/replication.c index 7fd76af5f8b..2042389d130 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1282,6 +1282,22 @@ void slaveofCommand(redisClient *c) { addReply(c,shared.ok); } +/* Send a REPLCONF ACK command to the master to inform it about the current + * processed offset. If we are not connected with a master, the command has + * no effects. */ +void replicationSendAck(void) { + redisClient *c = server.master; + + if (c != NULL) { + c->flags |= REDIS_MASTER_FORCE_REPLY; + addReplyMultiBulkLen(c,3); + addReplyBulkCString(c,"REPLCONF"); + addReplyBulkCString(c,"ACK"); + addReplyBulkLongLong(c,c->reploff); + c->flags &= ~REDIS_MASTER_FORCE_REPLY; + } +} + /* ---------------------- MASTER CACHING FOR PSYNC -------------------------- */ /* In order to implement partial synchronization we need to be able to cache @@ -1370,6 +1386,7 @@ void replicationResurrectCachedMaster(int newfd) { /* --------------------------- REPLICATION CRON ---------------------------- */ +/* Replication cron funciton, called 1 time per second. */ void replicationCron(void) { /* Non blocking connection timeout? */ if (server.masterhost && From 45e6a4023a0a40d5f882dd19aa874d94ada516b6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 May 2013 10:45:37 +0200 Subject: [PATCH 0630/1752] Send ACK to master once every second. ACKs can be also used as a base for synchronous replication. However in that case they'll be explicitly requested by the master when the client sends a request that needs to be replicated synchronously. --- src/replication.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/replication.c b/src/replication.c index 2042389d130..348428c56da 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1421,6 +1421,10 @@ void replicationCron(void) { redisLog(REDIS_NOTICE,"MASTER <-> SLAVE sync started"); } } + + /* Send ACK to master from time to time. */ + if (server.masterhost && server.master) + replicationSendAck(); /* If we have attached slaves, PING them from time to time. * So slaves can implement an explicit timeout to masters, and will From 308940aa2c22c60e62d4ae906ef3092079b92b14 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 May 2013 11:17:17 +0200 Subject: [PATCH 0631/1752] Close connection with timedout slaves. Now masters, using the time at which the last REPLCONF ACK was received, are able to explicitly disconnect slaves that are no longer responding. Previously the only chance was to see a very long output buffer, that was highly suboptimal. --- src/replication.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/replication.c b/src/replication.c index 348428c56da..a028359abbf 100644 --- a/src/replication.c +++ b/src/replication.c @@ -424,6 +424,7 @@ int masterTryPartialResynchronization(redisClient *c) { * 3) Send the backlog data (from the offset to the end) to the slave. */ c->flags |= REDIS_SLAVE; c->replstate = REDIS_REPL_ONLINE; + c->repl_ack_time = server.unixtime; listAddNodeTail(server.slaves,c); /* We can't use the connection buffers since they are used to accumulate * new commands at this stage. But we are sure the socket send buffer is @@ -655,6 +656,7 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { slave->repldbfd = -1; aeDeleteFileEvent(server.el,slave->fd,AE_WRITABLE); slave->replstate = REDIS_REPL_ONLINE; + slave->repl_ack_time = server.unixtime; if (aeCreateFileEvent(server.el, slave->fd, AE_WRITABLE, sendReplyToClient, slave) == AE_ERR) { freeClient(slave); @@ -1457,6 +1459,31 @@ void replicationCron(void) { } } + /* Disconnect timedout slaves. */ + if (listLength(server.slaves)) { + listIter li; + listNode *ln; + + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + redisClient *slave = ln->value; + + if (slave->replstate != REDIS_REPL_ONLINE) continue; + if ((server.unixtime - slave->repl_ack_time) > server.repl_timeout) + { + char ip[32]; + int port; + + if (anetPeerToString(slave->fd,ip,&port) != -1) { + redisLog(REDIS_WARNING, + "Disconnecting timedout slave: %s:%d", + ip, slave->slave_listening_port); + } + freeClient(slave); + } + } + } + /* If we have no attached slaves and there is a replication backlog * using memory, free it after some (configured) time. */ if (listLength(server.slaves) == 0 && server.repl_backlog_time_limit && From 75bf91abea5997d1d63cfc3b4fcaa04ff398d852 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 May 2013 11:24:05 +0200 Subject: [PATCH 0632/1752] redis.conf updated: repl-timeout now uesd by masters as well. --- redis.conf | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 55a4a91d268..6e728eac0a5 100644 --- a/redis.conf +++ b/redis.conf @@ -203,8 +203,11 @@ slave-read-only yes # # repl-ping-slave-period 10 -# The following option sets a timeout for both Bulk transfer I/O timeout and -# master data or ping response timeout. The default value is 60 seconds. +# The following option sets the replication timeout for: +# +# 1) Bulk transfer I/O during SYNC, from the point of view of slave. +# 2) Master timeout from the point of view of slaves (data, pings). +# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings). # # It is important to make sure that this value is greater than the value # specified for repl-ping-slave-period otherwise a timeout will be detected From 2d497014ea0aef649b79e267b3aa32c58ace97bc Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 May 2013 11:43:44 +0200 Subject: [PATCH 0633/1752] Version bumped to 2.7.2. --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index 4844e7017fa..ddede3772b0 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.1" +#define REDIS_VERSION "2.7.2" From 1b87b3ef3873093f0a781ef087fd8a546460fc24 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 27 May 2013 19:33:03 +0200 Subject: [PATCH 0634/1752] A comment about BLPOP timeout did not reflected actual behavior. --- src/redis.h | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis.h b/src/redis.h index ba09516e37c..1d19ba6a5dd 100644 --- a/src/redis.h +++ b/src/redis.h @@ -394,13 +394,15 @@ typedef struct multiCmd { typedef struct multiState { multiCmd *commands; /* Array of MULTI commands */ int count; /* Total number of MULTI commands */ + int minreplicas; /* MINREPLICAS for synchronous replication */ + time_t minreplicas_timeout; /* MINREPLICAS timeout as unixtime. */ } multiState; typedef struct blockingState { dict *keys; /* The keys we are waiting to terminate a blocking * operation such as BLPOP. Otherwise NULL. */ time_t timeout; /* Blocking operation timeout. If UNIX current time - * is >= timeout then the operation timed out. */ + * is > timeout then the operation timed out. */ robj *target; /* The key that should receive the element, * for BRPOPLPUSH. */ } blockingState; From 889d39b5baa36d7509c2162813e022716a1a1b07 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 May 2013 15:23:42 +0200 Subject: [PATCH 0635/1752] Accept REPLCONF in any state. --- src/redis.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 3bf05065e4a..0da88fb579a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -221,7 +221,7 @@ struct redisCommand redisCommandTable[] = { {"discard",discardCommand,1,"rs",0,NULL,0,0,0,0,0}, {"sync",syncCommand,1,"ars",0,NULL,0,0,0,0,0}, {"psync",syncCommand,3,"ars",0,NULL,0,0,0,0,0}, - {"replconf",replconfCommand,-1,"ars",0,NULL,0,0,0,0,0}, + {"replconf",replconfCommand,-1,"arslt",0,NULL,0,0,0,0,0}, {"flushdb",flushdbCommand,1,"w",0,NULL,0,0,0,0,0}, {"flushall",flushallCommand,1,"w",0,NULL,0,0,0,0,0}, {"sort",sortCommand,-2,"wm",0,NULL,1,1,1,0,0}, @@ -1772,9 +1772,10 @@ int processCommand(redisClient *c) { return REDIS_OK; } - /* Lua script too slow? Only allow commands with REDIS_CMD_STALE flag. */ + /* Lua script too slow? Only allow a limited number of commands. */ if (server.lua_timedout && c->cmd->proc != authCommand && + c->cmd->proc != replconfCommand && !(c->cmd->proc == shutdownCommand && c->argc == 2 && tolower(((char*)c->argv[1]->ptr)[0]) == 'n') && From c64338aa8f1360a6550c9297338dd6db937accb5 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 May 2013 19:52:54 +0200 Subject: [PATCH 0636/1752] Slaves list in INFO output: lag added, format changed. There is a new 'lag' information in the list of slaves, in the "replication" section of the INFO output. Also the format was changed in a backward incompatible way in order to make it more easy to parse if new fields are added in the future, as the new format is comma separated but has named fields (no longer positional fields). --- src/redis.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0da88fb579a..1fe694b556f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2250,6 +2250,7 @@ sds genRedisInfoString(char *section) { char *state = NULL; char ip[32]; int port; + long lag = 0; if (anetPeerToString(slave->fd,ip,&port) == -1) continue; switch(slave->replstate) { @@ -2265,9 +2266,14 @@ sds genRedisInfoString(char *section) { break; } if (state == NULL) continue; - info = sdscatprintf(info,"slave%d:%s,%d,%s,%lld\r\n", + if (slave->replstate == REDIS_REPL_ONLINE) + lag = time(NULL) - slave->repl_ack_time; + + info = sdscatprintf(info, + "slave%d:ip=%s,port=%d,state=%s," + "repl_offset=%lld,lag=%ld\r\n", slaveid,ip,slave->slave_listening_port,state, - slave->repl_ack_off); + slave->repl_ack_off, lag); slaveid++; } } From 75fcf5c936d198035f4edf2a31d2bb6c4dec07bf Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 May 2013 19:56:33 +0200 Subject: [PATCH 0637/1752] repl_offset field in INFO replication is now just offset. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 1fe694b556f..f6e680aa0a9 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2271,7 +2271,7 @@ sds genRedisInfoString(char *section) { info = sdscatprintf(info, "slave%d:ip=%s,port=%d,state=%s," - "repl_offset=%lld,lag=%ld\r\n", + "offset=%lld,lag=%ld\r\n", slaveid,ip,slave->slave_listening_port,state, slave->repl_ack_off, lag); slaveid++; From fc2587ec6ed9bac669723bc8243c4ad288f86eab Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 28 May 2013 16:48:21 +0200 Subject: [PATCH 0638/1752] min-slaves-to-write: initial description of the feature in redis.conf --- redis.conf | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/redis.conf b/redis.conf index 6e728eac0a5..8e7575a8ad6 100644 --- a/redis.conf +++ b/redis.conf @@ -267,6 +267,22 @@ repl-disable-tcp-nodelay no # By default the priority is 100. slave-priority 100 +# It is possible for a master to stop accepting writes if there are less than +# N slaves connected, having a lag less or equal than M seconds. +# +# The N slaves need to be in "online" state. +# +# The lag in seconds, that must be <= the specified value, is calculated from +# the last ping received from the slave, that is usually sent every second. +# +# This option does not GUARANTEES that N replicas will accept the write, but +# will limit the window of exposure for lost writes in case not enough slaves +# are available, to the specified number of seconds. +# +# For example to require at least 3 slaves with a lag <= 10 seconds use: +# +# repl-min-slaves-to-write 3 10 + ################################## SECURITY ################################### # Require clients to issue AUTH before processing any other From d0d67f8d426649e1ba0b27db1e51b6a971dd7eaf Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 May 2013 11:36:44 +0200 Subject: [PATCH 0639/1752] min-slaves-to-write: don't accept writes with less than N replicas. This feature allows the user to specify the minimum number of connected replicas having a lag less or equal than the specified amount of seconds for writes to be accepted. --- redis.conf | 8 +++++++- src/config.c | 20 ++++++++++++++++++++ src/networking.c | 1 + src/redis.c | 15 +++++++++++++++ src/redis.h | 8 +++++++- src/replication.c | 27 +++++++++++++++++++++++++++ 6 files changed, 77 insertions(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 8e7575a8ad6..83fc76add99 100644 --- a/redis.conf +++ b/redis.conf @@ -281,7 +281,13 @@ slave-priority 100 # # For example to require at least 3 slaves with a lag <= 10 seconds use: # -# repl-min-slaves-to-write 3 10 +# min-slaves-to-write 3 +# min-slaves-max-lag 10 +# +# Setting one or the other to 0 disables the feature. +# +# By default min-slaves-to-write is set to 0 (feature disabled) and +# min-slaves-max-lag is set to 10. ################################## SECURITY ################################### diff --git a/src/config.c b/src/config.c index b69f2e133d7..c9ee19bb08e 100644 --- a/src/config.c +++ b/src/config.c @@ -428,6 +428,16 @@ void loadServerConfigFromString(char *config) { } } else if (!strcasecmp(argv[0],"slave-priority") && argc == 2) { server.slave_priority = atoi(argv[1]); + } else if (!strcasecmp(argv[0],"min-slaves-to-write") && argc == 2) { + server.repl_min_slaves_to_write = atoi(argv[1]); + if (server.repl_min_slaves_to_write < 0) { + err = "Invalid value for min-slaves-to-write."; goto loaderr; + } + } else if (!strcasecmp(argv[0],"min-slaves-max-lag") && argc == 2) { + server.repl_min_slaves_max_lag = atoi(argv[1]); + if (server.repl_min_slaves_max_lag < 0) { + err = "Invalid value for min-slaves-max-lag."; goto loaderr; + } } else if (!strcasecmp(argv[0],"notify-keyspace-events") && argc == 2) { int flags = keyspaceEventsStringToFlags(argv[1]); @@ -783,6 +793,14 @@ void configSetCommand(redisClient *c) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll <= 0) goto badfmt; server.slave_priority = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"min-slaves-to-write")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.repl_min_slaves_to_write = ll; + } else if (!strcasecmp(c->argv[2]->ptr,"min-slaves-max-lag")) { + if (getLongLongFromObject(o,&ll) == REDIS_ERR || + ll < 0) goto badfmt; + server.repl_min_slaves_max_lag = ll; } else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); @@ -880,6 +898,8 @@ void configGetCommand(redisClient *c) { config_get_numerical_field("maxclients",server.maxclients); config_get_numerical_field("watchdog-period",server.watchdog_period); config_get_numerical_field("slave-priority",server.slave_priority); + config_get_numerical_field("min-slaves-to-write",server.repl_min_slaves_to_write); + config_get_numerical_field("min-slaves-max-lag",server.repl_min_slaves_max_lag); config_get_numerical_field("hz",server.hz); /* Bool (yes/no) values */ diff --git a/src/networking.c b/src/networking.c index b1ee2df84cd..3981f48ef28 100644 --- a/src/networking.c +++ b/src/networking.c @@ -692,6 +692,7 @@ void freeClient(redisClient *c) { * backlog. */ if (c->flags & REDIS_SLAVE && listLength(server.slaves) == 0) server.repl_no_slaves_since = server.unixtime; + refreshGoodSlavesCount(); } /* Case 2: we lost the connection with the master. */ diff --git a/src/redis.c b/src/redis.c index f6e680aa0a9..152235e537b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1125,6 +1125,8 @@ void createSharedObjects(void) { "-OOM command not allowed when used memory > 'maxmemory'.\r\n")); shared.execaborterr = createObject(REDIS_STRING,sdsnew( "-EXECABORT Transaction discarded because of previous errors.\r\n")); + shared.noreplicaserr = createObject(REDIS_STRING,sdsnew( + "-NOREPLICAS Not enough good slaves to write.\r\n")); shared.space = createObject(REDIS_STRING,sdsnew(" ")); shared.colon = createObject(REDIS_STRING,sdsnew(":")); shared.plus = createObject(REDIS_STRING,sdsnew("+")); @@ -1228,6 +1230,8 @@ void initServerConfig() { server.shutdown_asap = 0; server.repl_ping_slave_period = REDIS_REPL_PING_SLAVE_PERIOD; server.repl_timeout = REDIS_REPL_TIMEOUT; + server.repl_min_slaves_to_write = REDIS_DEFAULT_MIN_SLAVES_TO_WRITE; + server.repl_min_slaves_max_lag = REDIS_DEFAULT_MIN_SLAVES_MAX_LAG; server.lua_caller = NULL; server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; @@ -1429,6 +1433,7 @@ void initServer() { server.ops_sec_last_sample_ops = 0; server.unixtime = time(NULL); server.lastbgsave_status = REDIS_OK; + server.repl_good_slaves_count = 0; if(aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR) { redisPanic("Can't create the serverCron time event."); exit(1); @@ -1733,6 +1738,16 @@ int processCommand(redisClient *c) { return REDIS_OK; } + /* Don't accept write commands if there are not enough good slaves and + * used configured the min-slaves-to-write option. */ + if (server.repl_min_slaves_to_write && server.repl_min_slaves_max_lag && + server.repl_good_slaves_count < server.repl_min_slaves_to_write) + { + flagTransaction(c); + addReply(c, shared.noreplicaserr); + return REDIS_OK; + } + /* Don't accept write commands if this is a read only slave. But * accept write commands if this is our master. */ if (server.masterhost && server.repl_slave_ro && diff --git a/src/redis.h b/src/redis.h index 1d19ba6a5dd..935cf116ca6 100644 --- a/src/redis.h +++ b/src/redis.h @@ -118,6 +118,8 @@ #define REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE 0 #define REDIS_DEFAULT_ACTIVE_REHASHING 1 #define REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 +#define REDIS_DEFAULT_MIN_SLAVES_TO_WRITE 0 +#define REDIS_DEFAULT_MIN_SLAVES_MAX_LAG 10 /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ @@ -479,7 +481,7 @@ struct sharedObjectsStruct { *colon, *nullbulk, *nullmultibulk, *queued, *emptymultibulk, *wrongtypeerr, *nokeyerr, *syntaxerr, *sameobjecterr, *outofrangeerr, *noscripterr, *loadingerr, *slowscripterr, *bgsaveerr, - *masterdownerr, *roslaveerr, *execaborterr, *noautherr, + *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, *lpush, @@ -676,6 +678,9 @@ struct redisServer { gets released. */ time_t repl_no_slaves_since; /* We have no slaves since that time. Only valid if server.slaves len is 0. */ + int repl_min_slaves_to_write; /* Min number of slaves to write. */ + int repl_min_slaves_max_lag; /* Max lag of slaves to write. */ + int repl_good_slaves_count; /* Number of slaves with lag <= max_lag. */ /* Replication (slave) */ char *masterauth; /* AUTH with this password with master */ char *masterhost; /* Hostname of master */ @@ -986,6 +991,7 @@ void replicationCron(void); void replicationHandleMasterDisconnection(void); void replicationCacheMaster(redisClient *c); void resizeReplicationBacklog(long long newsize); +void refreshGoodSlavesCount(void); /* Generic persistence functions */ void startLoading(FILE *fp); diff --git a/src/replication.c b/src/replication.c index a028359abbf..1213df12237 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1386,6 +1386,30 @@ void replicationResurrectCachedMaster(int newfd) { } } +/* ------------------------- MIN-SLAVES-TO-WRITE --------------------------- */ + +/* This function counts the number of slaves with lag <= min-slaves-max-lag. + * If the option is active, the server will prevent writes if there are not + * enough connected slaves with the specified lag (or less). */ +void refreshGoodSlavesCount(void) { + listIter li; + listNode *ln; + int good = 0; + + if (!server.repl_min_slaves_to_write || + !server.repl_min_slaves_max_lag) return; + + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + redisClient *slave = ln->value; + time_t lag = server.unixtime - slave->repl_ack_time; + + if (slave->replstate == REDIS_REPL_ONLINE && + lag <= server.repl_min_slaves_max_lag) good++; + } + server.repl_good_slaves_count = good; +} + /* --------------------------- REPLICATION CRON ---------------------------- */ /* Replication cron funciton, called 1 time per second. */ @@ -1499,4 +1523,7 @@ void replicationCron(void) { (int) server.repl_backlog_time_limit); } } + + /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ + refreshGoodSlavesCount(); } From 9d1bc3efcf05a2ad93bb218c49a09e7fde813efc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 29 May 2013 11:45:40 +0200 Subject: [PATCH 0640/1752] min-replicas-to-write: only deny write commands. I guess I needed another coffee... --- src/redis.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 152235e537b..2e8226a5a91 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1740,7 +1740,9 @@ int processCommand(redisClient *c) { /* Don't accept write commands if there are not enough good slaves and * used configured the min-slaves-to-write option. */ - if (server.repl_min_slaves_to_write && server.repl_min_slaves_max_lag && + if (server.repl_min_slaves_to_write && + server.repl_min_slaves_max_lag && + c->cmd->flags & REDIS_CMD_WRITE && server.repl_good_slaves_count < server.repl_min_slaves_to_write) { flagTransaction(c); From 4333e6ce201a137da667133434cdbb4db5dcbfd0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 May 2013 11:43:43 +0200 Subject: [PATCH 0641/1752] Make tests compatible with new INFO replication output. --- tests/integration/replication.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/replication.tcl b/tests/integration/replication.tcl index 5ca6449f4fd..5a320e785a1 100644 --- a/tests/integration/replication.tcl +++ b/tests/integration/replication.tcl @@ -112,7 +112,7 @@ start_server {tags {"repl"}} { set retry 500 while {$retry} { set info [r -3 info] - if {[string match {*slave0:*,online*slave1:*,online*slave2:*,online*} $info]} { + if {[string match {*slave0:*state=online*slave1:*state=online*slave2:*state=online*} $info]} { break } else { incr retry -1 From 9a7ea3dae9d3249ceea9204a62775ae3d428f42b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 May 2013 18:54:28 +0200 Subject: [PATCH 0642/1752] Tests added for min-slaves feature. --- tests/integration/replication-4.tcl | 42 +++++++++++++++++++++++++++++ tests/support/redis.tcl | 4 +++ 2 files changed, 46 insertions(+) diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 29ab48fc269..7a3b0822fb1 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -54,3 +54,45 @@ start_server {tags {"repl"}} { } } } + +start_server {tags {"repl"}} { + start_server {} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + test {First server should have role slave after SLAVEOF} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [s 0 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + test {With min-slaves-to-write (1,3): master should be writable} { + $master config set min-slaves-max-lag 3 + $master config set min-slaves-to-write 1 + $master set foo bar + } {OK} + + test {With min-slaves-to-write (2,3): master should not be writable} { + $master config set min-slaves-max-lag 3 + $master config set min-slaves-to-write 2 + catch {$master set foo bar} e + set e + } {NOREPLICAS*} + + test {With min-slaves-to-write: master not writable with lagged slave} { + $master config set min-slaves-max-lag 2 + $master config set min-slaves-to-write 1 + assert {[$master set foo bar] eq {OK}} + $slave deferred 1 + $slave debug sleep 4 + after 3000 + catch {$master set foo bar} e + set e + } {NOREPLICAS*} + } +} diff --git a/tests/support/redis.tcl b/tests/support/redis.tcl index 99415b6409e..36b005a1775 100644 --- a/tests/support/redis.tcl +++ b/tests/support/redis.tcl @@ -115,6 +115,10 @@ proc ::redis::__method__channel {id fd} { return $fd } +proc ::redis::__method__deferred {id fd val} { + set ::redis::deferred($id) $val +} + proc ::redis::redis_write {fd buf} { puts -nonewline $fd $buf } From f5275da6e95ed91195b6b23c8dbd9b5186063d91 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 May 2013 12:13:25 +0200 Subject: [PATCH 0643/1752] Refresh good slaves count when setting slave state as online. --- src/replication.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/replication.c b/src/replication.c index 1213df12237..02452505e7d 100644 --- a/src/replication.c +++ b/src/replication.c @@ -441,6 +441,7 @@ int masterTryPartialResynchronization(redisClient *c) { * to -1 to force the master to emit SELECT, since the slave already * has this state from the previous connection with the master. */ + refreshGoodSlavesCount(); return REDIS_OK; /* The caller can return, no full resync needed. */ need_full_resync: @@ -662,6 +663,7 @@ void sendBulkToSlave(aeEventLoop *el, int fd, void *privdata, int mask) { freeClient(slave); return; } + refreshGoodSlavesCount(); redisLog(REDIS_NOTICE,"Synchronization with slave succeeded"); } } From 971217c7b404307de448a918fff2526a193d14aa Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 May 2013 12:18:31 +0200 Subject: [PATCH 0644/1752] New INFO field "min_slaves_good_slaves". When min-slaves-to-write feature is active, this field reports the number of slaves considered good (online state, lag within the specified range). --- src/redis.c | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/redis.c b/src/redis.c index 2e8226a5a91..b397b90db81 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2253,9 +2253,20 @@ sds genRedisInfoString(char *section) { server.slave_priority, server.repl_slave_ro); } + info = sdscatprintf(info, "connected_slaves:%lu\r\n", listLength(server.slaves)); + + /* If min-slaves-to-write is active, write the number of slaves + * currently considered 'good'. */ + if (server.repl_min_slaves_to_write && + server.repl_min_slaves_max_lag) { + info = sdscatprintf(info, + "min_slaves_good_slaves:%d\r\n", + server.repl_good_slaves_count); + } + if (listLength(server.slaves)) { int slaveid = 0; listNode *ln; From fe4b62fcaf034ac9f407321721e1109d9cc4fcb2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 30 May 2013 12:23:28 +0200 Subject: [PATCH 0645/1752] Refresh good slaves count after CONFIG SET min-slaves-... This way just after the CONFIG SET enabling the min-slaves feature it is possible to write to the database without delays. --- src/config.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/config.c b/src/config.c index c9ee19bb08e..242f7367f3d 100644 --- a/src/config.c +++ b/src/config.c @@ -797,10 +797,12 @@ void configSetCommand(redisClient *c) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.repl_min_slaves_to_write = ll; + refreshGoodSlavesCount(); } else if (!strcasecmp(c->argv[2]->ptr,"min-slaves-max-lag")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; server.repl_min_slaves_max_lag = ll; + refreshGoodSlavesCount(); } else { addReplyErrorFormat(c,"Unsupported CONFIG parameter: %s", (char*)c->argv[2]->ptr); From d34c623e100cb32a945b7848bb8dda9fac64ca49 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 31 May 2013 11:43:23 +0200 Subject: [PATCH 0646/1752] Test: avoid a false positive in min-slaves test. --- tests/integration/replication-4.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index 7a3b0822fb1..f84369f44da 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -89,8 +89,8 @@ start_server {tags {"repl"}} { $master config set min-slaves-to-write 1 assert {[$master set foo bar] eq {OK}} $slave deferred 1 - $slave debug sleep 4 - after 3000 + $slave debug sleep 6 + after 4000 catch {$master set foo bar} e set e } {NOREPLICAS*} From 054632f551384e70192035f515082ab2514627a3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 31 May 2013 19:31:36 +0200 Subject: [PATCH 0647/1752] CONFIG SET: accept slave-priority zero, it is valid. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 242f7367f3d..5595e924ab8 100644 --- a/src/config.c +++ b/src/config.c @@ -791,7 +791,7 @@ void configSetCommand(redisClient *c) { server.repl_disable_tcp_nodelay = yn; } else if (!strcasecmp(c->argv[2]->ptr,"slave-priority")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || - ll <= 0) goto badfmt; + ll < 0) goto badfmt; server.slave_priority = ll; } else if (!strcasecmp(c->argv[2]->ptr,"min-slaves-to-write")) { if (getLongLongFromObject(o,&ll) == REDIS_ERR || From b435817b17f9cfe63742cffbc4454fa616be4f73 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 4 Jun 2013 15:53:53 +0200 Subject: [PATCH 0648/1752] Binary safe dump of object content in redisLogObjectDebugInfo(). --- src/debug.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/debug.c b/src/debug.c index 2f62bedb055..eb08c38d0d3 100644 --- a/src/debug.c +++ b/src/debug.c @@ -390,8 +390,11 @@ void redisLogObjectDebugInfo(robj *o) { redisLog(REDIS_WARNING,"Object refcount: %d", o->refcount); if (o->type == REDIS_STRING && o->encoding == REDIS_ENCODING_RAW) { redisLog(REDIS_WARNING,"Object raw string len: %zu", sdslen(o->ptr)); - if (sdslen(o->ptr) < 4096) - redisLog(REDIS_WARNING,"Object raw string content: \"%s\"", (char*)o->ptr); + if (sdslen(o->ptr) < 4096) { + sds repr = sdscatrepr(sdsempty(),o->ptr,sdslen(o->ptr)); + redisLog(REDIS_WARNING,"Object raw string content: %s", repr); + sdsfree(repr); + } } else if (o->type == REDIS_LIST) { redisLog(REDIS_WARNING,"List length: %d", (int) listTypeLength(o)); } else if (o->type == REDIS_SET) { From e34f00842ff7c3e4bbca89908a7dad8c3b13eb75 Mon Sep 17 00:00:00 2001 From: ioddly Date: Wed, 22 May 2013 18:17:58 -0500 Subject: [PATCH 0649/1752] Try to report source of bad Lua API calls --- src/scripting.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 6661f37485d..2ffd92ef768 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -152,9 +152,20 @@ char *redisProtocolToLuaType_MultiBulk(lua_State *lua, char *reply) { } void luaPushError(lua_State *lua, char *error) { + lua_Debug dbg; + lua_newtable(lua); lua_pushstring(lua,"err"); - lua_pushstring(lua, error); + + /* Attempt to figure out where this function was called, if possible */ + if(lua_getstack(lua, 1, &dbg) && lua_getinfo(lua, "nSl", &dbg)) { + sds msg = sdscatprintf(sdsempty(), "%s: %d: %s", + dbg.source, dbg.currentline, error); + lua_pushstring(lua, msg); + sdsfree(msg); + } else { + lua_pushstring(lua, error); + } lua_settable(lua,-3); } @@ -866,9 +877,10 @@ void evalGenericCommand(redisClient *c, int evalsha) { delhook = 1; } - /* At this point whatever this script was never seen before or if it was + /* At this point whether this script was never seen before or if it was * already defined, we can call it. We have zero arguments and expect * a single return value. */ + err = lua_pcall(lua,0,1,0); /* Perform some cleanup that we need to do both on error and success. */ From 1cf839a216c53a9192c6d21a90914f073594efc3 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Perennou Date: Mon, 10 Dec 2012 18:21:10 +0100 Subject: [PATCH 0650/1752] test-server: only listen to 127.0.0.1 Signed-off-by: Marc-Antoine Perennou --- tests/assets/default.conf | 1 + tests/test_helper.tcl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/assets/default.conf b/tests/assets/default.conf index 17d21b07c31..902a094a57f 100644 --- a/tests/assets/default.conf +++ b/tests/assets/default.conf @@ -5,6 +5,7 @@ daemonize no pidfile /var/run/redis.pid port 6379 timeout 0 +bind 127.0.0.1 loglevel verbose logfile '' databases 16 diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 8d34a078307..cc1a801df57 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -186,7 +186,7 @@ proc test_server_main {} { if {!$::quiet} { puts "Starting test server at port $port" } - socket -server accept_test_clients $port + socket -server accept_test_clients -myaddr 127.0.0.1 $port # Start the client instances set ::clients_pids {} From b3fbc84cc1987b43c80e66609371d3aec445a3ef Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 18 Jun 2013 17:33:35 +0200 Subject: [PATCH 0651/1752] Lua scripting: improve error reporting. When calling Lua scripts we try to report not just the error but information about the code line causing the error. --- src/scripting.c | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index 2ffd92ef768..06539a4cbcf 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -620,6 +620,26 @@ void scriptingInit(void) { lua_pcall(lua,0,0,0); } + /* Add a helper function we use for pcall error reporting. + * Note that when the error is in the C function we want to report the + * information about the caller, that's what makes sense from the point + * of view of the user debugging a script. */ + { + char *errh_func = "function __redis__err__handler(err)\n" + " local i = debug.getinfo(2,'nSl')\n" + " if i and i.what == 'C' then\n" + " i = debug.getinfo(3,'nSl')\n" + " end\n" + " if i then\n" + " return err ..': '.. i.source .. ': ' .. i.currentline\n" + " else\n" + " return err\n" + " end\n" + "end\n"; + luaL_loadbuffer(lua,errh_func,strlen(errh_func),"@err_handler_def"); + lua_pcall(lua,0,0,0); + } + /* Create the (non connected) client that we use to execute Redis commands * inside the Lua interpreter. * Note: there is no need to create it again when this function is called @@ -840,21 +860,25 @@ void evalGenericCommand(redisClient *c, int evalsha) { funcname[42] = '\0'; } + /* Push the pcall error handler function on the stack. */ + lua_getglobal(lua, "__redis__err__handler"); + /* Try to lookup the Lua function */ lua_getglobal(lua, funcname); - if (lua_isnil(lua,1)) { + if (lua_isnil(lua,-1)) { lua_pop(lua,1); /* remove the nil from the stack */ /* Function not defined... let's define it if we have the * body of the function. If this is an EVALSHA call we can just * return an error. */ if (evalsha) { + lua_pop(lua,1); /* remove the error handler from the stack. */ addReply(c, shared.noscripterr); return; } if (luaCreateFunction(c,lua,funcname,c->argv[1]) == REDIS_ERR) return; /* Now the following is guaranteed to return non nil */ lua_getglobal(lua, funcname); - redisAssert(!lua_isnil(lua,1)); + redisAssert(!lua_isnil(lua,-1)); } /* Populate the argv and keys table accordingly to the arguments that @@ -881,7 +905,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { * already defined, we can call it. We have zero arguments and expect * a single return value. */ - err = lua_pcall(lua,0,1,0); + err = lua_pcall(lua,0,1,-2); /* Perform some cleanup that we need to do both on error and success. */ if (delhook) lua_sethook(lua,luaMaskCountHook,0,0); /* Disable hook */ From bf8e078aa2e06211dd271bd049e879c8bdb5ef45 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 18 Jun 2013 19:30:56 +0200 Subject: [PATCH 0652/1752] Lua script errors format more unified. lua_pcall error handler now formats errors in a way more similar to luaPushError() so that errors generated in different contexts look alike. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 06539a4cbcf..a707f1bac20 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -631,7 +631,7 @@ void scriptingInit(void) { " i = debug.getinfo(3,'nSl')\n" " end\n" " if i then\n" - " return err ..': '.. i.source .. ': ' .. i.currentline\n" + " return i.source .. ':' .. i.currentline .. ': ' .. err\n" " else\n" " return err\n" " end\n" From 449ec38fe46ab8a3ba8a49e6361f192fdb93aac6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 19 Jun 2013 14:44:40 +0200 Subject: [PATCH 0653/1752] Fix logStackTrace() when logging to stdout. When the semantics changed from logfile = NULL to logfile = "" to log into standard output, no proper change was made to logStackTrace() to make it able to work with the new setup. This commit fixes the issue. --- src/debug.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/debug.c b/src/debug.c index eb08c38d0d3..32d36120053 100644 --- a/src/debug.c +++ b/src/debug.c @@ -619,11 +619,12 @@ void logRegisters(ucontext_t *uc) { void logStackTrace(ucontext_t *uc) { void *trace[100]; int trace_size = 0, fd; + int log_to_stdout = server.logfile[0] == '\0'; /* Open the log file in append mode. */ - fd = server.logfile ? - open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644) : - STDOUT_FILENO; + fd = log_to_stdout ? + STDOUT_FILENO : + open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644); if (fd == -1) return; /* Generate the stack trace */ @@ -637,7 +638,7 @@ void logStackTrace(ucontext_t *uc) { backtrace_symbols_fd(trace, trace_size, fd); /* Cleanup */ - if (server.logfile) close(fd); + if (!log_to_stdout) close(fd); } /* Log information about the "current" client, that is, the client that is From 86147afd083e7776fc1f2f33f6ace49b4f9378ae Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 19 Jun 2013 18:25:03 +0200 Subject: [PATCH 0654/1752] Allow writes from scripts called by AOF loading in read-only slaves. This fixes issue #1163 --- src/scripting.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripting.c b/src/scripting.c index a707f1bac20..b94627c7f62 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -269,6 +269,7 @@ int luaRedisGenericCommand(lua_State *lua, int raise_error) { "Write commands not allowed after non deterministic commands"); goto cleanup; } else if (server.masterhost && server.repl_slave_ro && + !server.loading && !(server.lua_caller->flags & REDIS_MASTER)) { luaPushError(lua, shared.roslaveerr->ptr); From be8d5fd9549ae6894baf0e8400e9af30032a7184 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 19 Jun 2013 18:53:07 +0200 Subject: [PATCH 0655/1752] Test: regression test for #1163. --- tests/unit/scripting.tcl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index e42f87725d9..3e08f630c62 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -281,6 +281,23 @@ start_server {tags {"scripting"}} { assert_equal $rand1 $rand2 assert {$rand2 ne $rand3} } + + test {EVAL processes writes from AOF in read-only slaves} { + r flushall + r config set appendonly yes + r eval {redis.call("set","foo","100")} 0 + r eval {redis.call("incr","foo")} 0 + r eval {redis.call("incr","foo")} 0 + wait_for_condition 50 100 { + [s aof_rewrite_in_progress] == 0 + } else { + fail "AOF rewrite can't complete after CONFIG SET appendonly yes." + } + r config set slave-read-only yes + r slaveof 127.0.0.1 0 + r debug loadaof + r get foo + } {102} } # Start a new server since the last test in this stanza will kill the From 9af8125c7d2a008b7c3e4ea027b7bf0ffb886b1a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Jun 2013 10:21:38 +0200 Subject: [PATCH 0656/1752] Sentinel: parse new INFO replication output correctly. Sentinel was not able to detect slaves when connected to a very recent version of Redis master since a previos non-backward compatible change to INFO broken the parsing of the slaves ip:port INFO output. This fixes issue #1164 --- src/sentinel.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index a4db9408eca..ed0978694c9 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1397,20 +1397,33 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { } } - /* slave0:,, */ + /* old versions: slave0:,, + * new versions: slave0:ip=127.0.0.1,port=9999,... */ if ((ri->flags & SRI_MASTER) && sdslen(l) >= 7 && !memcmp(l,"slave",5) && isdigit(l[5])) { char *ip, *port, *end; - ip = strchr(l,':'); if (!ip) continue; - ip++; /* Now ip points to start of ip address. */ - port = strchr(ip,','); if (!port) continue; - *port = '\0'; /* nul term for easy access. */ - port++; /* Now port points to start of port number. */ - end = strchr(port,','); if (!end) continue; - *end = '\0'; /* nul term for easy access. */ + if (strstr(l,"ip=") == NULL) { + /* Old format. */ + ip = strchr(l,':'); if (!ip) continue; + ip++; /* Now ip points to start of ip address. */ + port = strchr(ip,','); if (!port) continue; + *port = '\0'; /* nul term for easy access. */ + port++; /* Now port points to start of port number. */ + end = strchr(port,','); if (!end) continue; + *end = '\0'; /* nul term for easy access. */ + } else { + /* New format. */ + ip = strstr(l,"ip="); if (!ip) continue; + ip += 3; /* Now ip points to start of ip address. */ + port = strstr(l,"port="); if (!port) continue; + port += 5; /* Now port points to start of port number. */ + /* Nul term both fields for easy access. */ + end = strchr(ip,','); if (end) *end = '\0'; + end = strchr(port,','); if (end) *end = '\0'; + } /* Check if we already have this slave into our table, * otherwise add it. */ From 90c3a14ac04a4bfd3b76729262bd835e6daaa807 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 20 Jun 2013 17:53:35 +0300 Subject: [PATCH 0657/1752] Initialize char* to NULL to remove compiler warning --- src/config.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/config.c b/src/config.c index 5595e924ab8..6e0d4a87eaf 100644 --- a/src/config.c +++ b/src/config.c @@ -1309,7 +1309,7 @@ void rewriteConfigOctalOption(struct rewriteConfigState *state, char *option, in * specified. See how the function is used for more information. */ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int value, ...) { va_list ap; - char *enum_name, *matching_name; + char *enum_name, *matching_name = NULL; int enum_val, def_val, force; sds line; @@ -1334,7 +1334,7 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int void rewriteConfigSyslogfacilityOption(struct rewriteConfigState *state) { int value = server.syslog_facility, j; int force = value != LOG_LOCAL0; - char *name, *option = "syslog-facility"; + char *name = NULL, *option = "syslog-facility"; sds line; for (j = 0; validSyslogFacilities[j].name; j++) { From 9a7b09ed3567e72c6dd61ca8832426fa10f1e00c Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 19 Jul 2012 21:25:30 +0900 Subject: [PATCH 0658/1752] don't define _XOPEN_SOURCE for NetBSD on NetBSD, defining _XOPEN_SOURCE hides extensions like inet_aton, strcasecmp, etc. --- src/fmacros.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/fmacros.h b/src/fmacros.h index a6cf3578c13..c16f5e204ac 100644 --- a/src/fmacros.h +++ b/src/fmacros.h @@ -36,9 +36,13 @@ #define _GNU_SOURCE #endif -#if defined(__linux__) || defined(__OpenBSD__) || defined(__NetBSD__) +#if defined(__linux__) || defined(__OpenBSD__) #define _XOPEN_SOURCE 700 -#else +/* + * On NetBSD, _XOPEN_SOURCE undefines _NETBSD_SOURCE and + * thus hides inet_aton etc. + */ +#elif !defined(__NetBSD__) #define _XOPEN_SOURCE #endif From f27896a17d7730cfc202b44cd8a00e5c77bd8707 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 19 Jul 2012 21:28:11 +0900 Subject: [PATCH 0659/1752] rename popcount to popcount_binary to avoid a conflict with NetBSD libc NetBSD-current's libc has a function named popcount. hiding these extensions using feature macros is not possible because redis uses other extensions covered by the same feature macro. eg. inet_aton --- src/bitops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 47f768c31d5..1c2e13ddcf4 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -58,7 +58,7 @@ static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to * work with a input string length up to 512 MB. */ -size_t popcount(void *s, long count) { +size_t popcount_binary(void *s, long count) { size_t bits = 0; unsigned char *p; uint32_t *p4 = s; @@ -407,6 +407,6 @@ void bitcountCommand(redisClient *c) { } else { long bytes = end-start+1; - addReplyLongLong(c,popcount(p+start,bytes)); + addReplyLongLong(c,popcount_binary(p+start,bytes)); } } From 30f3ae03a548156738cb12cfbd4e01801c5c5d96 Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Thu, 19 Jul 2012 21:37:34 +0900 Subject: [PATCH 0660/1752] use nanosleep instead of usleep SUSv3 says that: The useconds argument shall be less than one million. If the value of useconds is 0, then the call has no effect. and actually NetBSD's implementation rejects such a value with EINVAL. use nanosleep which has no such a limitation instead. --- src/debug.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index 32d36120053..0a947e56475 100644 --- a/src/debug.c +++ b/src/debug.c @@ -329,8 +329,11 @@ void debugCommand(redisClient *c) { } else if (!strcasecmp(c->argv[1]->ptr,"sleep") && c->argc == 3) { double dtime = strtod(c->argv[2]->ptr,NULL); long long utime = dtime*1000000; + struct timespec tv; - usleep(utime); + tv.tv_sec = utime / 1000000; + tv.tv_nsec = (utime % 1000000) * 1000; + nanosleep(&tv, NULL); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"set-active-expire") && c->argc == 3) From 3642557ae685cfb46e7ac9ab0a9b7a6fe2ff424e Mon Sep 17 00:00:00 2001 From: YAMAMOTO Takashi Date: Sun, 28 Oct 2012 13:33:04 +0900 Subject: [PATCH 0661/1752] don't assume time_t == long time_t is always 64bit on recent versions of NetBSD. --- src/config.c | 4 ++-- src/redis.c | 44 ++++++++++++++++++++++---------------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/src/config.c b/src/config.c index 6e0d4a87eaf..dad575965aa 100644 --- a/src/config.c +++ b/src/config.c @@ -973,8 +973,8 @@ void configGetCommand(redisClient *c) { int j; for (j = 0; j < server.saveparamslen; j++) { - buf = sdscatprintf(buf,"%ld %d", - server.saveparams[j].seconds, + buf = sdscatprintf(buf,"%jd %d", + (intmax_t)server.saveparams[j].seconds, server.saveparams[j].changes); if (j != server.saveparamslen-1) buf = sdscatlen(buf," ",1); diff --git a/src/redis.c b/src/redis.c index b397b90db81..034d7f3aeda 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2017,8 +2017,8 @@ sds genRedisInfoString(char *section) { "process_id:%ld\r\n" "run_id:%s\r\n" "tcp_port:%d\r\n" - "uptime_in_seconds:%ld\r\n" - "uptime_in_days:%ld\r\n" + "uptime_in_seconds:%jd\r\n" + "uptime_in_days:%jd\r\n" "hz:%d\r\n" "lru_clock:%ld\r\n" "config_file:%s\r\n", @@ -2038,8 +2038,8 @@ sds genRedisInfoString(char *section) { (long) getpid(), server.runid, server.port, - uptime, - uptime/(3600*24), + (intmax_t)uptime, + (intmax_t)(uptime/(3600*24)), server.hz, (unsigned long) server.lruclock, server.configfile ? server.configfile : ""); @@ -2096,30 +2096,30 @@ sds genRedisInfoString(char *section) { "loading:%d\r\n" "rdb_changes_since_last_save:%lld\r\n" "rdb_bgsave_in_progress:%d\r\n" - "rdb_last_save_time:%ld\r\n" + "rdb_last_save_time:%jd\r\n" "rdb_last_bgsave_status:%s\r\n" - "rdb_last_bgsave_time_sec:%ld\r\n" - "rdb_current_bgsave_time_sec:%ld\r\n" + "rdb_last_bgsave_time_sec:%jd\r\n" + "rdb_current_bgsave_time_sec:%jd\r\n" "aof_enabled:%d\r\n" "aof_rewrite_in_progress:%d\r\n" "aof_rewrite_scheduled:%d\r\n" - "aof_last_rewrite_time_sec:%ld\r\n" - "aof_current_rewrite_time_sec:%ld\r\n" + "aof_last_rewrite_time_sec:%jd\r\n" + "aof_current_rewrite_time_sec:%jd\r\n" "aof_last_bgrewrite_status:%s\r\n", server.loading, server.dirty, server.rdb_child_pid != -1, - server.lastsave, + (intmax_t)server.lastsave, (server.lastbgsave_status == REDIS_OK) ? "ok" : "err", - server.rdb_save_time_last, - (server.rdb_child_pid == -1) ? - -1 : time(NULL)-server.rdb_save_time_start, + (intmax_t)server.rdb_save_time_last, + (intmax_t)((server.rdb_child_pid == -1) ? + -1 : time(NULL)-server.rdb_save_time_start), server.aof_state != REDIS_AOF_OFF, server.aof_child_pid != -1, server.aof_rewrite_scheduled, - server.aof_rewrite_time_last, - (server.aof_child_pid == -1) ? - -1 : time(NULL)-server.aof_rewrite_time_start, + (intmax_t)server.aof_rewrite_time_last, + (intmax_t)((server.aof_child_pid == -1) ? + -1 : time(NULL)-server.aof_rewrite_time_start), (server.aof_lastbgrewrite_status == REDIS_OK) ? "ok" : "err"); if (server.aof_state != REDIS_AOF_OFF) { @@ -2158,16 +2158,16 @@ sds genRedisInfoString(char *section) { } info = sdscatprintf(info, - "loading_start_time:%ld\r\n" + "loading_start_time:%jd\r\n" "loading_total_bytes:%llu\r\n" "loading_loaded_bytes:%llu\r\n" "loading_loaded_perc:%.2f\r\n" - "loading_eta_seconds:%ld\r\n" - ,(unsigned long) server.loading_start_time, + "loading_eta_seconds:%jd\r\n", + (intmax_t) server.loading_start_time, (unsigned long long) server.loading_total_bytes, (unsigned long long) server.loading_loaded_bytes, perc, - eta + (intmax_t)eta ); } } @@ -2244,8 +2244,8 @@ sds genRedisInfoString(char *section) { if (server.repl_state != REDIS_REPL_CONNECTED) { info = sdscatprintf(info, - "master_link_down_since_seconds:%ld\r\n", - (long)server.unixtime-server.repl_down_since); + "master_link_down_since_seconds:%jd\r\n", + (intmax_t)server.unixtime-server.repl_down_since); } info = sdscatprintf(info, "slave_priority:%d\r\n" From 72d3fbea65b5a571a0e5fec9a35aff5e650c7bcc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 19 Jun 2013 18:31:33 +0200 Subject: [PATCH 0662/1752] Fix comment typo in integration/aof.tcl. --- tests/integration/aof.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/aof.tcl b/tests/integration/aof.tcl index ebf9cb56498..f255d7ec1f2 100644 --- a/tests/integration/aof.tcl +++ b/tests/integration/aof.tcl @@ -98,7 +98,7 @@ tags {"aof"} { } } - ## Test that SPOP (that modifies the client its argc/argv) is correctly free'd + ## Test that SPOP (that modifies the client's argc/argv) is correctly free'd create_aof { append_to_aof [formatCommand sadd set foo] append_to_aof [formatCommand sadd set bar] From a8f1474dfd699dbeac859c8e157af7af3e118fb0 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Jun 2013 15:32:00 +0200 Subject: [PATCH 0663/1752] PUBSUB command implemented. Currently it implements three subcommands: PUBSUB CHANNELS [] List channels with non-zero subscribers. PUBSUB NUMSUB [channel_1 ...] List number of subscribers for channels. PUBSUB NUMPAT Return number of subscribed patterns. --- src/pubsub.c | 47 +++++++++++++++++++++++++++++++++++++++++++++++ src/redis.c | 1 + src/redis.h | 1 + 3 files changed, 49 insertions(+) diff --git a/src/pubsub.c b/src/pubsub.c index 718f819ba27..ab63ca7e320 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -308,3 +308,50 @@ void publishCommand(redisClient *c) { int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]); addReplyLongLong(c,receivers); } + +/* PUBSUB command for Pub/Sub introspection. */ +void pubsubCommand(redisClient *c) { + if (!strcasecmp(c->argv[1]->ptr,"channels") && + (c->argc == 2 || c->argc ==3)) + { + /* PUBSUB CHANNELS [] */ + sds pat = (c->argc == 2) ? NULL : c->argv[2]->ptr; + dictIterator *di = dictGetIterator(server.pubsub_channels); + dictEntry *de; + long mblen = 0; + void *replylen; + + replylen = addDeferredMultiBulkLength(c); + while((de = dictNext(di)) != NULL) { + robj *cobj = dictGetKey(de); + sds channel = cobj->ptr; + + if (!pat || stringmatchlen(pat, sdslen(pat), + channel, sdslen(channel),0)) + { + addReplyBulk(c,cobj); + mblen++; + } + } + dictReleaseIterator(di); + setDeferredMultiBulkLength(c,replylen,mblen); + } else if (!strcasecmp(c->argv[1]->ptr,"numsub") && c->argc > 2) { + /* PUBSUB NUMSUB Channel_1 [... Channel_N] */ + int j; + + addReplyMultiBulkLen(c,(c->argc-2)*2); + for (j = 2; j < c->argc; j++) { + list *l = dictFetchValue(server.pubsub_channels,c->argv[j]); + + addReplyBulk(c,c->argv[j]); + addReplyBulkLongLong(c,l ? listLength(l) : 0); + } + } else if (!strcasecmp(c->argv[1]->ptr,"numpat") && c->argc == 2) { + /* PUBSUB NUMPAT */ + addReplyLongLong(c,listLength(server.pubsub_patterns)); + } else { + addReplyErrorFormat(c, + "Unknown PUBSUB subcommand or wrong number of arguments for '%s'", + (char*)c->argv[1]->ptr); + } +} diff --git a/src/redis.c b/src/redis.c index 034d7f3aeda..f34b2cba023 100644 --- a/src/redis.c +++ b/src/redis.c @@ -238,6 +238,7 @@ struct redisCommand redisCommandTable[] = { {"psubscribe",psubscribeCommand,-2,"rpslt",0,NULL,0,0,0,0,0}, {"punsubscribe",punsubscribeCommand,-1,"rpslt",0,NULL,0,0,0,0,0}, {"publish",publishCommand,3,"pfltr",0,NULL,0,0,0,0,0}, + {"pubsub",pubsubCommand,-2,"pltrR",0,NULL,0,0,0,0,0}, {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0}, {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0}, {"restore",restoreCommand,4,"awm",0,NULL,1,1,1,0,0}, diff --git a/src/redis.h b/src/redis.h index 935cf116ca6..dee82da6d71 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1277,6 +1277,7 @@ void unsubscribeCommand(redisClient *c); void psubscribeCommand(redisClient *c); void punsubscribeCommand(redisClient *c); void publishCommand(redisClient *c); +void pubsubCommand(redisClient *c); void watchCommand(redisClient *c); void unwatchCommand(redisClient *c); void restoreCommand(redisClient *c); From aeab473dfbc6d5ad67cc7fb9775395eb31c64a44 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 20 Jun 2013 15:34:56 +0200 Subject: [PATCH 0664/1752] Allow PUBSUB NUMSUB without channels. The result is an empty list but it is handy to call it programmatically. --- src/pubsub.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index ab63ca7e320..9fafbdcb0b7 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -335,8 +335,8 @@ void pubsubCommand(redisClient *c) { } dictReleaseIterator(di); setDeferredMultiBulkLength(c,replylen,mblen); - } else if (!strcasecmp(c->argv[1]->ptr,"numsub") && c->argc > 2) { - /* PUBSUB NUMSUB Channel_1 [... Channel_N] */ + } else if (!strcasecmp(c->argv[1]->ptr,"numsub") && c->argc >= 2) { + /* PUBSUB NUMSUB [Channel_1 ... Channel_N] */ int j; addReplyMultiBulkLen(c,(c->argc-2)*2); From 8328d993e1c57d9dde4fec017abfca9853757cde Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 21 Jun 2013 12:07:53 +0200 Subject: [PATCH 0665/1752] New API to force propagation. The old REDIS_CMD_FORCE_REPLICATION flag was removed from the implementation of Redis, now there is a new API to force specific executions of a command to be propagated to AOF / Replication link: void forceCommandPropagation(int flags); The new API is also compatible with Lua scripting, so a script that will execute commands that are forced to be propagated, will also be propagated itself accordingly even if no change to data is operated. As a side effect, this new design fixes the issue with scripts not able to propagate PUBLISH to slaves (issue #873). --- src/pubsub.c | 1 + src/redis.c | 32 ++++++++++++++++++++++++++++---- src/redis.h | 5 ++++- src/scripting.c | 1 + 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/src/pubsub.c b/src/pubsub.c index 9fafbdcb0b7..736d248da24 100644 --- a/src/pubsub.c +++ b/src/pubsub.c @@ -306,6 +306,7 @@ void punsubscribeCommand(redisClient *c) { void publishCommand(redisClient *c) { int receivers = pubsubPublishMessage(c->argv[1],c->argv[2]); + forceCommandPropagation(c,REDIS_PROPAGATE_REPL); addReplyLongLong(c,receivers); } diff --git a/src/redis.c b/src/redis.c index f34b2cba023..c6c5c77f573 100644 --- a/src/redis.c +++ b/src/redis.c @@ -237,7 +237,7 @@ struct redisCommand redisCommandTable[] = { {"unsubscribe",unsubscribeCommand,-1,"rpslt",0,NULL,0,0,0,0,0}, {"psubscribe",psubscribeCommand,-2,"rpslt",0,NULL,0,0,0,0,0}, {"punsubscribe",punsubscribeCommand,-1,"rpslt",0,NULL,0,0,0,0,0}, - {"publish",publishCommand,3,"pfltr",0,NULL,0,0,0,0,0}, + {"publish",publishCommand,3,"pltr",0,NULL,0,0,0,0,0}, {"pubsub",pubsubCommand,-2,"pltrR",0,NULL,0,0,0,0,0}, {"watch",watchCommand,-2,"rs",0,noPreloadGetKeys,1,-1,1,0,0}, {"unwatch",unwatchCommand,1,"rs",0,NULL,0,0,0,0,0}, @@ -1487,7 +1487,6 @@ void populateCommandTable(void) { case 'm': c->flags |= REDIS_CMD_DENYOOM; break; case 'a': c->flags |= REDIS_CMD_ADMIN; break; case 'p': c->flags |= REDIS_CMD_PUBSUB; break; - case 'f': c->flags |= REDIS_CMD_FORCE_REPLICATION; break; case 's': c->flags |= REDIS_CMD_NOSCRIPT; break; case 'R': c->flags |= REDIS_CMD_RANDOM; break; case 'S': c->flags |= REDIS_CMD_SORT_FOR_SCRIPT; break; @@ -1610,9 +1609,18 @@ void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, redisOpArrayAppend(&server.also_propagate,cmd,dbid,argv,argc,target); } +/* It is possible to call the function forceCommandPropagation() inside a + * Redis command implementaiton in order to to force the propagation of a + * specific command execution into AOF / Replication. */ +void forceCommandPropagation(redisClient *c, int flags) { + if (flags & REDIS_PROPAGATE_REPL) c->flags |= REDIS_FORCE_REPL; + if (flags & REDIS_PROPAGATE_AOF) c->flags |= REDIS_FORCE_AOF; +} + /* Call() is the core of Redis execution of a command */ void call(redisClient *c, int flags) { long long dirty, start = ustime(), duration; + int client_old_flags = c->flags; /* Sent the command to clients in MONITOR mode, only if the commands are * not generated from reading an AOF. */ @@ -1624,6 +1632,7 @@ void call(redisClient *c, int flags) { } /* Call the command. */ + c->flags &= ~(REDIS_FORCE_AOF|REDIS_FORCE_REPL); redisOpArrayInit(&server.also_propagate); dirty = server.dirty; c->cmd->proc(c); @@ -1635,6 +1644,16 @@ void call(redisClient *c, int flags) { if (server.loading && c->flags & REDIS_LUA_CLIENT) flags &= ~(REDIS_CALL_SLOWLOG | REDIS_CALL_STATS); + /* If the caller is Lua, we want to force the EVAL caller to propagate + * the script if the command flag or client flag are forcing the + * propagation. */ + if (c->flags & REDIS_LUA_CLIENT && server.lua_caller) { + if (c->flags & REDIS_FORCE_REPL) + server.lua_caller->flags |= REDIS_FORCE_REPL; + if (c->flags & REDIS_FORCE_AOF) + server.lua_caller->flags |= REDIS_FORCE_AOF; + } + /* Log the command into the Slow log if needed, and populate the * per-command statistics that we show in INFO commandstats. */ if (flags & REDIS_CALL_SLOWLOG && c->cmd->proc != execCommand) @@ -1648,14 +1667,19 @@ void call(redisClient *c, int flags) { if (flags & REDIS_CALL_PROPAGATE) { int flags = REDIS_PROPAGATE_NONE; - if (c->cmd->flags & REDIS_CMD_FORCE_REPLICATION) - flags |= REDIS_PROPAGATE_REPL; + if (c->flags & REDIS_FORCE_REPL) flags |= REDIS_PROPAGATE_REPL; + if (c->flags & REDIS_FORCE_AOF) flags |= REDIS_PROPAGATE_AOF; if (dirty) flags |= (REDIS_PROPAGATE_REPL | REDIS_PROPAGATE_AOF); if (flags != REDIS_PROPAGATE_NONE) propagate(c->cmd,c->db->id,c->argv,c->argc,flags); } + /* Restore the old FORCE_AOF/REPL flags, since call can be executed + * recursively. */ + c->flags &= ~(REDIS_FORCE_AOF|REDIS_FORCE_REPL); + c->flags |= client_old_flags & (REDIS_FORCE_AOF|REDIS_FORCE_REPL); + /* Handle the alsoPropagate() API to handle commands that want to propagate * multiple separated commands. */ if (server.also_propagate.numops) { diff --git a/src/redis.h b/src/redis.h index dee82da6d71..ee6f14f0160 100644 --- a/src/redis.h +++ b/src/redis.h @@ -138,7 +138,7 @@ #define REDIS_CMD_WRITE 1 /* "w" flag */ #define REDIS_CMD_READONLY 2 /* "r" flag */ #define REDIS_CMD_DENYOOM 4 /* "m" flag */ -#define REDIS_CMD_FORCE_REPLICATION 8 /* "f" flag */ +#define REDIS_CMD_NOT_USED_1 8 /* no longer used flag */ #define REDIS_CMD_ADMIN 16 /* "a" flag */ #define REDIS_CMD_PUBSUB 32 /* "p" flag */ #define REDIS_CMD_NOSCRIPT 64 /* "s" flag */ @@ -215,6 +215,8 @@ #define REDIS_UNIX_SOCKET (1<<11) /* Client connected via Unix domain socket */ #define REDIS_DIRTY_EXEC (1<<12) /* EXEC will fail for errors while queueing */ #define REDIS_MASTER_FORCE_REPLY (1<<13) /* Queue replies even if is master */ +#define REDIS_FORCE_AOF (1<<14) /* Force AOF propagation of current cmd. */ +#define REDIS_FORCE_REPL (1<<15) /* Force replication of current cmd. */ /* Client request types */ #define REDIS_REQ_INLINE 1 @@ -1043,6 +1045,7 @@ struct redisCommand *lookupCommandOrOriginal(sds name); void call(redisClient *c, int flags); void propagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int flags); void alsoPropagate(struct redisCommand *cmd, int dbid, robj **argv, int argc, int target); +void forceCommandPropagation(redisClient *c, int flags); int prepareForShutdown(); #ifdef __GNUC__ void redisLog(int level, const char *fmt, ...) diff --git a/src/scripting.c b/src/scripting.c index b94627c7f62..104bd3ddeef 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1042,6 +1042,7 @@ void scriptCommand(redisClient *c) { } addReplyBulkCBuffer(c,funcname+2,40); sdsfree(sha); + forceCommandPropagation(c,REDIS_PROPAGATE_REPL); } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { if (server.lua_caller == NULL) { addReplySds(c,sdsnew("-NOTBUSY No scripts in execution right now.\r\n")); From 9d894b1b8c49977d6812207b91ffff617f6db08c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 24 Jun 2013 10:26:04 +0200 Subject: [PATCH 0666/1752] Replication of scripts as EVALSHA: sha1 caching implemented. This code is only responsible to take an LRU-evicted fixed length cache of SHA1 that we are sure all the slaves received. In this commit only the implementation is provided, but the Redis core does not use it to actually send EVALSHA to slaves when possible. --- src/redis.c | 12 +++++++ src/redis.h | 5 +++ src/replication.c | 80 ++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index c6c5c77f573..04d33162f07 100644 --- a/src/redis.c +++ b/src/redis.c @@ -561,6 +561,18 @@ dictType keylistDictType = { dictListDestructor /* val destructor */ }; +/* Replication cached script dict (server.repl_scriptcache_dict). + * Keys are sds SHA1 strings, while values are not used at all in the current + * implementation. */ +dictType replScriptCacheDictType = { + dictSdsHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + NULL /* val destructor */ +}; + int htNeedsResize(dict *dict) { long long size, used; diff --git a/src/redis.h b/src/redis.h index ee6f14f0160..224094e7b8f 100644 --- a/src/redis.h +++ b/src/redis.h @@ -706,6 +706,10 @@ struct redisServer { int slave_priority; /* Reported in INFO and used by Sentinel. */ char repl_master_runid[REDIS_RUN_ID_SIZE+1]; /* Master run id for PSYNC. */ long long repl_master_initial_offset; /* Master PSYNC offset. */ + /* Replication script cache. */ + dict *repl_scriptcache_dict; /* SHA1 all slaves are aware of. */ + list *repl_scriptcache_fifo; /* First in, first out LRU eviction. */ + int repl_scriptcache_size; /* Max number of elements. */ /* Limits */ unsigned int maxclients; /* Max number of simultaneous clients */ unsigned long long maxmemory; /* Max number of memory bytes to use */ @@ -849,6 +853,7 @@ extern dictType dbDictType; extern dictType shaScriptObjectDictType; extern double R_Zero, R_PosInf, R_NegInf, R_Nan; extern dictType hashDictType; +extern dictType replScriptCacheDictType; /*----------------------------------------------------------------------------- * Functions prototypes diff --git a/src/replication.c b/src/replication.c index 02452505e7d..8f809ae6408 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1412,7 +1412,85 @@ void refreshGoodSlavesCount(void) { server.repl_good_slaves_count = good; } -/* --------------------------- REPLICATION CRON ---------------------------- */ +/* ----------------------- REPLICATION SCRIPT CACHE -------------------------- + * The goal of this code is to keep track of scripts already sent to every + * connected slave, in order to be able to replicate EVALSHA as it is without + * translating it to EVAL every time it is possible. + * + * We use a capped collection implemented by an hash table for fast lookup + * of scripts we can send as EVALSHA, plus a linked list that is used for + * eviction of the oldest entry when the max number of items is reached. + * + * We don't care about taking a different cache for every different slave + * since to fill the cache again is not very costly, the goal of this code + * is to avoid that the same big script is trasmitted a big number of times + * per second wasting bandwidth and processor speed, but it is not a problem + * if we need to rebuild the cache from scratch from time to time, every used + * script will need to be transmitted a single time to reappear in the cache. + * + * This is how the system works: + * + * 1) Every time a new slave connects, we flush the whole script cache. + * 2) We only send as EVALSHA what was sent to the master as EVALSHA, without + * trying to convert EVAL into EVALSHA specifically for slaves. + * 3) Every time we trasmit a script as EVAL to the slaves, we also add the + * corresponding SHA1 of the script into the cache as we are sure every + * slave knows about the script starting from now. + * 4) On SCRIPT FLUSH command, we replicate the command to all the slaves + * and at the same time flush the script cache. + * 5) When the last slave disconnects, flush the cache. + * 6) We handle SCRIPT LOAD as well since that's how scripts are loaded + * in the master sometimes. + */ + +/* Initialize the script cache, only called at startup. */ +void replicationScriptCacheInit(void) { + server.repl_scriptcache_size = 10000; + server.repl_scriptcache_dict = dictCreate(&replScriptCacheDictType,NULL); + server.repl_scriptcache_fifo = listCreate(); +} + +/* Empty the script cache. Should be called every time we are no longer sure + * that every slave knows about all the scripts in our set, for example + * every time a new slave connects to this master and performs a full + * resynchronization. There is no need to flush the cache when a partial + * resynchronization is performed. */ +void replicationScriptCacheFlush(void) { + dictEmpty(server.repl_scriptcache_dict); + listRelease(server.repl_scriptcache_fifo); + server.repl_scriptcache_fifo = listCreate(); +} + +/* Add an entry into the script cache, if we reach max number of entries the + * oldest is removed from the list. */ +void replicationScriptCacheAdd(sds sha1) { + int retval; + sds key = sdsdup(sha1); + + /* Evict oldest. */ + if (listLength(server.repl_scriptcache_fifo) == server.repl_scriptcache_size) + { + listNode *ln = listLast(server.repl_scriptcache_fifo); + sds oldest = listNodeValue(ln); + + retval = dictDelete(server.repl_scriptcache_dict,oldest); + redisAssert(retval == DICT_OK); + listDelNode(server.repl_scriptcache_fifo,ln); + } + + /* Add current. */ + retval = dictAdd(server.repl_scriptcache_dict,key,NULL); + listAddNodeHead(server.repl_scriptcache_fifo,key); + redisAssert(retval == DICT_OK); +} + +/* Returns non-zero if the specified entry exists inside the cache, that is, + * if all the slaves are aware of this script SHA1. */ +int replicationScriptCacheExists(sds sha1) { + return dictFetchValue(server.repl_scriptcache_dict,sha1) != NULL; +} + +/* --------------------------- REPLICATION CRON ----------------------------- */ /* Replication cron funciton, called 1 time per second. */ void replicationCron(void) { From 545fe0c31833981c1676ce56ae3a800f9a25fdb7 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 24 Jun 2013 18:57:31 +0200 Subject: [PATCH 0667/1752] Use the RSC to replicate EVALSHA unmodified. This commit uses the Replication Script Cache in order to avoid translating EVALSHA into EVAL whenever possible for both the AOF and slaves. --- src/aof.c | 1 + src/redis.c | 4 ++-- src/redis.h | 4 ++++ src/replication.c | 33 ++++++++++++++++++++++++++++----- src/scripting.c | 40 +++++++++++++++++++++++++--------------- 5 files changed, 60 insertions(+), 22 deletions(-) diff --git a/src/aof.c b/src/aof.c index 9e602ce0178..9ad85c5367b 100644 --- a/src/aof.c +++ b/src/aof.c @@ -998,6 +998,7 @@ int rewriteAppendOnlyFileBackground(void) { * accumulated by the parent into server.aof_rewrite_buf will start * with a SELECT statement and it will be safe to merge. */ server.aof_selected_db = -1; + replicationScriptCacheFlush(); return REDIS_OK; } return REDIS_OK; /* unreached */ diff --git a/src/redis.c b/src/redis.c index 04d33162f07..737d66494e0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -565,10 +565,10 @@ dictType keylistDictType = { * Keys are sds SHA1 strings, while values are not used at all in the current * implementation. */ dictType replScriptCacheDictType = { - dictSdsHash, /* hash function */ + dictSdsCaseHash, /* hash function */ NULL, /* key dup */ NULL, /* val dup */ - dictSdsKeyCompare, /* key compare */ + dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ NULL /* val destructor */ }; diff --git a/src/redis.h b/src/redis.h index 224094e7b8f..5d1bcaab4ff 100644 --- a/src/redis.h +++ b/src/redis.h @@ -999,6 +999,10 @@ void replicationHandleMasterDisconnection(void); void replicationCacheMaster(redisClient *c); void resizeReplicationBacklog(long long newsize); void refreshGoodSlavesCount(void); +void replicationScriptCacheInit(void); +void replicationScriptCacheFlush(void); +void replicationScriptCacheAdd(sds sha1); +int replicationScriptCacheExists(sds sha1); /* Generic persistence functions */ void startLoading(FILE *fp); diff --git a/src/replication.c b/src/replication.c index 8f809ae6408..0e3699d8adb 100644 --- a/src/replication.c +++ b/src/replication.c @@ -546,6 +546,8 @@ void syncCommand(redisClient *c) { return; } c->replstate = REDIS_REPL_WAIT_BGSAVE_END; + /* Flush the script cache for the new slave. */ + replicationScriptCacheFlush(); } if (server.repl_disable_tcp_nodelay) @@ -711,6 +713,11 @@ void updateSlavesWaitingBgsave(int bgsaveerr) { } } if (startbgsave) { + /* Since we are starting a new background save for one or more slaves, + * we flush the Replication Script Cache to use EVAL to propagate every + * new EVALSHA for the first time, since all the new slaves don't know + * about previous scripts. */ + replicationScriptCacheFlush(); if (rdbSaveBackground(server.rdb_filename) != REDIS_OK) { listIter li; @@ -1451,10 +1458,16 @@ void replicationScriptCacheInit(void) { } /* Empty the script cache. Should be called every time we are no longer sure - * that every slave knows about all the scripts in our set, for example - * every time a new slave connects to this master and performs a full - * resynchronization. There is no need to flush the cache when a partial - * resynchronization is performed. */ + * that every slave knows about all the scripts in our set, or when the + * current AOF "context" is no longer aware of the script. In general we + * should flush the cache: + * + * 1) Every time a new slave reconnects to this master and performs a + * full SYNC (PSYNC does not require flushing). + * 2) Every time an AOF rewrite is performed. + * 3) Every time we are left without slaves at all, and AOF is off, in order + * to reclaim otherwise unused memory. + */ void replicationScriptCacheFlush(void) { dictEmpty(server.repl_scriptcache_dict); listRelease(server.repl_scriptcache_fifo); @@ -1487,7 +1500,7 @@ void replicationScriptCacheAdd(sds sha1) { /* Returns non-zero if the specified entry exists inside the cache, that is, * if all the slaves are aware of this script SHA1. */ int replicationScriptCacheExists(sds sha1) { - return dictFetchValue(server.repl_scriptcache_dict,sha1) != NULL; + return dictFind(server.repl_scriptcache_dict,sha1) != NULL; } /* --------------------------- REPLICATION CRON ----------------------------- */ @@ -1604,6 +1617,16 @@ void replicationCron(void) { } } + /* If AOF is disabled and we no longer have attached slaves, we can + * free our Replication Script Cache as there is no need to propagate + * EVALSHA at all. */ + if (listLength(server.slaves) == 0 && + server.aof_state == REDIS_AOF_OFF && + listLength(server.repl_scriptcache_fifo) != 0) + { + replicationScriptCacheFlush(); + } + /* Refresh the number of slaves with lag <= min-slaves-max-lag. */ refreshGoodSlavesCount(); } diff --git a/src/scripting.c b/src/scripting.c index 104bd3ddeef..f30956bd2de 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -655,6 +655,10 @@ void scriptingInit(void) { * to global variables. */ scriptingEnableGlobalsProtection(lua); + /* Initialize the Replication Script Cache for EVALSHA propagation to + * slaves and AOF. */ + replicationScriptCacheInit(); + server.lua = lua; } @@ -931,23 +935,29 @@ void evalGenericCommand(redisClient *c, int evalsha) { luaReplyToRedisReply(c,lua); } - /* If we have slaves attached we want to replicate this command as - * EVAL instead of EVALSHA. We do this also in the AOF as currently there - * is no easy way to propagate a command in a different way in the AOF - * and in the replication link. + /* EVALSHA should be propagated to Slave and AOF file as full EVAL, unless + * we are sure that the script was already in the context of all the + * attached slaves *and* the current AOF file if enabled. + * + * To do so we use a cache of SHA1s of scripts that we already propagated + * as full EVAL, that's called the Replication Script Cache. * - * IMPROVEMENT POSSIBLE: - * 1) Replicate this command as EVALSHA in the AOF. - * 2) Remember what slave already received a given script, and replicate - * the EVALSHA against this slaves when possible. - */ + * For repliation, everytime a new slave attaches to the master, we need to + * flush our cache of scripts that can be replicated as EVALSHA, while + * for AOF we need to do so every time we rewrite the AOF file. */ if (evalsha) { - robj *script = dictFetchValue(server.lua_scripts,c->argv[1]->ptr); - - redisAssertWithInfo(c,NULL,script != NULL); - rewriteClientCommandArgument(c,0, - resetRefCount(createStringObject("EVAL",4))); - rewriteClientCommandArgument(c,1,script); + if (!replicationScriptCacheExists(c->argv[1]->ptr)) { + /* This script is not in our script cache, replicate it as + * EVAL, then add it into the script cache, as from now on + * slaves and AOF know about it. */ + robj *script = dictFetchValue(server.lua_scripts,c->argv[1]->ptr); + + replicationScriptCacheAdd(c->argv[1]->ptr); + redisAssertWithInfo(c,NULL,script != NULL); + rewriteClientCommandArgument(c,0, + resetRefCount(createStringObject("EVAL",4))); + rewriteClientCommandArgument(c,1,script); + } } } From 8450d6e24d1ed2a30b63591fe202914e1d3d8c2c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 24 Jun 2013 19:27:49 +0200 Subject: [PATCH 0668/1752] Move Replication Script Cache initialization in safer place. It should be called just one time at startup and not every time the Lua scripting engine is re-initialized, otherwise memory is leaked. --- src/redis.c | 1 + src/scripting.c | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index 737d66494e0..4ba73ab7fbb 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1476,6 +1476,7 @@ void initServer() { server.maxmemory_policy = REDIS_MAXMEMORY_NO_EVICTION; } + replicationScriptCacheInit(); scriptingInit(); slowlogInit(); bioInit(); diff --git a/src/scripting.c b/src/scripting.c index f30956bd2de..ff94ae165c1 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -655,10 +655,6 @@ void scriptingInit(void) { * to global variables. */ scriptingEnableGlobalsProtection(lua); - /* Initialize the Replication Script Cache for EVALSHA propagation to - * slaves and AOF. */ - replicationScriptCacheInit(); - server.lua = lua; } From b7233f40fb7348d687eb1e670f9e41cfcd8f0676 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 10:56:59 +0200 Subject: [PATCH 0669/1752] SCRIPT FLUSH comment minor pedantic improvement. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index ff94ae165c1..11ed4c616ec 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1020,7 +1020,7 @@ void scriptCommand(redisClient *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"flush")) { scriptingReset(); addReply(c,shared.ok); - server.dirty++; /* Replicating this command is a good idea. */ + server.dirty++; /* Propagating this command is a good idea. */ } else if (c->argc >= 2 && !strcasecmp(c->argv[1]->ptr,"exists")) { int j; From d34c1ac20816a18e424a529fb612b434797efb67 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 12:49:56 +0200 Subject: [PATCH 0670/1752] Force propagation of SCRIPT LOAD to AOF. --- src/scripting.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/scripting.c b/src/scripting.c index 11ed4c616ec..a794f34fff7 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1048,7 +1048,7 @@ void scriptCommand(redisClient *c) { } addReplyBulkCBuffer(c,funcname+2,40); sdsfree(sha); - forceCommandPropagation(c,REDIS_PROPAGATE_REPL); + forceCommandPropagation(c,REDIS_PROPAGATE_REPL|REDIS_PROPAGATE_AOF); } else if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"kill")) { if (server.lua_caller == NULL) { addReplySds(c,sdsnew("-NOTBUSY No scripts in execution right now.\r\n")); From 4e2b97bb893255becedc54d4bd0e64e51098bc84 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 15:13:14 +0200 Subject: [PATCH 0671/1752] Test: replication-3 test speedup in master-slave setup. --- tests/integration/replication-3.tcl | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/replication-3.tcl b/tests/integration/replication-3.tcl index e660bf4e526..57ed77c1899 100644 --- a/tests/integration/replication-3.tcl +++ b/tests/integration/replication-3.tcl @@ -2,9 +2,12 @@ start_server {tags {"repl"}} { start_server {} { test {First server should have role slave after SLAVEOF} { r -1 slaveof [srv 0 host] [srv 0 port] - after 1000 - s -1 role - } {slave} + wait_for_condition 50 100 { + [s -1 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } if {$::accurate} {set numops 50000} else {set numops 5000} From b0e0f3c5b2fe972408f558b345ae632ae72fef76 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 15:32:37 +0200 Subject: [PATCH 0672/1752] Test: randomInt() behavior commented. --- tests/support/util.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/support/util.tcl b/tests/support/util.tcl index 48d06b74157..c5a6853b39b 100644 --- a/tests/support/util.tcl +++ b/tests/support/util.tcl @@ -91,10 +91,12 @@ proc wait_for_sync r { } } +# Random integer between 0 and max (excluded). proc randomInt {max} { expr {int(rand()*$max)} } +# Random signed integer between -max and max (both extremes excluded). proc randomSignedInt {max} { set i [randomInt $max] if {rand() > 0.5} { From 9faa6b9dc3339c26e694aa7ba3779b885fabed4f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 15:35:48 +0200 Subject: [PATCH 0673/1752] Test: EVALSHA replication. --- tests/integration/replication-3.tcl | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/integration/replication-3.tcl b/tests/integration/replication-3.tcl index 57ed77c1899..48ffc6ec9ed 100644 --- a/tests/integration/replication-3.tcl +++ b/tests/integration/replication-3.tcl @@ -32,3 +32,53 @@ start_server {tags {"repl"}} { } } } + +start_server {tags {"repl"}} { + start_server {} { + test {First server should have role slave after SLAVEOF} { + r -1 slaveof [srv 0 host] [srv 0 port] + wait_for_condition 50 100 { + [s -1 master_link_status] eq {up} + } else { + fail "Replication not started." + } + } + + set numops 20000 ;# Enough to trigger the Script Cache LRU eviction. + + test {MASTER and SLAVE consistency with EVALSHA replication} { + array set oldsha {} + for {set j 0} {$j < $numops} {incr j} { + set key "key:$j" + # Make sure to create scripts that have different SHA1s + set script "return redis.call('incr','$key')" + set sha1 [r eval "return redis.sha1hex(\"$script\")" 0] + set oldsha($j) $sha1 + r eval $script 0 + set res [r evalsha $sha1 0] + assert {$res == 2} + # Additionally call one of the old scripts as well, at random. + set res [r evalsha $oldsha([randomInt $j]) 0] + assert {$res > 2} + } + + wait_for_condition 50 100 { + [r dbsize] == $numops && + [r -1 dbsize] == $numops && + [r debug digest] eq [r -1 debug digest] + } else { + set csv1 [csvdump r] + set csv2 [csvdump {r -1}] + set fd [open /tmp/repldump1.txt w] + puts -nonewline $fd $csv1 + close $fd + set fd [open /tmp/repldump2.txt w] + puts -nonewline $fd $csv2 + close $fd + puts "Master - Slave inconsistency" + puts "Run diff -u against /tmp/repldump*.txt for more info" + + } + } + } +} From 29308077a827fa120ca62e797bb9a46349a0aa10 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 15:36:48 +0200 Subject: [PATCH 0674/1752] Flush the replication script cache after SCRIPT FLUSH. --- src/scripting.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/scripting.c b/src/scripting.c index a794f34fff7..baf58527923 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -1020,6 +1020,7 @@ void scriptCommand(redisClient *c) { if (c->argc == 2 && !strcasecmp(c->argv[1]->ptr,"flush")) { scriptingReset(); addReply(c,shared.ok); + replicationScriptCacheFlush(); server.dirty++; /* Propagating this command is a good idea. */ } else if (c->argc >= 2 && !strcasecmp(c->argv[1]->ptr,"exists")) { int j; From 7cca200603a2a68c08b8cd0739304e79f1ef15c9 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 25 Jun 2013 15:49:07 +0200 Subject: [PATCH 0675/1752] Test: add some AOF testing to EVALSHA replication test. --- tests/integration/replication-3.tcl | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/integration/replication-3.tcl b/tests/integration/replication-3.tcl index 48ffc6ec9ed..0fcbad45b0e 100644 --- a/tests/integration/replication-3.tcl +++ b/tests/integration/replication-3.tcl @@ -46,6 +46,10 @@ start_server {tags {"repl"}} { set numops 20000 ;# Enough to trigger the Script Cache LRU eviction. + # While we are at it, enable AOF to test it will be consistent as well + # after the test. + r config set appendonly yes + test {MASTER and SLAVE consistency with EVALSHA replication} { array set oldsha {} for {set j 0} {$j < $numops} {incr j} { @@ -60,6 +64,13 @@ start_server {tags {"repl"}} { # Additionally call one of the old scripts as well, at random. set res [r evalsha $oldsha([randomInt $j]) 0] assert {$res > 2} + + # Trigger an AOF rewrite while we are half-way, this also + # forces the flush of the script cache, and we will cover + # more code as a result. + if {$j == $numops / 2} { + catch {r bgrewriteaof} + } } wait_for_condition 50 100 { @@ -79,6 +90,12 @@ start_server {tags {"repl"}} { puts "Run diff -u against /tmp/repldump*.txt for more info" } + + set old_digest [r debug digest] + r config set appendonly no + r debug loadaof + set new_digest [r debug digest] + assert {$old_digest eq $new_digest} } } } From cdf79c063fc601b7c2636dc11a2d41ce6d8fdb6c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 26 Jun 2013 10:11:20 +0200 Subject: [PATCH 0676/1752] Don't disconnect pre PSYNC replication clients for timeout. Clients using SYNC to replicate are older implementations, such as redis-cli --slave, and are not designed to acknowledge the master with REPLCONF ACK commands, so we don't have any feedback and should not disconnect them on timeout. --- src/redis.h | 1 + src/replication.c | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/redis.h b/src/redis.h index 5d1bcaab4ff..394333a7d94 100644 --- a/src/redis.h +++ b/src/redis.h @@ -217,6 +217,7 @@ #define REDIS_MASTER_FORCE_REPLY (1<<13) /* Queue replies even if is master */ #define REDIS_FORCE_AOF (1<<14) /* Force AOF propagation of current cmd. */ #define REDIS_FORCE_REPL (1<<15) /* Force replication of current cmd. */ +#define REDIS_PRE_PSYNC_SLAVE (1<<16) /* Slave don't understand PSYNC. */ /* Client request types */ #define REDIS_REQ_INLINE 1 diff --git a/src/replication.c b/src/replication.c index 0e3699d8adb..c142b9cc58e 100644 --- a/src/replication.c +++ b/src/replication.c @@ -505,6 +505,11 @@ void syncCommand(redisClient *c) { * resync. */ if (master_runid[0] != '?') server.stat_sync_partial_err++; } + } else { + /* If a slave uses SYNC, we are dealing with an old implementation + * of the replication protocol (like redis-cli --slave). Flag the client + * so that we don't expect to receive REPLCONF ACK feedbacks. */ + c->flags |= REDIS_PRE_PSYNC_SLAVE; } /* Full resynchronization. */ @@ -1586,6 +1591,7 @@ void replicationCron(void) { redisClient *slave = ln->value; if (slave->replstate != REDIS_REPL_ONLINE) continue; + if (slave->flags & REDIS_PRE_PSYNC_SLAVE) continue; if ((server.unixtime - slave->repl_ack_time) > server.repl_timeout) { char ip[32]; From bb0d0fd47950e2766b7df84640a0702ea453b82c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 26 Jun 2013 15:19:06 +0200 Subject: [PATCH 0677/1752] function renamed: popcount_binary -> redisPopcount. --- src/bitops.c | 4 ++-- src/redis.h | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bitops.c b/src/bitops.c index 1c2e13ddcf4..c96a9e3c74d 100644 --- a/src/bitops.c +++ b/src/bitops.c @@ -58,7 +58,7 @@ static int getBitOffsetFromArgument(redisClient *c, robj *o, size_t *offset) { /* Count number of bits set in the binary array pointed by 's' and long * 'count' bytes. The implementation of this function is required to * work with a input string length up to 512 MB. */ -size_t popcount_binary(void *s, long count) { +size_t redisPopcount(void *s, long count) { size_t bits = 0; unsigned char *p; uint32_t *p4 = s; @@ -407,6 +407,6 @@ void bitcountCommand(redisClient *c) { } else { long bytes = end-start+1; - addReplyLongLong(c,popcount_binary(p+start,bytes)); + addReplyLongLong(c,redisPopcount(p+start,bytes)); } } diff --git a/src/redis.h b/src/redis.h index 394333a7d94..d20c9ed5121 100644 --- a/src/redis.h +++ b/src/redis.h @@ -866,6 +866,7 @@ long long mstime(void); void getRandomHexChars(char *p, unsigned int len); uint64_t crc64(uint64_t crc, const unsigned char *s, uint64_t l); void exitFromChild(int retcode); +size_t redisPopcount(void *s, long count); void redisSetProcTitle(char *title); /* networking.c -- Networking and Client related operations */ From 7bfbc38068f3577fde57bcadbf983a85b1ca0632 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 26 Jun 2013 15:25:53 +0200 Subject: [PATCH 0678/1752] Redis 2.7.3 --- src/version.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.h b/src/version.h index ddede3772b0..da35b43e159 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.2" +#define REDIS_VERSION "2.7.3" From 0a0fd37e5b7290ad04c2d1426c42ef38a0726c18 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 27 Jun 2013 12:14:23 +0200 Subject: [PATCH 0679/1752] Allow SHUTDOWN in loading state. --- src/db.c | 6 ++++++ src/redis.c | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 61f3b4ffb0e..34e9d3476fd 100644 --- a/src/db.c +++ b/src/db.c @@ -353,6 +353,12 @@ void shutdownCommand(redisClient *c) { return; } } + /* SHUTDOWN can be called even while the server is in "loading" state. + * When this happens we need to make sure no attempt is performed to save + * the dataset on shutdown (otherwise it could overwrite the current DB + * with half-read data). */ + if (server.loading) + flags = (flags & ~REDIS_SHUTDOWN_SAVE) | REDIS_SHUTDOWN_NOSAVE; if (prepareForShutdown(flags) == REDIS_OK) exit(0); addReplyError(c,"Errors trying to SHUTDOWN. Check logs."); } diff --git a/src/redis.c b/src/redis.c index 4ba73ab7fbb..a5d90e517e2 100644 --- a/src/redis.c +++ b/src/redis.c @@ -213,7 +213,7 @@ struct redisCommand redisCommandTable[] = { {"save",saveCommand,1,"ars",0,NULL,0,0,0,0,0}, {"bgsave",bgsaveCommand,1,"ar",0,NULL,0,0,0,0,0}, {"bgrewriteaof",bgrewriteaofCommand,1,"ar",0,NULL,0,0,0,0,0}, - {"shutdown",shutdownCommand,-1,"ar",0,NULL,0,0,0,0,0}, + {"shutdown",shutdownCommand,-1,"arl",0,NULL,0,0,0,0,0}, {"lastsave",lastsaveCommand,1,"rR",0,NULL,0,0,0,0,0}, {"type",typeCommand,2,"r",0,NULL,1,1,1,0,0}, {"multi",multiCommand,1,"rs",0,NULL,0,0,0,0,0}, From 75dc48f04a0050abc4f78b81e38bcf59c26b74df Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Jun 2013 16:39:49 +0200 Subject: [PATCH 0680/1752] ae.c event loop: API to resize the fd set size on the run. --- src/ae.c | 30 ++++++++++++++++++++++++++++++ src/ae.h | 2 ++ src/ae_epoll.c | 7 +++++++ src/ae_evport.c | 5 +++++ src/ae_kqueue.c | 8 +++++++- src/ae_select.c | 6 ++++++ 6 files changed, 57 insertions(+), 1 deletion(-) diff --git a/src/ae.c b/src/ae.c index 6ca9a51538d..164f8fdeb9f 100644 --- a/src/ae.c +++ b/src/ae.c @@ -91,6 +91,36 @@ aeEventLoop *aeCreateEventLoop(int setsize) { return NULL; } +/* Return the current set size. */ +int aeGetSetSize(aeEventLoop *eventLoop) { + return eventLoop->setsize; +} + +/* Resize the maximum set size of the event loop. + * If the requested set size is smaller than the current set size, but + * there is already a file descriptor in use that is >= the requested + * set size minus one, AE_ERR is returned and the operation is not + * performed at all. + * + * Otherwise AE_OK is returned and the operation is successful. */ +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize) { + int i; + + if (setsize == eventLoop->setsize) return AE_OK; + if (eventLoop->maxfd >= setsize) return AE_ERR; + if (aeApiResize(eventLoop,setsize) == -1) return AE_ERR; + + eventLoop->events = zrealloc(eventLoop->events,sizeof(aeFileEvent)*setsize); + eventLoop->fired = zrealloc(eventLoop->fired,sizeof(aeFiredEvent)*setsize); + eventLoop->setsize = setsize; + + /* Make sure that if we created new slots, they are initialized with + * an AE_NONE mask. */ + for (i = eventLoop->maxfd+1; i < setsize; i++) + eventLoop->events[i].mask = AE_NONE; + return AE_OK; +} + void aeDeleteEventLoop(aeEventLoop *eventLoop) { aeApiFree(eventLoop); zfree(eventLoop->events); diff --git a/src/ae.h b/src/ae.h index 4d89502429d..15ca1b5e750 100644 --- a/src/ae.h +++ b/src/ae.h @@ -114,5 +114,7 @@ int aeWait(int fd, int mask, long long milliseconds); void aeMain(aeEventLoop *eventLoop); char *aeGetApiName(void); void aeSetBeforeSleepProc(aeEventLoop *eventLoop, aeBeforeSleepProc *beforesleep); +int aeGetSetSize(aeEventLoop *eventLoop); +int aeResizeSetSize(aeEventLoop *eventLoop, int setsize); #endif diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 4823c281eac..41af3e8749f 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -55,6 +55,13 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return 0; } +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct epoll_event)*setsize); + return 0; +} + static void aeApiFree(aeEventLoop *eventLoop) { aeApiState *state = eventLoop->apidata; diff --git a/src/ae_evport.c b/src/ae_evport.c index 94413c132d0..5c317becb6f 100644 --- a/src/ae_evport.c +++ b/src/ae_evport.c @@ -94,6 +94,11 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return 0; } +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Nothing to resize here. */ + return 0; +} + static void aeApiFree(aeEventLoop *eventLoop) { aeApiState *state = eventLoop->apidata; diff --git a/src/ae_kqueue.c b/src/ae_kqueue.c index 458772f7e45..dbcc5805f3f 100644 --- a/src/ae_kqueue.c +++ b/src/ae_kqueue.c @@ -54,10 +54,16 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return -1; } eventLoop->apidata = state; - return 0; } +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + aeApiState *state = eventLoop->apidata; + + state->events = zrealloc(state->events, sizeof(struct kevent)*setsize); + return 0; +} + static void aeApiFree(aeEventLoop *eventLoop) { aeApiState *state = eventLoop->apidata; diff --git a/src/ae_select.c b/src/ae_select.c index f732e8e1e28..e2b7a9e8aca 100644 --- a/src/ae_select.c +++ b/src/ae_select.c @@ -48,6 +48,12 @@ static int aeApiCreate(aeEventLoop *eventLoop) { return 0; } +static int aeApiResize(aeEventLoop *eventLoop, int setsize) { + /* Just ensure we have enough room in the fd_set type. */ + if (setsize >= FD_SETSIZE) return -1; + return 0; +} + static void aeApiFree(aeEventLoop *eventLoop) { zfree(eventLoop->apidata); } From 8090c37f71737a788bf8f0492a9240266e2a6a5f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 28 Jun 2013 17:08:03 +0200 Subject: [PATCH 0681/1752] CONFIG SET maxclients. --- src/config.c | 31 ++++++++++++++++++++++++++++--- src/redis.c | 2 +- src/redis.h | 5 +++++ 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index dad575965aa..364f7d1311c 100644 --- a/src/config.c +++ b/src/config.c @@ -541,10 +541,35 @@ void configSetCommand(redisClient *c) { } freeMemoryIfNeeded(); } + } else if (!strcasecmp(c->argv[2]->ptr,"maxclients")) { + int orig_value = server.maxclients; + + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + + /* Try to check if the OS is capable of supporting so many FDs. */ + server.maxclients = ll; + if (ll > orig_value) { + adjustOpenFilesLimit(); + if (server.maxclients != ll) { + addReplyErrorFormat(c,"The operating system is not able to handle the specified number of clients, try with %d", server.maxclients); + server.maxclients = orig_value; + return; + } + if (aeGetSetSize(server.el) < + server.maxclients + REDIS_EVENTLOOP_FDSET_INCR) + { + if (aeResizeSetSize(server.el, + server.maxclients + REDIS_EVENTLOOP_FDSET_INCR) == AE_ERR) + { + addReplyError(c,"The event loop API used by Redis is not able to handle the specified number of clients"); + server.maxclients = orig_value; + return; + } + } + } } else if (!strcasecmp(c->argv[2]->ptr,"hz")) { - if (getLongLongFromObject(o,&ll) == REDIS_ERR || - ll < 0) goto badfmt; - server.hz = (int) ll; + if (getLongLongFromObject(o,&ll) == REDIS_ERR || ll < 0) goto badfmt; + server.hz = ll; if (server.hz < REDIS_MIN_HZ) server.hz = REDIS_MIN_HZ; if (server.hz > REDIS_MAX_HZ) server.hz = REDIS_MAX_HZ; } else if (!strcasecmp(c->argv[2]->ptr,"maxmemory-policy")) { diff --git a/src/redis.c b/src/redis.c index a5d90e517e2..a7d36fe933b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1382,7 +1382,7 @@ void initServer() { createSharedObjects(); adjustOpenFilesLimit(); - server.el = aeCreateEventLoop(server.maxclients+1024); + server.el = aeCreateEventLoop(server.maxclients+REDIS_EVENTLOOP_FDSET_INCR); server.db = zmalloc(sizeof(redisDb)*server.dbnum); if (server.port != 0) { diff --git a/src/redis.h b/src/redis.h index d20c9ed5121..20e282acf53 100644 --- a/src/redis.h +++ b/src/redis.h @@ -129,6 +129,10 @@ #define REDIS_MBULK_BIG_ARG (1024*32) #define REDIS_LONGSTR_SIZE 21 /* Bytes needed for long -> str */ #define REDIS_AOF_AUTOSYNC_BYTES (1024*1024*32) /* fdatasync every 32MB */ +/* When configuring the Redis eventloop, we setup it so that the total number + * of file descriptors we can handle are server.maxclients + FDSET_INCR + * that is our safety margin. */ +#define REDIS_EVENTLOOP_FDSET_INCR 128 /* Hash table parameters */ #define REDIS_HT_MINFILL 10 /* Minimal hash table fill 10% */ @@ -1072,6 +1076,7 @@ int htNeedsResize(dict *dict); void oom(const char *msg); void populateCommandTable(void); void resetCommandTableStats(void); +void adjustOpenFilesLimit(void); /* Set data type */ robj *setTypeCreate(robj *value); From 028fdbb70b05263fc13330f722a1236b2656718e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 2 Jul 2013 11:56:52 +0200 Subject: [PATCH 0682/1752] getAbsolutePath() moved into utils.c --- src/redis.c | 52 ---------------------------------------------------- src/util.c | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ src/util.h | 3 +++ 3 files changed, 55 insertions(+), 52 deletions(-) diff --git a/src/redis.c b/src/redis.c index a7d36fe933b..b8a66af8970 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2761,58 +2761,6 @@ void redisSetProcTitle(char *title) { #endif } -/* Given the filename, return the absolute path as an SDS string, or NULL - * if it fails for some reason. Note that "filename" may be an absolute path - * already, this will be detected and handled correctly. - * - * The function does not try to normalize everything, but only the obvious - * case of one or more "../" appearning at the start of "filename" - * relative path. */ -sds getAbsolutePath(char *filename) { - char cwd[1024]; - sds abspath; - sds relpath = sdsnew(filename); - - relpath = sdstrim(relpath," \r\n\t"); - if (relpath[0] == '/') return relpath; /* Path is already absolute. */ - - /* If path is relative, join cwd and relative path. */ - if (getcwd(cwd,sizeof(cwd)) == NULL) { - sdsfree(relpath); - return NULL; - } - abspath = sdsnew(cwd); - if (sdslen(abspath) && abspath[sdslen(abspath)-1] != '/') - abspath = sdscat(abspath,"/"); - - /* At this point we have the current path always ending with "/", and - * the trimmed relative path. Try to normalize the obvious case of - * trailing ../ elements at the start of the path. - * - * For every "../" we find in the filename, we remove it and also remove - * the last element of the cwd, unless the current cwd is "/". */ - while (sdslen(relpath) >= 3 && - relpath[0] == '.' && relpath[1] == '.' && relpath[2] == '/') - { - relpath = sdsrange(relpath,3,-1); - if (sdslen(abspath) > 1) { - char *p = abspath + sdslen(abspath)-2; - int trimlen = 1; - - while(*p != '/') { - p--; - trimlen++; - } - abspath = sdsrange(abspath,0,-(trimlen+1)); - } - } - - /* Finally glue the two parts together. */ - abspath = sdscatsds(abspath,relpath); - sdsfree(relpath); - return abspath; -} - int main(int argc, char **argv) { struct timeval tv; diff --git a/src/util.c b/src/util.c index 24f936b6641..4b77e9fef5d 100644 --- a/src/util.c +++ b/src/util.c @@ -405,6 +405,58 @@ void getRandomHexChars(char *p, unsigned int len) { fclose(fp); } +/* Given the filename, return the absolute path as an SDS string, or NULL + * if it fails for some reason. Note that "filename" may be an absolute path + * already, this will be detected and handled correctly. + * + * The function does not try to normalize everything, but only the obvious + * case of one or more "../" appearning at the start of "filename" + * relative path. */ +sds getAbsolutePath(char *filename) { + char cwd[1024]; + sds abspath; + sds relpath = sdsnew(filename); + + relpath = sdstrim(relpath," \r\n\t"); + if (relpath[0] == '/') return relpath; /* Path is already absolute. */ + + /* If path is relative, join cwd and relative path. */ + if (getcwd(cwd,sizeof(cwd)) == NULL) { + sdsfree(relpath); + return NULL; + } + abspath = sdsnew(cwd); + if (sdslen(abspath) && abspath[sdslen(abspath)-1] != '/') + abspath = sdscat(abspath,"/"); + + /* At this point we have the current path always ending with "/", and + * the trimmed relative path. Try to normalize the obvious case of + * trailing ../ elements at the start of the path. + * + * For every "../" we find in the filename, we remove it and also remove + * the last element of the cwd, unless the current cwd is "/". */ + while (sdslen(relpath) >= 3 && + relpath[0] == '.' && relpath[1] == '.' && relpath[2] == '/') + { + relpath = sdsrange(relpath,3,-1); + if (sdslen(abspath) > 1) { + char *p = abspath + sdslen(abspath)-2; + int trimlen = 1; + + while(*p != '/') { + p--; + trimlen++; + } + abspath = sdsrange(abspath,0,-(trimlen+1)); + } + } + + /* Finally glue the two parts together. */ + abspath = sdscatsds(abspath,relpath); + sdsfree(relpath); + return abspath; +} + #ifdef UTIL_TEST_MAIN #include diff --git a/src/util.h b/src/util.h index 48425245a16..8e9b0281dd0 100644 --- a/src/util.h +++ b/src/util.h @@ -30,6 +30,8 @@ #ifndef __REDIS_UTIL_H #define __REDIS_UTIL_H +#include "sds.h" + int stringmatchlen(const char *p, int plen, const char *s, int slen, int nocase); int stringmatch(const char *p, const char *s, int nocase); long long memtoll(const char *p, int *err); @@ -37,5 +39,6 @@ int ll2string(char *s, size_t len, long long value); int string2ll(const char *s, size_t slen, long long *value); int string2l(const char *s, size_t slen, long *value); int d2string(char *buf, size_t len, double value); +sds getAbsolutePath(char *filename); #endif From 25a3ad7f7477e99bc101827b0ac80336889c4c64 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 2 Jul 2013 12:08:07 +0200 Subject: [PATCH 0683/1752] pathIsBaseName() added to utils.c The function is used to test that the specified string looks like just as the basename of a path, without any absolute or relative path. --- src/util.c | 8 ++++++++ src/util.h | 1 + 2 files changed, 9 insertions(+) diff --git a/src/util.c b/src/util.c index 4b77e9fef5d..022a6adf4cc 100644 --- a/src/util.c +++ b/src/util.c @@ -457,6 +457,14 @@ sds getAbsolutePath(char *filename) { return abspath; } +/* Return true if the specified path is just a file basename without any + * relative or absolute path. This function just checks that no / or \ + * character exists inside the specified path, that's enough in the + * environments where Redis runs. */ +int pathIsBaseName(char *path) { + return strchr(path,'/') == NULL && strchr(path,'\\') == NULL; +} + #ifdef UTIL_TEST_MAIN #include diff --git a/src/util.h b/src/util.h index 8e9b0281dd0..b3667cd6faa 100644 --- a/src/util.h +++ b/src/util.h @@ -40,5 +40,6 @@ int string2ll(const char *s, size_t slen, long long *value); int string2l(const char *s, size_t slen, long *value); int d2string(char *buf, size_t len, double value); sds getAbsolutePath(char *filename); +int pathIsBaseName(char *path); #endif From aaf56235353b400706ac0d58a5c4de500112c710 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 2 Jul 2013 12:14:28 +0200 Subject: [PATCH 0684/1752] Only allow basenames for dbfilename and appendfilename. This fixes issue #1094. --- src/config.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/config.c b/src/config.c index 364f7d1311c..24bf2e1c3e1 100644 --- a/src/config.c +++ b/src/config.c @@ -306,6 +306,10 @@ void loadServerConfigFromString(char *config) { } server.aof_state = yes ? REDIS_AOF_ON : REDIS_AOF_OFF; } else if (!strcasecmp(argv[0],"appendfilename") && argc == 2) { + if (!pathIsBaseName(argv[1])) { + err = "appendfilename can't be a path, just a filename"; + goto loaderr; + } zfree(server.aof_filename); server.aof_filename = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"no-appendfsync-on-rewrite") @@ -352,6 +356,10 @@ void loadServerConfigFromString(char *config) { zfree(server.pidfile); server.pidfile = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"dbfilename") && argc == 2) { + if (!pathIsBaseName(argv[1])) { + err = "dbfilename can't be a path, just a filename"; + goto loaderr; + } zfree(server.rdb_filename); server.rdb_filename = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"hash-max-ziplist-entries") && argc == 2) { @@ -522,6 +530,10 @@ void configSetCommand(redisClient *c) { o = c->argv[3]; if (!strcasecmp(c->argv[2]->ptr,"dbfilename")) { + if (!pathIsBaseName(o->ptr)) { + addReplyError(c, "dbfilename can't be a path, just a filename"); + return; + } zfree(server.rdb_filename); server.rdb_filename = zstrdup(o->ptr); } else if (!strcasecmp(c->argv[2]->ptr,"requirepass")) { From 847a40805feb77f278efd5c67c774ea6db7c1e2c Mon Sep 17 00:00:00 2001 From: charsyam Date: Tue, 20 Nov 2012 02:50:31 +0800 Subject: [PATCH 0685/1752] fix randstring bug --- src/ziplist.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/ziplist.c b/src/ziplist.c index 8a6b54a7c09..fdfc2e9ce09 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -1041,7 +1041,8 @@ void pop(unsigned char *zl, int where) { } int randstring(char *target, unsigned int min, unsigned int max) { - int p, len = min+rand()%(max-min+1); + int p = 0; + int len = min+rand()%(max-min+1); int minval, maxval; switch(rand() % 3) { case 0: From 2f6560f3519a8aab1609a3f7b1934b7f86ae45b3 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 2 Jul 2013 17:44:42 +0200 Subject: [PATCH 0686/1752] pqsort.c: remove the "switch to insertion sort" optimization. It causes catastrophic performance for certain inputs. Relevant NetBSD commit: http://cvsweb.netbsd.org/bsdweb.cgi/src/lib/libc/stdlib/qsort.c?rev=1.20&content-type=text/x-cvsweb-markup&only_with_tag=MAIN This fixes issue #968. --- src/pqsort.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/pqsort.c b/src/pqsort.c index 9c57aacd074..57c217f9446 100644 --- a/src/pqsort.c +++ b/src/pqsort.c @@ -102,10 +102,9 @@ _pqsort(void *a, size_t n, size_t es, { char *pa, *pb, *pc, *pd, *pl, *pm, *pn; size_t d, r; - int swaptype, swap_cnt, cmp_result; + int swaptype, cmp_result; loop: SWAPINIT(a, es); - swap_cnt = 0; if (n < 7) { for (pm = (char *) a + es; pm < (char *) a + n * es; pm += es) for (pl = pm; pl > (char *) a && cmp(pl - es, pl) > 0; @@ -132,7 +131,6 @@ loop: SWAPINIT(a, es); for (;;) { while (pb <= pc && (cmp_result = cmp(pb, a)) <= 0) { if (cmp_result == 0) { - swap_cnt = 1; swap(pa, pb); pa += es; } @@ -140,7 +138,6 @@ loop: SWAPINIT(a, es); } while (pb <= pc && (cmp_result = cmp(pc, a)) >= 0) { if (cmp_result == 0) { - swap_cnt = 1; swap(pc, pd); pd -= es; } @@ -149,17 +146,9 @@ loop: SWAPINIT(a, es); if (pb > pc) break; swap(pb, pc); - swap_cnt = 1; pb += es; pc -= es; } - if (swap_cnt == 0) { /* Switch to insertion sort */ - for (pm = (char *) a + es; pm < (char *) a + n * es; pm += es) - for (pl = pm; pl > (char *) a && cmp(pl - es, pl) > 0; - pl -= es) - swap(pl, pl - es); - return; - } pn = (char *) a + n * es; r = min(pa - (char *) a, pb - pa); From 24b3799238099a6fc438d57c9b5fff789c8a032e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 3 Jul 2013 11:59:44 +0200 Subject: [PATCH 0687/1752] redis-cli --pipe: send final ECHO in a safer way. If the protocol read from stdin happened to contain grabage (invalid random chars), in the previous implementation it was possible to end with something like: dksfjdksjflskfjl*2\r\n$4\r\nECHO.... That is invalid as the *2 should start into a new line. Now we prefix the ECHO with a CRLF that has no effects on the server but prevents this issues most of the times. Of course if the offending wrong sequence is something like: $3248772349\r\n No one is going to save us as Redis will wait for data in the context of a big argument, so this fix does not cover all the cases. This partially fixes issue #681. --- src/redis-cli.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index f12059ca4bd..d750867abac 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1203,8 +1203,12 @@ static void pipeMode(void) { ssize_t nread = read(STDIN_FILENO,obuf,sizeof(obuf)); if (nread == 0) { + /* The ECHO sequence starts with a "\r\n" so that if there + * is garbage in the protocol we read from stdin, the ECHO + * will likely still be properly formatted. + * CRLF is ignored by Redis, so it has no effects. */ char echo[] = - "*2\r\n$4\r\nECHO\r\n$20\r\n01234567890123456789\r\n"; + "\r\n*2\r\n$4\r\nECHO\r\n$20\r\n01234567890123456789\r\n"; int j; eof = 1; @@ -1213,7 +1217,7 @@ static void pipeMode(void) { * to make sure everything was read from the server. */ for (j = 0; j < 20; j++) magic[j] = rand() & 0xff; - memcpy(echo+19,magic,20); + memcpy(echo+21,magic,20); memcpy(obuf,echo,sizeof(echo)-1); obuf_len = sizeof(echo)-1; obuf_pos = 0; From 3e3501262b350f914a568464a663dc0af6c03c6a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 3 Jul 2013 12:18:55 +0200 Subject: [PATCH 0688/1752] redis-cli: introduced --pipe-timeout. When in --pipe mode, after all the data transfer to the server is complete, now redis-cli waits at max the specified amount of seconds (30 by default, use 0 to wait forever) without receiving any reply at all from the server. After this time limit the operation is aborted with an error. That's related to issue #681. --- src/redis-cli.c | 74 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index d750867abac..c9eba1b5914 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -58,6 +58,7 @@ #define OUTPUT_RAW 1 #define OUTPUT_CSV 2 #define REDIS_CLI_KEEPALIVE_INTERVAL 15 /* seconds */ +#define REDIS_DEFAULT_PIPE_TIMEOUT 30 /* seconds */ static redisContext *context; static struct config { @@ -77,6 +78,7 @@ static struct config { int cluster_reissue_command; int slave_mode; int pipe_mode; + int pipe_timeout; int getrdb_mode; int stat_mode; char *rdb_filename; @@ -713,6 +715,8 @@ static int parseOptions(int argc, char **argv) { config.rdb_filename = argv[++i]; } else if (!strcmp(argv[i],"--pipe")) { config.pipe_mode = 1; + } else if (!strcmp(argv[i],"--pipe-timeout") && !lastarg) { + config.pipe_timeout = atoi(argv[++i]); } else if (!strcmp(argv[i],"--bigkeys")) { config.bigkeys = 1; } else if (!strcmp(argv[i],"--eval") && !lastarg) { @@ -765,29 +769,32 @@ static void usage() { "redis-cli %s\n" "\n" "Usage: redis-cli [OPTIONS] [cmd [arg [arg ...]]]\n" -" -h Server hostname (default: 127.0.0.1)\n" -" -p Server port (default: 6379)\n" -" -s Server socket (overrides hostname and port)\n" -" -a Password to use when connecting to the server\n" -" -r Execute specified command N times\n" -" -i When -r is used, waits seconds per command.\n" -" It is possible to specify sub-second times like -i 0.1\n" -" -n Database number\n" -" -x Read last argument from STDIN\n" -" -d Multi-bulk delimiter in for raw formatting (default: \\n)\n" -" -c Enable cluster mode (follow -ASK and -MOVED redirections)\n" -" --raw Use raw formatting for replies (default when STDOUT is\n" -" not a tty)\n" -" --latency Enter a special mode continuously sampling latency\n" -" --latency-history Like --latency but tracking latency changes over time.\n" -" Default time interval is 15 sec. Change it using -i.\n" -" --slave Simulate a slave showing commands received from the master\n" -" --rdb Transfer an RDB dump from remote server to local file.\n" -" --pipe Transfer raw Redis protocol from stdin to server\n" -" --bigkeys Sample Redis keys looking for big keys\n" -" --eval Send an EVAL command using the Lua script at \n" -" --help Output this help and exit\n" -" --version Output version and exit\n" +" -h Server hostname (default: 127.0.0.1)\n" +" -p Server port (default: 6379)\n" +" -s Server socket (overrides hostname and port)\n" +" -a Password to use when connecting to the server\n" +" -r Execute specified command N times\n" +" -i When -r is used, waits seconds per command.\n" +" It is possible to specify sub-second times like -i 0.1\n" +" -n Database number\n" +" -x Read last argument from STDIN\n" +" -d Multi-bulk delimiter in for raw formatting (default: \\n)\n" +" -c Enable cluster mode (follow -ASK and -MOVED redirections)\n" +" --raw Use raw formatting for replies (default when STDOUT is\n" +" not a tty)\n" +" --latency Enter a special mode continuously sampling latency\n" +" --latency-history Like --latency but tracking latency changes over time.\n" +" Default time interval is 15 sec. Change it using -i.\n" +" --slave Simulate a slave showing commands received from the master\n" +" --rdb Transfer an RDB dump from remote server to local file.\n" +" --pipe Transfer raw Redis protocol from stdin to server\n" +" --pipe-timeout In --pipe mode, abort with error if after sending all data\n" +" no reply is received within seconds.\n" +" Default timeout: %d. Use 0 to wait forever.\n" +" --bigkeys Sample Redis keys looking for big keys\n" +" --eval Send an EVAL command using the Lua script at \n" +" --help Output this help and exit\n" +" --version Output version and exit\n" "\n" "Examples:\n" " cat /etc/passwd | redis-cli -x set mypasswd\n" @@ -800,7 +807,7 @@ static void usage() { "When no command is given, redis-cli starts in interactive mode.\n" "Type \"help\" in interactive mode for information on available commands.\n" "\n", - version); + version, REDIS_DEFAULT_PIPE_TIMEOUT); sdsfree(version); exit(1); } @@ -1118,6 +1125,7 @@ static void pipeMode(void) { int eof = 0; /* True once we consumed all the standard input. */ int done = 0; char magic[20]; /* Special reply we recognize. */ + time_t last_read_time = time(NULL); srand(time(NULL)); @@ -1148,7 +1156,10 @@ static void pipeMode(void) { strerror(errno)); exit(1); } - if (nread > 0) redisReaderFeed(reader,ibuf,nread); + if (nread > 0) { + redisReaderFeed(reader,ibuf,nread); + last_read_time = time(NULL); + } } while(nread > 0); /* Consume replies. */ @@ -1234,6 +1245,18 @@ static void pipeMode(void) { if (obuf_len == 0 && eof) break; } } + + /* Handle timeout, that is, we reached EOF, and we are not getting + * replies from the server for a few seconds, nor the final ECHO is + * received. */ + if (eof && config.pipe_timeout > 0 && + time(NULL)-last_read_time > config.pipe_timeout) + { + fprintf(stderr,"No replies for %d seconds: exiting.\n", + config.pipe_timeout); + errors++; + break; + } } redisReaderFree(reader); printf("errors: %lld, replies: %lld\n", errors, replies); @@ -1489,6 +1512,7 @@ int main(int argc, char **argv) { config.getrdb_mode = 0; config.rdb_filename = NULL; config.pipe_mode = 0; + config.pipe_timeout = REDIS_DEFAULT_PIPE_TIMEOUT; config.bigkeys = 0; config.stdinarg = 0; config.auth = NULL; From 2150cfebb7204978ff40734af09d69a2126237c3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 4 Jul 2013 18:30:59 +0200 Subject: [PATCH 0689/1752] sds.c: new function sdsjoin() to join strings. --- src/sds.c | 13 +++++++++++++ src/sds.h | 1 + 2 files changed, 14 insertions(+) diff --git a/src/sds.c b/src/sds.c index 4cf700b9afd..aa45fc4f516 100644 --- a/src/sds.c +++ b/src/sds.c @@ -621,6 +621,19 @@ sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { return s; } +/* Join an array of C strings using the specified separator (also a C string). + * Returns the result as an sds string. */ +sds sdsjoin(char **argv, int argc, char *sep) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscat(join, argv[j]); + if (j != argc-1) join = sdscat(join,sep); + } + return join; +} + #ifdef SDS_TEST_MAIN #include #include "testhelp.h" diff --git a/src/sds.h b/src/sds.h index c5a4f30a931..46d914fd145 100644 --- a/src/sds.h +++ b/src/sds.h @@ -89,6 +89,7 @@ sds sdsfromlonglong(long long value); sds sdscatrepr(sds s, const char *p, size_t len); sds *sdssplitargs(const char *line, int *argc); sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep); /* Low level functions exposed to the user API */ sds sdsMakeRoomFor(sds s, size_t addlen); From 14857decfa54c0ae7f07b69bb9b6388e59519a7e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 4 Jul 2013 18:48:46 +0200 Subject: [PATCH 0690/1752] anet.c: Allow creation of TCP listening sockets bound to N addresses. --- src/anet.c | 23 +++++++++++++++-------- src/anet.h | 2 +- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/anet.c b/src/anet.c index 963b6688e53..7a6b7d4bf21 100644 --- a/src/anet.c +++ b/src/anet.c @@ -331,9 +331,9 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { return ANET_OK; } -int anetTcpServer(char *err, int port, char *bindaddr) +int anetTcpServer(char *err, int port, char **bindaddr, int bindaddr_count) { - int s; + int s, j; struct sockaddr_in sa; if ((s = anetCreateSocket(err,AF_INET)) == ANET_ERR) @@ -343,13 +343,20 @@ int anetTcpServer(char *err, int port, char *bindaddr) sa.sin_family = AF_INET; sa.sin_port = htons(port); sa.sin_addr.s_addr = htonl(INADDR_ANY); - if (bindaddr && inet_aton(bindaddr, &sa.sin_addr) == 0) { - anetSetError(err, "invalid bind address"); - close(s); - return ANET_ERR; + if (bindaddr_count) { + for (j = 0; j < bindaddr_count; j++) { + if (inet_aton(bindaddr[j], &sa.sin_addr) == 0) { + anetSetError(err, "invalid bind address"); + close(s); + return ANET_ERR; + } + if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) + return ANET_ERR; + } + } else { + if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) + return ANET_ERR; } - if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) - return ANET_ERR; return s; } diff --git a/src/anet.h b/src/anet.h index 696c2c22586..bf76dd24df2 100644 --- a/src/anet.h +++ b/src/anet.h @@ -45,7 +45,7 @@ int anetUnixConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf); -int anetTcpServer(char *err, int port, char *bindaddr); +int anetTcpServer(char *err, int port, char **bindaddr, int bindaddr_count); int anetUnixServer(char *err, char *path, mode_t perm); int anetTcpAccept(char *err, int serversock, char *ip, int *port); int anetUnixAccept(char *err, int serversock); From 6dabd34ad08eeca4dfd483bdb98126c97ff9302e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 4 Jul 2013 18:50:15 +0200 Subject: [PATCH 0691/1752] Ability to bind multiple addresses. --- src/config.c | 41 +++++++++++++++++++++++++++++++++++++---- src/redis.c | 6 +++--- src/redis.h | 4 +++- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/config.c b/src/config.c index 24bf2e1c3e1..dc9e50ea312 100644 --- a/src/config.c +++ b/src/config.c @@ -125,8 +125,15 @@ void loadServerConfigFromString(char *config) { if (server.port < 0 || server.port > 65535) { err = "Invalid port"; goto loaderr; } - } else if (!strcasecmp(argv[0],"bind") && argc == 2) { - server.bindaddr = zstrdup(argv[1]); + } else if (!strcasecmp(argv[0],"bind") && argc >= 2) { + int j, addresses = argc-1; + + if (addresses > REDIS_BINDADDR_MAX) { + err = "Too many bind addresses specified"; goto loaderr; + } + for (j = 0; j < addresses; j++) + server.bindaddr[j] = zstrdup(argv[j+1]); + server.bindaddr_count = addresses; } else if (!strcasecmp(argv[0],"unixsocket") && argc == 2) { server.unixsocket = zstrdup(argv[1]); } else if (!strcasecmp(argv[0],"unixsocketperm") && argc == 2) { @@ -895,7 +902,6 @@ void configGetCommand(redisClient *c) { config_get_string_field("dbfilename",server.rdb_filename); config_get_string_field("requirepass",server.requirepass); config_get_string_field("masterauth",server.masterauth); - config_get_string_field("bind",server.bindaddr); config_get_string_field("unixsocket",server.unixsocket); config_get_string_field("logfile",server.logfile); config_get_string_field("pidfile",server.pidfile); @@ -1081,6 +1087,14 @@ void configGetCommand(redisClient *c) { decrRefCount(flagsobj); matches++; } + if (stringmatch(pattern,"bind",0)) { + sds aux = sdsjoin(server.bindaddr,server.bindaddr_count," "); + + addReplyBulkCString(c,"bind"); + addReplyBulkCString(c,aux); + sdsfree(aux); + matches++; + } setDeferredMultiBulkLength(c,replylen,matches*2); } @@ -1472,6 +1486,25 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state } } +/* Rewrite the bind option. */ +void rewriteConfigBindOption(struct rewriteConfigState *state) { + int force = 1; + sds line, addresses; + char *option = "bind"; + + /* Nothing to rewrite if we don't have bind addresses. */ + if (server.bindaddr_count == 0) return; + + /* Rewrite as bind ... */ + addresses = sdsjoin(server.bindaddr,server.bindaddr_count," "); + line = sdsnew(option); + line = sdscatlen(line, " ", 1); + line = sdscatsds(line, addresses); + sdsfree(addresses); + + rewriteConfigRewriteLine(state,option,line,force); +} + /* Glue together the configuration lines in the current configuration * rewrite state into a single string, stripping multiple empty lines. */ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { @@ -1607,7 +1640,7 @@ int rewriteConfig(char *path) { rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); rewriteConfigStringOption(state,"pidfile",server.pidfile,REDIS_DEFAULT_PID_FILE); rewriteConfigNumericalOption(state,"port",server.port,REDIS_SERVERPORT); - rewriteConfigStringOption(state,"bind",server.bindaddr,NULL); + rewriteConfigBindOption(state); rewriteConfigStringOption(state,"unixsocket",server.unixsocket,NULL); rewriteConfigOctalOption(state,"unixsocketperm",server.unixsocketperm,REDIS_DEFAULT_UNIX_SOCKET_PERM); rewriteConfigNumericalOption(state,"timeout",server.maxidletime,REDIS_MAXIDLETIME); diff --git a/src/redis.c b/src/redis.c index b8a66af8970..f5bf01af0f4 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1185,7 +1185,7 @@ void initServerConfig() { server.runid[REDIS_RUN_ID_SIZE] = '\0'; server.arch_bits = (sizeof(long) == 8) ? 64 : 32; server.port = REDIS_SERVERPORT; - server.bindaddr = NULL; + server.bindaddr_count = 0; server.unixsocket = NULL; server.unixsocketperm = REDIS_DEFAULT_UNIX_SOCKET_PERM; server.ipfd = -1; @@ -1386,7 +1386,7 @@ void initServer() { server.db = zmalloc(sizeof(redisDb)*server.dbnum); if (server.port != 0) { - server.ipfd = anetTcpServer(server.neterr,server.port,server.bindaddr); + server.ipfd = anetTcpServer(server.neterr,server.port,server.bindaddr,server.bindaddr_count); if (server.ipfd == ANET_ERR) { redisLog(REDIS_WARNING, "Opening port %d: %s", server.port, server.neterr); @@ -2754,7 +2754,7 @@ void redisSetProcTitle(char *title) { #ifdef USE_SETPROCTITLE setproctitle("%s %s:%d", title, - server.bindaddr ? server.bindaddr : "*", + server.bindaddr_count ? server.bindaddr[0] : "*", server.port); #else REDIS_NOTUSED(title); diff --git a/src/redis.h b/src/redis.h index 20e282acf53..b4dc0932af5 100644 --- a/src/redis.h +++ b/src/redis.h @@ -120,6 +120,7 @@ #define REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 #define REDIS_DEFAULT_MIN_SLAVES_TO_WRITE 0 #define REDIS_DEFAULT_MIN_SLAVES_MAX_LAG 10 +#define REDIS_BINDADDR_MAX 16 /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ @@ -576,7 +577,8 @@ struct redisServer { int sentinel_mode; /* True if this instance is a Sentinel. */ /* Networking */ int port; /* TCP listening port */ - char *bindaddr; /* Bind address or NULL */ + char *bindaddr[REDIS_BINDADDR_MAX]; /* Addresses we should bind to */ + int bindaddr_count; /* Number of addresses in server.bindaddr[] */ char *unixsocket; /* UNIX socket path */ mode_t unixsocketperm; /* UNIX socket permission */ int ipfd; /* TCP socket file descriptor */ From 00063bea332660d1d87c9f49e94d0419977f6915 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 5 Jul 2013 11:07:55 +0200 Subject: [PATCH 0692/1752] Revert "anet.c: Allow creation of TCP listening sockets bound to N addresses." Bind() can't be called multiple times against the same socket, multiple sockets are required to bind multiple interfaces, silly me. This reverts commit bd234d62bbfe9feb735fd6d1cdb8f5ce811f54b4. --- src/anet.c | 23 ++++++++--------------- src/anet.h | 2 +- 2 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/anet.c b/src/anet.c index 7a6b7d4bf21..963b6688e53 100644 --- a/src/anet.c +++ b/src/anet.c @@ -331,9 +331,9 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { return ANET_OK; } -int anetTcpServer(char *err, int port, char **bindaddr, int bindaddr_count) +int anetTcpServer(char *err, int port, char *bindaddr) { - int s, j; + int s; struct sockaddr_in sa; if ((s = anetCreateSocket(err,AF_INET)) == ANET_ERR) @@ -343,20 +343,13 @@ int anetTcpServer(char *err, int port, char **bindaddr, int bindaddr_count) sa.sin_family = AF_INET; sa.sin_port = htons(port); sa.sin_addr.s_addr = htonl(INADDR_ANY); - if (bindaddr_count) { - for (j = 0; j < bindaddr_count; j++) { - if (inet_aton(bindaddr[j], &sa.sin_addr) == 0) { - anetSetError(err, "invalid bind address"); - close(s); - return ANET_ERR; - } - if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) - return ANET_ERR; - } - } else { - if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) - return ANET_ERR; + if (bindaddr && inet_aton(bindaddr, &sa.sin_addr) == 0) { + anetSetError(err, "invalid bind address"); + close(s); + return ANET_ERR; } + if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) + return ANET_ERR; return s; } diff --git a/src/anet.h b/src/anet.h index bf76dd24df2..696c2c22586 100644 --- a/src/anet.h +++ b/src/anet.h @@ -45,7 +45,7 @@ int anetUnixConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf); -int anetTcpServer(char *err, int port, char **bindaddr, int bindaddr_count); +int anetTcpServer(char *err, int port, char *bindaddr); int anetUnixServer(char *err, char *path, mode_t perm); int anetTcpAccept(char *err, int serversock, char *ip, int *port); int anetUnixAccept(char *err, int serversock); From fc022ca300c6e8e8e691f2d9630003df3dbe1847 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 5 Jul 2013 11:47:20 +0200 Subject: [PATCH 0693/1752] Binding multiple IPs done properly with multiple sockets. --- src/aof.c | 3 +-- src/rdb.c | 3 +-- src/redis.c | 71 ++++++++++++++++++++++++++++++++++++++++------------- src/redis.h | 4 ++- 4 files changed, 59 insertions(+), 22 deletions(-) diff --git a/src/aof.c b/src/aof.c index 9ad85c5367b..89f17abab8d 100644 --- a/src/aof.c +++ b/src/aof.c @@ -962,8 +962,7 @@ int rewriteAppendOnlyFileBackground(void) { char tmpfile[256]; /* Child */ - if (server.ipfd > 0) close(server.ipfd); - if (server.sofd > 0) close(server.sofd); + closeListeningSockets(0); redisSetProcTitle("redis-aof-rewrite"); snprintf(tmpfile,256,"temp-rewriteaof-bg-%d.aof", (int) getpid()); if (rewriteAppendOnlyFile(tmpfile) == REDIS_OK) { diff --git a/src/rdb.c b/src/rdb.c index 3118e320f53..e27efd15aeb 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -729,8 +729,7 @@ int rdbSaveBackground(char *filename) { int retval; /* Child */ - if (server.ipfd > 0) close(server.ipfd); - if (server.sofd > 0) close(server.sofd); + closeListeningSockets(0); redisSetProcTitle("redis-rdb-bgsave"); retval = rdbSave(filename); if (retval == REDIS_OK) { diff --git a/src/redis.c b/src/redis.c index f5bf01af0f4..31082bec77b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1188,7 +1188,7 @@ void initServerConfig() { server.bindaddr_count = 0; server.unixsocket = NULL; server.unixsocketperm = REDIS_DEFAULT_UNIX_SOCKET_PERM; - server.ipfd = -1; + server.ipfd_count = 0; server.sofd = -1; server.dbnum = REDIS_DEFAULT_DBNUM; server.verbosity = REDIS_DEFAULT_VERBOSITY; @@ -1385,14 +1385,25 @@ void initServer() { server.el = aeCreateEventLoop(server.maxclients+REDIS_EVENTLOOP_FDSET_INCR); server.db = zmalloc(sizeof(redisDb)*server.dbnum); + /* Open the TCP listening sockets. */ if (server.port != 0) { - server.ipfd = anetTcpServer(server.neterr,server.port,server.bindaddr,server.bindaddr_count); - if (server.ipfd == ANET_ERR) { - redisLog(REDIS_WARNING, "Opening port %d: %s", - server.port, server.neterr); - exit(1); + /* Force binding of 0.0.0.0 if no bind address is specified, always + * entering the loop if j == 0. */ + if (server.bindaddr_count == 0) server.bindaddr[0] = NULL; + for (j = 0; j < server.bindaddr_count || j == 0; j++) { + server.ipfd[server.ipfd_count] = anetTcpServer(server.neterr,server.port,server.bindaddr[j]); + if (server.ipfd[server.ipfd_count] == ANET_ERR) { + redisLog(REDIS_WARNING, + "Creating Server TCP listening socket %s:%d: %s", + server.bindaddr[j] ? server.bindaddr[j] : "*", + server.port, server.neterr); + exit(1); + } + server.ipfd_count++; } } + + /* Open the listening Unix domain socket. */ if (server.unixsocket != NULL) { unlink(server.unixsocket); /* don't care if this fails */ server.sofd = anetUnixServer(server.neterr,server.unixsocket,server.unixsocketperm); @@ -1401,10 +1412,14 @@ void initServer() { exit(1); } } - if (server.ipfd < 0 && server.sofd < 0) { + + /* Abort if there are no listening sockets at all. */ + if (server.ipfd_count == 0 && server.sofd < 0) { redisLog(REDIS_WARNING, "Configured to not listen anywhere, exiting."); exit(1); } + + /* Create the Redis databases, and initialize other internal state. */ for (j = 0; j < server.dbnum; j++) { server.db[j].dict = dictCreate(&dbDictType,NULL); server.db[j].expires = dictCreate(&keyptrDictType,NULL); @@ -1447,15 +1462,28 @@ void initServer() { server.unixtime = time(NULL); server.lastbgsave_status = REDIS_OK; server.repl_good_slaves_count = 0; + + /* Create the serverCron() time event, that's our main way to process + * background operations. */ if(aeCreateTimeEvent(server.el, 1, serverCron, NULL, NULL) == AE_ERR) { redisPanic("Can't create the serverCron time event."); exit(1); } - if (server.ipfd > 0 && aeCreateFileEvent(server.el,server.ipfd,AE_READABLE, - acceptTcpHandler,NULL) == AE_ERR) redisPanic("Unrecoverable error creating server.ipfd file event."); + + /* Create an event handler for accepting new connections in TCP and Unix + * domain sockets. */ + for (j = 0; j < server.ipfd_count; j++) { + if (aeCreateFileEvent(server.el, server.ipfd[j], AE_READABLE, + acceptTcpHandler,NULL) == AE_ERR) + { + redisPanic( + "Unrecoverable error creating server.ipfd file event."); + } + } if (server.sofd > 0 && aeCreateFileEvent(server.el,server.sofd,AE_READABLE, acceptUnixHandler,NULL) == AE_ERR) redisPanic("Unrecoverable error creating server.sofd file event."); + /* Open the AOF file if needed. */ if (server.aof_state == REDIS_AOF_ON) { server.aof_fd = open(server.aof_filename, O_WRONLY|O_APPEND|O_CREAT,0644); @@ -1860,6 +1888,21 @@ int processCommand(redisClient *c) { /*================================== Shutdown =============================== */ +/* Close listening sockets. Also unlink the unix domain socket if + * unlink_unix_socket is non-zero. */ +void closeListeningSockets(int unlink_unix_socket) { + int j; + + for (j = 0; j < server.ipfd_count; j++) close(server.ipfd[j]); + if (server.sofd != -1) close(server.sofd); + if (server.cluster_enabled) + for (j = 0; j < server.cfd_count; j++) close(server.cfd[j]); + if (unlink_unix_socket && server.unixsocket) { + redisLog(REDIS_NOTICE,"Removing the unix socket file."); + unlink(server.unixsocket); /* don't care if this fails */ + } +} + int prepareForShutdown(int flags) { int save = flags & REDIS_SHUTDOWN_SAVE; int nosave = flags & REDIS_SHUTDOWN_NOSAVE; @@ -1903,13 +1946,7 @@ int prepareForShutdown(int flags) { unlink(server.pidfile); } /* Close the listening sockets. Apparently this allows faster restarts. */ - if (server.ipfd != -1) close(server.ipfd); - if (server.sofd != -1) close(server.sofd); - if (server.unixsocket) { - redisLog(REDIS_NOTICE,"Removing the unix socket file."); - unlink(server.unixsocket); /* don't care if this fails */ - } - + closeListeningSockets(1); redisLog(REDIS_WARNING,"Redis is now ready to exit, bye bye..."); return REDIS_OK; } @@ -2845,7 +2882,7 @@ int main(int argc, char **argv) { linuxOvercommitMemoryWarning(); #endif loadDataFromDisk(); - if (server.ipfd > 0) + if (server.ipfd_count > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port); if (server.sofd > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); diff --git a/src/redis.h b/src/redis.h index b4dc0932af5..4cf3c51d666 100644 --- a/src/redis.h +++ b/src/redis.h @@ -581,7 +581,8 @@ struct redisServer { int bindaddr_count; /* Number of addresses in server.bindaddr[] */ char *unixsocket; /* UNIX socket path */ mode_t unixsocketperm; /* UNIX socket permission */ - int ipfd; /* TCP socket file descriptor */ + int ipfd[REDIS_BINDADDR_MAX]; /* TCP socket file descriptors */ + int ipfd_count; /* Used slots in ipfd[] */ int sofd; /* Unix socket file descriptor */ list *clients; /* List of active clients */ list *clients_to_close; /* Clients to close asynchronously */ @@ -1079,6 +1080,7 @@ void oom(const char *msg); void populateCommandTable(void); void resetCommandTableStats(void); void adjustOpenFilesLimit(void); +void closeListeningSockets(int unlink_unix_socket); /* Set data type */ robj *setTypeCreate(robj *value); From 323e5e2058d9d83dd554ae21f7c732a84f574d1b Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 8 Jul 2013 10:31:49 +0200 Subject: [PATCH 0694/1752] Remove cluster stuff from closeListeningSockets() for 2.8. --- src/redis.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 31082bec77b..0db6cb3ccb0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1895,8 +1895,6 @@ void closeListeningSockets(int unlink_unix_socket) { for (j = 0; j < server.ipfd_count; j++) close(server.ipfd[j]); if (server.sofd != -1) close(server.sofd); - if (server.cluster_enabled) - for (j = 0; j < server.cfd_count; j++) close(server.cfd[j]); if (unlink_unix_socket && server.unixsocket) { redisLog(REDIS_NOTICE,"Removing the unix socket file."); unlink(server.unixsocket); /* don't care if this fails */ From 33c5f0a243cfcf2d42c55d4143ae88e9d252ce95 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 8 Jul 2013 10:42:16 +0200 Subject: [PATCH 0695/1752] Example redis.conf: bind to multiple interfaces documented. --- redis.conf | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 83fc76add99..2e3239c8027 100644 --- a/redis.conf +++ b/redis.conf @@ -24,9 +24,14 @@ pidfile /var/run/redis.pid # If port 0 is specified Redis will not listen on a TCP socket. port 6379 -# If you want you can bind a single interface, if the bind option is not -# specified all the interfaces will listen for incoming connections. +# By default Redis listens for connections from all the network interfaces +# available on the server. It is possible to listen to just one or multiple +# interfaces using the "bind" configuration directive, followed by one or +# more IP addresses. # +# Examples: +# +# bind 192.168.1.100 10.0.0.1 # bind 127.0.0.1 # Specify the path for the unix socket that will be used to listen for From 855ff8df9f872b19519ca99f1e4bb505d66f7f20 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Fri, 17 Jun 2011 00:29:49 +0100 Subject: [PATCH 0696/1752] Use getaddrinfo(3) in anetResolve. #apichange Change anetResolve() function to use getaddrinfo(3) to resolve hostnames. Resolved hostnames are limited to those reachable by the AF_INET address family. API Change: anetResolve requires additional argument. additional argument required to specify the length of the character buffer the IP address is written to in order to comply with inet_ntop(3) function semantics. inet_ntop(3) replaces inet_ntoa(3) as it has been designed to be compatible with more address families. --- src/anet.c | 28 ++++++++++++++++------------ src/anet.h | 2 +- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/src/anet.c b/src/anet.c index 963b6688e53..afa1d647834 100644 --- a/src/anet.c +++ b/src/anet.c @@ -163,22 +163,26 @@ int anetTcpKeepAlive(char *err, int fd) return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf) +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { - struct sockaddr_in sa; + struct addrinfo hints, *info; + void *addr; + int rv; - sa.sin_family = AF_INET; - if (inet_aton(host, &sa.sin_addr) == 0) { - struct hostent *he; + memset(&hints,0,sizeof(hints)); + hints.ai_family = AF_INET; - he = gethostbyname(host); - if (he == NULL) { - anetSetError(err, "can't resolve: %s", host); - return ANET_ERR; - } - memcpy(&sa.sin_addr, he->h_addr, sizeof(struct in_addr)); + if ((rv = getaddrinfo(host, NULL, &hints, &info)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); + return ANET_ERR; } - strcpy(ipbuf,inet_ntoa(sa.sin_addr)); + if (info->ai_family == AF_INET) { + struct sockaddr_in *sa = (struct sockaddr_in *)info->ai_addr; + addr = &(sa->sin_addr); + } + + inet_ntop(info->ai_family, addr, ipbuf, ipbuf_len); + freeaddrinfo(info); return ANET_OK; } diff --git a/src/anet.h b/src/anet.h index 696c2c22586..efaa2cc9650 100644 --- a/src/anet.h +++ b/src/anet.h @@ -44,7 +44,7 @@ int anetTcpNonBlockConnect(char *err, char *addr, int port); int anetUnixConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); -int anetResolve(char *err, char *host, char *ipbuf); +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetTcpServer(char *err, int port, char *bindaddr); int anetUnixServer(char *err, char *path, mode_t perm); int anetTcpAccept(char *err, int serversock, char *ip, int *port); From d36dc1fc829f8596e0c1e35c818a22743ef760cb Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Fri, 17 Jun 2011 00:55:00 +0100 Subject: [PATCH 0697/1752] Add anetSetReuseAddr(err, fd) static function. Extract setting SO_REUSEADDR socket option into separate function so the same code can be more easily used by anetCreateSocket and other functions. --- src/anet.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/anet.c b/src/anet.c index afa1d647834..78c24bf2520 100644 --- a/src/anet.c +++ b/src/anet.c @@ -186,8 +186,19 @@ int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) return ANET_OK; } +static int anetSetReuseAddr(char *err, int fd) { + int yes = 1; + /* Make sure connection-intensive things like the redis benckmark + * will be able to close/open sockets a zillion of times */ + if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)) == -1) { + anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno)); + return ANET_ERR; + } + return ANET_OK; +} + static int anetCreateSocket(char *err, int domain) { - int s, on = 1; + int s; if ((s = socket(domain, SOCK_STREAM, 0)) == -1) { anetSetError(err, "creating socket: %s", strerror(errno)); return ANET_ERR; @@ -195,8 +206,8 @@ static int anetCreateSocket(char *err, int domain) { /* Make sure connection-intensive things like the redis benchmark * will be able to close/open sockets a zillion of times */ - if (setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)) == -1) { - anetSetError(err, "setsockopt SO_REUSEADDR: %s", strerror(errno)); + if (anetSetReuseAddr(err,s) == ANET_ERR) { + close(s); return ANET_ERR; } return s; From e50f2c93bc958bd0f40be51a35dd42084724e81c Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Fri, 17 Jun 2011 01:06:19 +0100 Subject: [PATCH 0698/1752] Use getaddrinfo(3) in anetTcpGenericConnect. Change anetTcpGenericConnect() function to use getaddrinfo(3) to perform address resolution, socket creation and connection. Resolved addresses are limited to those reachable by the AF_INET family. --- src/anet.c | 60 +++++++++++++++++++++++++++++++++--------------------- 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/anet.c b/src/anet.c index 78c24bf2520..bbff3e3e9da 100644 --- a/src/anet.c +++ b/src/anet.c @@ -217,38 +217,52 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONBLOCK 1 static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) { - int s; - struct sockaddr_in sa; + int s, rv; + char _port[6]; /* strlen("65535"); */ + struct addrinfo hints, *servinfo, *p; - if ((s = anetCreateSocket(err,AF_INET)) == ANET_ERR) + snprintf(_port,6,"%d",port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + + if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); return ANET_ERR; + } + for (p = servinfo; p != NULL; p = p->ai_next) { + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; - sa.sin_family = AF_INET; - sa.sin_port = htons(port); - if (inet_aton(addr, &sa.sin_addr) == 0) { - struct hostent *he; + /* if we set err then goto cleanup, otherwise next */ + if (anetSetReuseAddr(err,s) == ANET_ERR) { + goto error; + } + if (flags & ANET_CONNECT_NONBLOCK) { + if (anetNonBlock(err,s) != ANET_OK) + goto error; + } + if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { + if (errno == EINPROGRESS && + flags & ANET_CONNECT_NONBLOCK) + goto end; - he = gethostbyname(addr); - if (he == NULL) { - anetSetError(err, "can't resolve: %s", addr); close(s); - return ANET_ERR; + continue; } - memcpy(&sa.sin_addr, he->h_addr, sizeof(struct in_addr)); + + /* break with the socket */ + goto end; } - if (flags & ANET_CONNECT_NONBLOCK) { - if (anetNonBlock(err,s) != ANET_OK) - return ANET_ERR; + if (p == NULL) { + anetSetError(err, "creating socket: %s", strerror(errno)); + goto error; } - if (connect(s, (struct sockaddr*)&sa, sizeof(sa)) == -1) { - if (errno == EINPROGRESS && - flags & ANET_CONNECT_NONBLOCK) - return s; - anetSetError(err, "connect: %s", strerror(errno)); - close(s); - return ANET_ERR; - } +error: + s = ANET_ERR; +end: + freeaddrinfo(servinfo); return s; } From aee5ee70fad3f0c76eddf93e84d05ac5d7acf8b9 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Fri, 17 Jun 2011 01:49:21 +0100 Subject: [PATCH 0699/1752] Use getaddrinfo(3) in a anetTcpServer. Change anetTcpServer() function to use getaddrinfo(3) to perform address resolution, socket creation and binding. Resolved addresses are limited to those reachable by the AF_INET address family. --- src/anet.c | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/src/anet.c b/src/anet.c index bbff3e3e9da..0c33fbf45ad 100644 --- a/src/anet.c +++ b/src/anet.c @@ -362,23 +362,37 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { int anetTcpServer(char *err, int port, char *bindaddr) { - int s; - struct sockaddr_in sa; + int s, rv; + char _port[6]; /* strlen("65535") */ + struct addrinfo hints, *servinfo, *p; - if ((s = anetCreateSocket(err,AF_INET)) == ANET_ERR) - return ANET_ERR; + snprintf(_port,6,"%d",port); + memset(&hints,0,sizeof(hints)); + hints.ai_family = AF_INET; + hints.ai_socktype = SOCK_STREAM; + hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ - memset(&sa,0,sizeof(sa)); - sa.sin_family = AF_INET; - sa.sin_port = htons(port); - sa.sin_addr.s_addr = htonl(INADDR_ANY); - if (bindaddr && inet_aton(bindaddr, &sa.sin_addr) == 0) { - anetSetError(err, "invalid bind address"); - close(s); + if ((rv = getaddrinfo(bindaddr,_port,&hints,&servinfo)) != 0) { + anetSetError(err, "%s", gai_strerror(rv)); return ANET_ERR; } - if (anetListen(err,s,(struct sockaddr*)&sa,sizeof(sa)) == ANET_ERR) - return ANET_ERR; + for (p = servinfo; p != NULL; p = p->ai_next) { + if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) + continue; + + if (anetListen(err,s,p->ai_addr,p->ai_addrlen) == ANET_ERR) + goto error; /* could continue here? */ + goto end; + } + if (p == NULL) { + anetSetError(err, "unable to bind socket"); + goto error; + } + +error: + s = ANET_ERR; +end: + freeaddrinfo(servinfo); return s; } From 920ab4c926db13ec0aeef8e67c58e4792973fbc8 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Fri, 17 Jun 2011 19:47:49 +0100 Subject: [PATCH 0700/1752] Use inet_ntop(3) in anet. #apichange Replace inet_ntoa(3) calls with the more future proof inet_ntop(3) function which is capable of handling additional address families. API Change: anetTcpAccept() & anetPeerToString() additional argument additional argument required to specify the length of the character buffer the IP address is written to in order to comply with inet_ntop(3) function semantics. --- src/anet.c | 12 ++++++------ src/anet.h | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/anet.c b/src/anet.c index 0c33fbf45ad..358802b1ca8 100644 --- a/src/anet.c +++ b/src/anet.c @@ -431,14 +431,14 @@ static int anetGenericAccept(char *err, int s, struct sockaddr *sa, socklen_t *l return fd; } -int anetTcpAccept(char *err, int s, char *ip, int *port) { +int anetTcpAccept(char *err, int s, char *ip, size_t ip_len, int *port) { int fd; struct sockaddr_in sa; socklen_t salen = sizeof(sa); if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == ANET_ERR) return ANET_ERR; - if (ip) strcpy(ip,inet_ntoa(sa.sin_addr)); + if (ip) inet_ntop(sa.sin_family,(void*)&(sa.sin_addr),ip,ip_len); if (port) *port = ntohs(sa.sin_port); return fd; } @@ -453,7 +453,7 @@ int anetUnixAccept(char *err, int s) { return fd; } -int anetPeerToString(int fd, char *ip, int *port) { +int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { struct sockaddr_in sa; socklen_t salen = sizeof(sa); @@ -463,12 +463,12 @@ int anetPeerToString(int fd, char *ip, int *port) { ip[1] = '\0'; return -1; } - if (ip) strcpy(ip,inet_ntoa(sa.sin_addr)); + if (ip) inet_ntop(sa.sin_family,(void*)&(sa.sin_addr),ip,ip_len); if (port) *port = ntohs(sa.sin_port); return 0; } -int anetSockName(int fd, char *ip, int *port) { +int anetSockName(int fd, char *ip, size_t ip_len, int *port) { struct sockaddr_in sa; socklen_t salen = sizeof(sa); @@ -478,7 +478,7 @@ int anetSockName(int fd, char *ip, int *port) { ip[1] = '\0'; return -1; } - if (ip) strcpy(ip,inet_ntoa(sa.sin_addr)); + if (ip) inet_ntop(sa.sin_family,(void*)&(sa.sin_addr),ip,ip_len); if (port) *port = ntohs(sa.sin_port); return 0; } diff --git a/src/anet.h b/src/anet.h index efaa2cc9650..ccc67c634c6 100644 --- a/src/anet.h +++ b/src/anet.h @@ -47,14 +47,14 @@ int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetTcpServer(char *err, int port, char *bindaddr); int anetUnixServer(char *err, char *path, mode_t perm); -int anetTcpAccept(char *err, int serversock, char *ip, int *port); +int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); int anetWrite(int fd, char *buf, int count); int anetNonBlock(char *err, int fd); int anetEnableTcpNoDelay(char *err, int fd); int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); -int anetPeerToString(int fd, char *ip, int *port); +int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); int anetKeepAlive(char *err, int fd, int interval); #endif From 30836721f210f474c317d464f3231b660fcc22b8 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Fri, 17 Jun 2011 19:54:17 +0100 Subject: [PATCH 0701/1752] Update anetTcpAccept & anetPeerToString calls. Add the additional ip buffer length argument to function calls of anetTcpAccept and anetPeerToString in network.c and cluster.c --- src/networking.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 3981f48ef28..ecad9f7597c 100644 --- a/src/networking.c +++ b/src/networking.c @@ -557,7 +557,7 @@ void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { REDIS_NOTUSED(mask); REDIS_NOTUSED(privdata); - cfd = anetTcpAccept(server.neterr, fd, cip, &cport); + cfd = anetTcpAccept(server.neterr, fd, cip, sizeof(cip), &cport); if (cfd == AE_ERR) { redisLog(REDIS_WARNING,"Accepting client connection: %s", server.neterr); return; @@ -1129,7 +1129,7 @@ sds getClientInfoString(redisClient *client) { int emask; if (!(client->flags & REDIS_UNIX_SOCKET)) - anetPeerToString(client->fd,ip,&port); + anetPeerToString(client->fd,ip,sizeof(ip),&port); p = flags; if (client->flags & REDIS_SLAVE) { if (client->flags & REDIS_MONITOR) @@ -1210,7 +1210,7 @@ void clientCommand(redisClient *c) { int port; client = listNodeValue(ln); - if (anetPeerToString(client->fd,ip,&port) == -1) continue; + if (anetPeerToString(client->fd,ip,sizeof(ip),&port) == -1) continue; snprintf(addr,sizeof(addr),"%s:%d",ip,port); if (strcmp(addr,c->argv[2]->ptr) == 0) { addReply(c,shared.ok); From 68d72aa5b16a23886b801523b116b66936f7e957 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 19:06:26 +0100 Subject: [PATCH 0702/1752] Add macro to define clusterNode.ip buffer size. Add REDIS_CLUSTER_IPLEN macro to define the size of the clusterNode ip character array. Additionally use this macro in inet_ntop(3) calls where the size of the array was being defined manually. The REDIS_CLUSTER_IPLEN is defined as INET_ADDRSTRLEN which defines the correct size of a buffer to store an IPv4 address in. The INET_ADDRSTRLEN macro itself is defined in the header file and should be portable across the majority of systems. --- src/redis.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis.h b/src/redis.h index 4cf3c51d666..2fb03441b20 100644 --- a/src/redis.h +++ b/src/redis.h @@ -120,6 +120,7 @@ #define REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 #define REDIS_DEFAULT_MIN_SLAVES_TO_WRITE 0 #define REDIS_DEFAULT_MIN_SLAVES_MAX_LAG 10 +#define REDIS_IP_STR_LEN INET6_ADDRSTRLEN #define REDIS_BINDADDR_MAX 16 /* Protocol and I/O related defines */ From 3570411def708ec68d9292de3ba7e76a45d49a68 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Wed, 17 Oct 2012 22:32:21 +0100 Subject: [PATCH 0703/1752] Update calls to anetPeerToString to include ip_len. --- src/redis.c | 2 +- src/replication.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0db6cb3ccb0..971ebadb749 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2353,7 +2353,7 @@ sds genRedisInfoString(char *section) { int port; long lag = 0; - if (anetPeerToString(slave->fd,ip,&port) == -1) continue; + if (anetPeerToString(slave->fd,ip,sizeof(ip),&port) == -1) continue; switch(slave->replstate) { case REDIS_REPL_WAIT_BGSAVE_START: case REDIS_REPL_WAIT_BGSAVE_END: diff --git a/src/replication.c b/src/replication.c index c142b9cc58e..b2ceba193fc 100644 --- a/src/replication.c +++ b/src/replication.c @@ -293,7 +293,7 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** int j, port; sds cmdrepr = sdsnew("+"); robj *cmdobj; - char ip[32]; + char ip[REDIS_IP_STR_LEN]; struct timeval tv; gettimeofday(&tv,NULL); @@ -303,7 +303,7 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** } else if (c->flags & REDIS_UNIX_SOCKET) { cmdrepr = sdscatprintf(cmdrepr,"[%d unix:%s] ",dictid,server.unixsocket); } else { - anetPeerToString(c->fd,ip,&port); + anetPeerToString(c->fd,ip,sizeof(ip),&port); cmdrepr = sdscatprintf(cmdrepr,"[%d %s:%d] ",dictid,ip,port); } From 4b2e374e4a35d3e0173b4bcabe2f985d4ae6202f Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Wed, 17 Oct 2012 22:32:48 +0100 Subject: [PATCH 0704/1752] Update calls to anetResolve to include buffer size --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index ed0978694c9..cb70087272a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -426,7 +426,7 @@ sentinelAddr *createSentinelAddr(char *hostname, int port) { errno = EINVAL; return NULL; } - if (anetResolve(NULL,hostname,buf) == ANET_ERR) { + if (anetResolve(NULL,hostname,buf,sizeof(buf)) == ANET_ERR) { errno = ENOENT; return NULL; } From 77ddec8582c1c7b9d8f406f166ce83676ebd2e7d Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 13:25:31 +0100 Subject: [PATCH 0705/1752] Update anetResolve to resolve AF_INET6 as well. Change the getaddrinfo(3) hints family from AF_INET to AF_UNSPEC to allow resolution of IPv6 addresses as well as IPv4 addresses. The function will return the IP address of whichever address family is preferenced by the operating system. Most current operating systems will preference AF_INET6 over AF_INET. Unfortunately without attempting to establish a connection to the remote address we can't know if the host is capable of using the returned IP address. It might be desirable to have anetResolve accept an additional argument specifying the AF_INET/AF_INET6 address the caller would like to receive. Currently though it does not appear as though the anetResolve function is ever used within Redis. --- src/anet.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/anet.c b/src/anet.c index 358802b1ca8..d75a4802a30 100644 --- a/src/anet.c +++ b/src/anet.c @@ -166,11 +166,11 @@ int anetTcpKeepAlive(char *err, int fd) int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { struct addrinfo hints, *info; - void *addr; int rv; memset(&hints,0,sizeof(hints)); - hints.ai_family = AF_INET; + hints.ai_family = AF_UNSPEC; + hints.ai_socktype = SOCK_STREAM; /* specify socktype to avoid dups */ if ((rv = getaddrinfo(host, NULL, &hints, &info)) != 0) { anetSetError(err, "%s", gai_strerror(rv)); @@ -178,10 +178,12 @@ int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) } if (info->ai_family == AF_INET) { struct sockaddr_in *sa = (struct sockaddr_in *)info->ai_addr; - addr = &(sa->sin_addr); + inet_ntop(AF_INET, &(sa->sin_addr), ipbuf, ipbuf_len); + } else { + struct sockaddr_in6 *sa = (struct sockaddr_in6 *)info->ai_addr; + inet_ntop(AF_INET6, &(sa->sin6_addr), ipbuf, ipbuf_len); } - inet_ntop(info->ai_family, addr, ipbuf, ipbuf_len); freeaddrinfo(info); return ANET_OK; } From fefa14914e0cbad22ac4afea3bb3421a48d13d71 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 14:47:38 +0100 Subject: [PATCH 0706/1752] Update anetTcpAccept to handle AF_INET6 addresses. Change the sockaddr_in to sockaddr_storage which is capable of storing both AF_INET and AF_INET6 sockets. Uses the sockaddr_storage ss_family to correctly return the printable IP address and port. --- src/anet.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/anet.c b/src/anet.c index d75a4802a30..0d6a358e33b 100644 --- a/src/anet.c +++ b/src/anet.c @@ -435,13 +435,20 @@ static int anetGenericAccept(char *err, int s, struct sockaddr *sa, socklen_t *l int anetTcpAccept(char *err, int s, char *ip, size_t ip_len, int *port) { int fd; - struct sockaddr_in sa; + struct sockaddr_storage sa; socklen_t salen = sizeof(sa); if ((fd = anetGenericAccept(err,s,(struct sockaddr*)&sa,&salen)) == ANET_ERR) return ANET_ERR; - if (ip) inet_ntop(sa.sin_family,(void*)&(sa.sin_addr),ip,ip_len); - if (port) *port = ntohs(sa.sin_port); + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } return fd; } From f1151f869e250d582467f2f1ef1f7461cc4970bd Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 16:20:08 +0100 Subject: [PATCH 0707/1752] Update anetPeerToString to handle AF_INET6 addrs. Change the sockaddr_in to sockaddr_storage which is capable of storing both AF_INET and AF_INET6 sockets. Uses the sockaddr_storage ss_family to correctly return the printable IP address and port. --- src/anet.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/anet.c b/src/anet.c index 0d6a358e33b..1996a487ba3 100644 --- a/src/anet.c +++ b/src/anet.c @@ -463,7 +463,7 @@ int anetUnixAccept(char *err, int s) { } int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { - struct sockaddr_in sa; + struct sockaddr_storage sa; socklen_t salen = sizeof(sa); if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) { @@ -472,13 +472,20 @@ int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { ip[1] = '\0'; return -1; } - if (ip) inet_ntop(sa.sin_family,(void*)&(sa.sin_addr),ip,ip_len); - if (port) *port = ntohs(sa.sin_port); + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } return 0; } int anetSockName(int fd, char *ip, size_t ip_len, int *port) { - struct sockaddr_in sa; + struct sockaddr_storage sa; socklen_t salen = sizeof(sa); if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) { @@ -487,7 +494,14 @@ int anetSockName(int fd, char *ip, size_t ip_len, int *port) { ip[1] = '\0'; return -1; } - if (ip) inet_ntop(sa.sin_family,(void*)&(sa.sin_addr),ip,ip_len); - if (port) *port = ntohs(sa.sin_port); + if (sa.ss_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&sa; + if (ip) inet_ntop(AF_INET,(void*)&(s->sin_addr),ip,ip_len); + if (port) *port = ntohs(s->sin_port); + } else { + struct sockaddr_in6 *s = (struct sockaddr_in6 *)&sa; + if (ip) inet_ntop(AF_INET6,(void*)&(s->sin6_addr),ip,ip_len); + if (port) *port = ntohs(s->sin6_port); + } return 0; } From 39d57c057f3cf9dd2682548f554fe117007d289b Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 19:34:16 +0100 Subject: [PATCH 0708/1752] Mark ip string buffers which could be reduced. In two places buffers have been created with a size of 128 bytes which could be reduced to INET6_ADDRSTRLEN to still hold a full IP address. These places have been marked as they are presently big enough to handle the needs of storing a printable IPv6 address. --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index ecad9f7597c..fcaea13420f 100644 --- a/src/networking.c +++ b/src/networking.c @@ -552,7 +552,7 @@ static void acceptCommonHandler(int fd, int flags) { void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cport, cfd; - char cip[128]; + char cip[128]; /* Could use INET6_ADDRSTRLEN here, but its smaller */ REDIS_NOTUSED(el); REDIS_NOTUSED(mask); REDIS_NOTUSED(privdata); From 57e2b23dfb687d8ee28bfb4b9ae054ade3b7196b Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 19:37:45 +0100 Subject: [PATCH 0709/1752] Expand ip char buffers which are too small for v6. Increase the size of character buffers being used to store printable IP addresses so that they can safely store IPv6 addresses. --- src/networking.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index fcaea13420f..3e37ae66bfa 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1124,7 +1124,7 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, /* Turn a Redis client into an sds string representing its state. */ sds getClientInfoString(redisClient *client) { - char ip[32], flags[16], events[3], *p; + char ip[REDIS_IP_STR_LEN], flags[16], events[3], *p; int port = 0; /* initialized to zero for the unix socket case. */ int emask; @@ -1206,7 +1206,8 @@ void clientCommand(redisClient *c) { } else if (!strcasecmp(c->argv[1]->ptr,"kill") && c->argc == 3) { listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - char ip[32], addr[64]; + /* addr size 64 > INET6_ADDRSTRLEN + : + strlen("65535") */ + char ip[INET6_ADDRSTRLEN], addr[64]; int port; client = listNodeValue(ln); From d37d006cd2fddb302f699028b365e17d20646247 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 19:43:47 +0100 Subject: [PATCH 0710/1752] Mark places that might want changing for IPv6. Any places which I feel might want to be updated to work differently with IPv6 have been marked with a comment starting "IPV6:". Currently the only comments address places where an IP address is combined with a port using the standard : separated form. These may want to be changed when printing IPv6 addresses to wrap the address in [] such as [2001:db8::c0:ffee]:6379 instead of 2001:db8::c0:ffee:6379 as the latter format is a technically valid IPv6 address and it is hard to distinguish the IPv6 address component from the port unless you know the port is supposed to be there. --- src/networking.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/networking.c b/src/networking.c index 3e37ae66bfa..b788bc7a6e5 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1212,6 +1212,7 @@ void clientCommand(redisClient *c) { client = listNodeValue(ln); if (anetPeerToString(client->fd,ip,sizeof(ip),&port) == -1) continue; + /* IPV6: might want to wrap a v6 address in [] */ snprintf(addr,sizeof(addr),"%s:%d",ip,port); if (strcmp(addr,c->argv[2]->ptr) == 0) { addReply(c,shared.ok); From 71795d4e7e72166756d72b0133126313a5a1c552 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Sat, 18 Jun 2011 19:54:05 +0100 Subject: [PATCH 0711/1752] Change anetTcpGenericConnect to use AF_UNSPEC. This allows anetTcpGenericConnect to try to connect to AF_INET6 addresses in addition to any resolved AF_INET addresses. --- src/anet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/anet.c b/src/anet.c index 1996a487ba3..f86c524936d 100644 --- a/src/anet.c +++ b/src/anet.c @@ -225,7 +225,7 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) snprintf(_port,6,"%d",port); memset(&hints,0,sizeof(hints)); - hints.ai_family = AF_INET; + hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { From f6382369022685f5c5063b3616104b406a7fe50c Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Mon, 19 Sep 2011 23:31:41 +0100 Subject: [PATCH 0712/1752] Add static anetV6Only() function. This function sets the IPV6_V6ONLY option to 1 to use separate stack IPv6 sockets. --- src/anet.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/anet.c b/src/anet.c index f86c524936d..bbefa2fe901 100644 --- a/src/anet.c +++ b/src/anet.c @@ -362,6 +362,16 @@ static int anetListen(char *err, int s, struct sockaddr *sa, socklen_t len) { return ANET_OK; } +static int anetV6Only(char *err, int s) { + int yes = 1; + if (setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,&yes,sizeof(yes)) == -1) { + anetSetError(err, "setsockopt: %s", strerror(errno)); + close(s); + return ANET_ERR; + } + return ANET_OK; +} + int anetTcpServer(char *err, int port, char *bindaddr) { int s, rv; From fd8a6ae7a6e2375a02fc292c89334ddf366b0b78 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Mon, 19 Sep 2011 23:32:41 +0100 Subject: [PATCH 0713/1752] Add anetTcp6Server() function. Refactor the common code from anetTcpServer into internal function which can be used by both anetTcpServer and anetTcp6Server. --- src/anet.c | 17 +++++++++++++++-- src/anet.h | 1 + 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/anet.c b/src/anet.c index bbefa2fe901..434d8311e74 100644 --- a/src/anet.c +++ b/src/anet.c @@ -372,7 +372,7 @@ static int anetV6Only(char *err, int s) { return ANET_OK; } -int anetTcpServer(char *err, int port, char *bindaddr) +static int _anetTcpServer(char *err, int port, char *bindaddr, int af) { int s, rv; char _port[6]; /* strlen("65535") */ @@ -380,7 +380,7 @@ int anetTcpServer(char *err, int port, char *bindaddr) snprintf(_port,6,"%d",port); memset(&hints,0,sizeof(hints)); - hints.ai_family = AF_INET; + hints.ai_family = af; hints.ai_socktype = SOCK_STREAM; hints.ai_flags = AI_PASSIVE; /* No effect if bindaddr != NULL */ @@ -392,6 +392,9 @@ int anetTcpServer(char *err, int port, char *bindaddr) if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) continue; + if (AF_INET6 == af && anetV6Only(err,s) == ANET_ERR) + goto error; /* could continue here? */ + if (anetListen(err,s,p->ai_addr,p->ai_addrlen) == ANET_ERR) goto error; /* could continue here? */ goto end; @@ -408,6 +411,16 @@ int anetTcpServer(char *err, int port, char *bindaddr) return s; } +int anetTcpServer(char *err, int port, char *bindaddr) +{ + return _anetTcpServer(err, port, bindaddr, AF_INET); +} + +int anetTcp6Server(char *err, int port, char *bindaddr) +{ + return _anetTcpServer(err, port, bindaddr, AF_INET6); +} + int anetUnixServer(char *err, char *path, mode_t perm) { int s; diff --git a/src/anet.h b/src/anet.h index ccc67c634c6..ff5897af105 100644 --- a/src/anet.h +++ b/src/anet.h @@ -46,6 +46,7 @@ int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetTcpServer(char *err, int port, char *bindaddr); +int anetTcp6Server(char *err, int port, char *bindaddr); int anetUnixServer(char *err, char *path, mode_t perm); int anetTcpAccept(char *err, int serversock, char *ip, size_t ip_len, int *port); int anetUnixAccept(char *err, int serversock); From 47620a31868638d3f5f23c633ece138b3a749a32 Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Mon, 19 Sep 2011 23:41:39 +0100 Subject: [PATCH 0714/1752] Document port6 and bind6 config options. Add commented port6 and bind6 options to default redis.conf file. --- redis.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/redis.conf b/redis.conf index 2e3239c8027..30823990565 100644 --- a/redis.conf +++ b/redis.conf @@ -23,6 +23,7 @@ pidfile /var/run/redis.pid # Accept connections on the specified port, default is 6379. # If port 0 is specified Redis will not listen on a TCP socket. port 6379 +# port6 6379 # By default Redis listens for connections from all the network interfaces # available on the server. It is possible to listen to just one or multiple @@ -33,6 +34,7 @@ port 6379 # # bind 192.168.1.100 10.0.0.1 # bind 127.0.0.1 +# bind6 ::1 # Specify the path for the unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen From d3bf85423a2330dcbb30250194aa5e886dcd45cf Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Thu, 7 Jun 2012 19:01:51 +0100 Subject: [PATCH 0715/1752] Fix calls to anetPeerToString() missing buffer size. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 971ebadb749..2f616564d4d 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2349,7 +2349,7 @@ sds genRedisInfoString(char *section) { while((ln = listNext(&li))) { redisClient *slave = listNodeValue(ln); char *state = NULL; - char ip[32]; + char ip[INET6_ADDRSTRLEN]; int port; long lag = 0; From 5938126c988b6b3bf378cbc5017862f0835e55fa Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Wed, 17 Oct 2012 23:26:30 +0100 Subject: [PATCH 0716/1752] Cleanup main() and BACKTRACE mistaken pulled while rebasing. --- src/redis.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis.c b/src/redis.c index 2f616564d4d..f453ce70fc9 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2882,6 +2882,8 @@ int main(int argc, char **argv) { loadDataFromDisk(); if (server.ipfd_count > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port); + if (server.ip6fd > 0) + redisLog(REDIS_NOTICE,"The server is now ready to accept IPv6 connections on port %d", server.port6); if (server.sofd > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); } From 0d8f254359605769da0d146fc2ac80eae1c2932f Mon Sep 17 00:00:00 2001 From: Geoff Garside Date: Wed, 17 Oct 2012 23:45:44 +0100 Subject: [PATCH 0717/1752] Add IPv6 support to sentinel.c. This has been done by exposing the anetSockName() function anet.c to be used when the sentinel is publishing its existence to the masters. This implementation is very unintelligent as it will likely break if used with IPv6 as the nested colons will break any parsing of the PUBLISH string by the master. --- src/anet.h | 1 + src/sentinel.c | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/anet.h b/src/anet.h index ff5897af105..b23411cbbe3 100644 --- a/src/anet.h +++ b/src/anet.h @@ -57,5 +57,6 @@ int anetDisableTcpNoDelay(char *err, int fd); int anetTcpKeepAlive(char *err, int fd); int anetPeerToString(int fd, char *ip, size_t ip_len, int *port); int anetKeepAlive(char *err, int fd, int interval); +int anetSockName(int fd, char *ip, size_t ip_len, int *port); #endif diff --git a/src/sentinel.c b/src/sentinel.c index cb70087272a..e89199feccb 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1837,14 +1837,13 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { (now - ri->last_pub_time) > SENTINEL_PUBLISH_PERIOD) { /* PUBLISH hello messages only to masters. */ - struct sockaddr_in sa; - socklen_t salen = sizeof(sa); - - if (getsockname(ri->cc->c.fd,(struct sockaddr*)&sa,&salen) != -1) { + char ip[INET6_ADDRSTRLEN]; + if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { char myaddr[128]; + // FIXME: IPv6 will break this due to nested : characters -geoffgarside snprintf(myaddr,sizeof(myaddr),"%s:%d:%s:%d", - inet_ntoa(sa.sin_addr), server.port, server.runid, + ip, server.port, server.runid, (ri->flags & SRI_CAN_FAILOVER) != 0); retval = redisAsyncCommand(ri->cc, sentinelPublishReplyCallback, NULL, "PUBLISH %s %s", From 1e7949227c27072d70360779dbe0eea27e867558 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 8 Jul 2013 16:08:43 +0200 Subject: [PATCH 0718/1752] Revert "Document port6 and bind6 config options." IPv6 support is not going to use IPv6 specific options, just it will be possible to specify all the ipv4 / ipv6 addresses of the interfaces to bind, otherwise connections will be accepted from all the interfaces in both IPv4 and IPv6 addresses. This reverts commit 93570e179e96dc096b85aa0fcd5021b05208594a. --- redis.conf | 2 -- 1 file changed, 2 deletions(-) diff --git a/redis.conf b/redis.conf index 30823990565..2e3239c8027 100644 --- a/redis.conf +++ b/redis.conf @@ -23,7 +23,6 @@ pidfile /var/run/redis.pid # Accept connections on the specified port, default is 6379. # If port 0 is specified Redis will not listen on a TCP socket. port 6379 -# port6 6379 # By default Redis listens for connections from all the network interfaces # available on the server. It is possible to listen to just one or multiple @@ -34,7 +33,6 @@ port 6379 # # bind 192.168.1.100 10.0.0.1 # bind 127.0.0.1 -# bind6 ::1 # Specify the path for the unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen From b5423b099cde78fad788d0af1563eafff4e3e4dc Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 8 Jul 2013 16:11:52 +0200 Subject: [PATCH 0719/1752] Fix old anetPeerToString() API call in replication.c --- src/redis.c | 2 -- src/replication.c | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index f453ce70fc9..2f616564d4d 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2882,8 +2882,6 @@ int main(int argc, char **argv) { loadDataFromDisk(); if (server.ipfd_count > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port); - if (server.ip6fd > 0) - redisLog(REDIS_NOTICE,"The server is now ready to accept IPv6 connections on port %d", server.port6); if (server.sofd > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); } diff --git a/src/replication.c b/src/replication.c index b2ceba193fc..4cd5710320b 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1594,10 +1594,10 @@ void replicationCron(void) { if (slave->flags & REDIS_PRE_PSYNC_SLAVE) continue; if ((server.unixtime - slave->repl_ack_time) > server.repl_timeout) { - char ip[32]; + char ip[REDIS_IP_STR_LEN]; int port; - if (anetPeerToString(slave->fd,ip,&port) != -1) { + if (anetPeerToString(slave->fd,ip,sizeof(ip),&port) != -1) { redisLog(REDIS_WARNING, "Disconnecting timedout slave: %s:%d", ip, slave->slave_listening_port); From 2e75d3947ce8c64bc66c32c6e20fe18c8e09f499 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 9 Jul 2013 10:47:17 +0200 Subject: [PATCH 0720/1752] IPv6: bind IPv4 and IPv6 interfaces by default. --- src/anet.c | 2 +- src/redis.c | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/anet.c b/src/anet.c index 434d8311e74..bf8c925491d 100644 --- a/src/anet.c +++ b/src/anet.c @@ -392,7 +392,7 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af) if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) continue; - if (AF_INET6 == af && anetV6Only(err,s) == ANET_ERR) + if (af == AF_INET6 && anetV6Only(err,s) == ANET_ERR) goto error; /* could continue here? */ if (anetListen(err,s,p->ai_addr,p->ai_addrlen) == ANET_ERR) diff --git a/src/redis.c b/src/redis.c index 2f616564d4d..0db41bcc9bb 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1391,7 +1391,18 @@ void initServer() { * entering the loop if j == 0. */ if (server.bindaddr_count == 0) server.bindaddr[0] = NULL; for (j = 0; j < server.bindaddr_count || j == 0; j++) { - server.ipfd[server.ipfd_count] = anetTcpServer(server.neterr,server.port,server.bindaddr[j]); + if (server.bindaddr[j] == NULL) { + /* Bind * for both IPv6 and IPv4. */ + server.ipfd[0] = anetTcp6Server(server.neterr,server.port,NULL); + if (server.ipfd[0] != ANET_ERR) server.ipfd_count++; + server.ipfd[1] = anetTcpServer(server.neterr,server.port,NULL); + } else if (strchr(server.bindaddr[j],':')) { + /* Bind IPv6 address. */ + server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,server.bindaddr[j]); + } else { + /* Bind IPv4 address. */ + server.ipfd[server.ipfd_count] = anetTcpServer(server.neterr,server.port,server.bindaddr[j]); + } if (server.ipfd[server.ipfd_count] == ANET_ERR) { redisLog(REDIS_WARNING, "Creating Server TCP listening socket %s:%d: %s", From a7451c1b6de7dd6d7437d511493f338862e1e68a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 9 Jul 2013 11:32:52 +0200 Subject: [PATCH 0721/1752] All IP string repr buffers are now REDIS_IP_STR_LEN bytes. --- src/networking.c | 5 ++--- src/redis.c | 2 +- src/sentinel.c | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/networking.c b/src/networking.c index b788bc7a6e5..19d12863915 100644 --- a/src/networking.c +++ b/src/networking.c @@ -552,7 +552,7 @@ static void acceptCommonHandler(int fd, int flags) { void acceptTcpHandler(aeEventLoop *el, int fd, void *privdata, int mask) { int cport, cfd; - char cip[128]; /* Could use INET6_ADDRSTRLEN here, but its smaller */ + char cip[REDIS_IP_STR_LEN]; REDIS_NOTUSED(el); REDIS_NOTUSED(mask); REDIS_NOTUSED(privdata); @@ -1206,8 +1206,7 @@ void clientCommand(redisClient *c) { } else if (!strcasecmp(c->argv[1]->ptr,"kill") && c->argc == 3) { listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - /* addr size 64 > INET6_ADDRSTRLEN + : + strlen("65535") */ - char ip[INET6_ADDRSTRLEN], addr[64]; + char ip[REDIS_IP_STR_LEN], addr[REDIS_IP_STR_LEN+64]; int port; client = listNodeValue(ln); diff --git a/src/redis.c b/src/redis.c index 0db41bcc9bb..7eea63d8369 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2360,7 +2360,7 @@ sds genRedisInfoString(char *section) { while((ln = listNext(&li))) { redisClient *slave = listNodeValue(ln); char *state = NULL; - char ip[INET6_ADDRSTRLEN]; + char ip[REDIS_IP_STR_LEN]; int port; long lag = 0; diff --git a/src/sentinel.c b/src/sentinel.c index e89199feccb..eb729966ad2 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1837,7 +1837,7 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { (now - ri->last_pub_time) > SENTINEL_PUBLISH_PERIOD) { /* PUBLISH hello messages only to masters. */ - char ip[INET6_ADDRSTRLEN]; + char ip[REDIS_IP_STR_LEN]; if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { char myaddr[128]; From 4fa68b281594e63a2c0fd2ba1a531ea8e80402ae Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 9 Jul 2013 12:49:20 +0200 Subject: [PATCH 0722/1752] getClientPeerID introduced. The function returns an unique identifier for the client, as ip:port for IPv4 and IPv6 clients, or as path:0 for Unix socket clients. See the top comment in the function for more info. --- src/networking.c | 41 ++++++++++++++++++++++++++++++++++------- src/redis.h | 2 ++ 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/src/networking.c b/src/networking.c index 19d12863915..295a45acd0b 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1122,14 +1122,42 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, *biggest_input_buffer = bib; } +/* A Redis "Peer ID" is a colon separated ip:port pair. + * For IPv4 it's in the form x.y.z.k:pork, example: "127.0.0.1:1234". + * For IPv6 addresses we use [] around the IP part, like in "[::1]:1234". + * For Unix socekts we use path:0, like in "/tmp/redis:0". + * + * A Peer ID always fits inside a buffer of REDIS_PEER_ID_LEN bytes, including + * the null term. + * + * The function is always successful, but if the IP or port can't be extracted + * for some reason, "?" and "0" are used (this is the semantics of + * anetPeerToString() from anet.c). In practical terms this should never + * happen. */ +void getClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { + char ip[REDIS_IP_STR_LEN]; + int port; + + if (client->flags & REDIS_UNIX_SOCKET) { + /* Unix socket client. */ + snprintf(peerid,peerid_len,"%s:0",server.unixsocket); + return; + } else { + /* TCP client. */ + anetPeerToString(client->fd,ip,sizeof(ip),&port); + if (strchr(ip,':')) + snprintf(peerid,peerid_len,"[%s]:%d",ip,port); + else + snprintf(peerid,peerid_len,"%s:%d",ip,port); + } +} + /* Turn a Redis client into an sds string representing its state. */ sds getClientInfoString(redisClient *client) { - char ip[REDIS_IP_STR_LEN], flags[16], events[3], *p; - int port = 0; /* initialized to zero for the unix socket case. */ + char peerid[REDIS_PEER_ID_LEN], flags[16], events[3], *p; int emask; - if (!(client->flags & REDIS_UNIX_SOCKET)) - anetPeerToString(client->fd,ip,sizeof(ip),&port); + getClientPeerId(client,peerid,sizeof(peerid)); p = flags; if (client->flags & REDIS_SLAVE) { if (client->flags & REDIS_MONITOR) @@ -1154,9 +1182,8 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatprintf(sdsempty(), - "addr=%s:%d fd=%d name=%s age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", - (client->flags & REDIS_UNIX_SOCKET) ? server.unixsocket : ip, - port, + "addr=%s fd=%d name=%s age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", + peerid, client->fd, client->name ? (char*)client->name->ptr : "", (long)(server.unixtime - client->ctime), diff --git a/src/redis.h b/src/redis.h index 2fb03441b20..e7a0f7fd89d 100644 --- a/src/redis.h +++ b/src/redis.h @@ -121,6 +121,7 @@ #define REDIS_DEFAULT_MIN_SLAVES_TO_WRITE 0 #define REDIS_DEFAULT_MIN_SLAVES_MAX_LAG 10 #define REDIS_IP_STR_LEN INET6_ADDRSTRLEN +#define REDIS_PEER_ID_LEN (REDIS_IP_STR_LEN+32) /* Must be enough for ip:port */ #define REDIS_BINDADDR_MAX 16 /* Protocol and I/O related defines */ @@ -908,6 +909,7 @@ void copyClientOutputBuffer(redisClient *dst, redisClient *src); void *dupClientReplyValue(void *o); void getClientsMaxBuffers(unsigned long *longest_output_list, unsigned long *biggest_input_buffer); +void getClientPeerId(redisClient *client, char *peerid, size_t peerid_len); sds getClientInfoString(redisClient *client); sds getAllClientsInfoString(void); void rewriteClientCommandVector(redisClient *c, int argc, ...); From da1836660903dbead2c952f5fd5f4ebd42c3789a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 9 Jul 2013 15:28:30 +0200 Subject: [PATCH 0723/1752] getClientPeerId() now reports errors. We now also use it in CLIENT KILL implementation. --- src/networking.c | 26 +++++++++++++------------- src/redis.h | 2 +- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/networking.c b/src/networking.c index 295a45acd0b..759abe7beca 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1130,25 +1130,27 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, * A Peer ID always fits inside a buffer of REDIS_PEER_ID_LEN bytes, including * the null term. * - * The function is always successful, but if the IP or port can't be extracted - * for some reason, "?" and "0" are used (this is the semantics of - * anetPeerToString() from anet.c). In practical terms this should never - * happen. */ -void getClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { + * The function returns REDIS_OK on succcess, and REDIS_ERR on failure. + * + * On failure the function still populates 'peerid' with the "?:0" string + * in case you want to relax error checking or need to display something + * anyway (see anetPeerToString implementation for more info). */ +int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { char ip[REDIS_IP_STR_LEN]; int port; if (client->flags & REDIS_UNIX_SOCKET) { /* Unix socket client. */ snprintf(peerid,peerid_len,"%s:0",server.unixsocket); - return; + return REDIS_OK; } else { /* TCP client. */ - anetPeerToString(client->fd,ip,sizeof(ip),&port); + int retval = anetPeerToString(client->fd,ip,sizeof(ip),&port); if (strchr(ip,':')) snprintf(peerid,peerid_len,"[%s]:%d",ip,port); else snprintf(peerid,peerid_len,"%s:%d",ip,port); + return (retval == -1) ? REDIS_ERR : REDIS_OK; } } @@ -1233,14 +1235,12 @@ void clientCommand(redisClient *c) { } else if (!strcasecmp(c->argv[1]->ptr,"kill") && c->argc == 3) { listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - char ip[REDIS_IP_STR_LEN], addr[REDIS_IP_STR_LEN+64]; - int port; + char peerid[REDIS_PEER_ID_LEN]; client = listNodeValue(ln); - if (anetPeerToString(client->fd,ip,sizeof(ip),&port) == -1) continue; - /* IPV6: might want to wrap a v6 address in [] */ - snprintf(addr,sizeof(addr),"%s:%d",ip,port); - if (strcmp(addr,c->argv[2]->ptr) == 0) { + if (getClientPeerId(client,peerid,sizeof(peerid)) == REDIS_ERR) + continue; + if (strcmp(peerid,c->argv[2]->ptr) == 0) { addReply(c,shared.ok); if (c == client) { client->flags |= REDIS_CLOSE_AFTER_REPLY; diff --git a/src/redis.h b/src/redis.h index e7a0f7fd89d..48c966956fa 100644 --- a/src/redis.h +++ b/src/redis.h @@ -909,7 +909,7 @@ void copyClientOutputBuffer(redisClient *dst, redisClient *src); void *dupClientReplyValue(void *o); void getClientsMaxBuffers(unsigned long *longest_output_list, unsigned long *biggest_input_buffer); -void getClientPeerId(redisClient *client, char *peerid, size_t peerid_len); +int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len); sds getClientInfoString(redisClient *client); sds getAllClientsInfoString(void); void rewriteClientCommandVector(redisClient *c, int argc, ...); From 3472d045d8bf983e9e017bcf00146ee8bbe5df35 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 9 Jul 2013 15:46:34 +0200 Subject: [PATCH 0724/1752] getClientPeerId() refactored into two functions. --- src/networking.c | 16 ++++++++++++---- src/redis.h | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/networking.c b/src/networking.c index 759abe7beca..84895262696 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1122,6 +1122,17 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, *biggest_input_buffer = bib; } +/* This is an helper function for getClientPeerId(). + * It writes the specified ip/port to "peerid" as a null termiated string + * in the form ip:port if ip does not contain ":" itself, otherwise + * [ip]:port format is used (for IPv6 addresses basically). */ +void formatPeerId(char *peerid, size_t peerid_len, char *ip, int port) { + if (strchr(ip,':')) + snprintf(peerid,peerid_len,"[%s]:%d",ip,port); + else + snprintf(peerid,peerid_len,"%s:%d",ip,port); +} + /* A Redis "Peer ID" is a colon separated ip:port pair. * For IPv4 it's in the form x.y.z.k:pork, example: "127.0.0.1:1234". * For IPv6 addresses we use [] around the IP part, like in "[::1]:1234". @@ -1146,10 +1157,7 @@ int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { } else { /* TCP client. */ int retval = anetPeerToString(client->fd,ip,sizeof(ip),&port); - if (strchr(ip,':')) - snprintf(peerid,peerid_len,"[%s]:%d",ip,port); - else - snprintf(peerid,peerid_len,"%s:%d",ip,port); + formatPeerId(peerid,peerid_len,ip,port); return (retval == -1) ? REDIS_ERR : REDIS_OK; } } diff --git a/src/redis.h b/src/redis.h index 48c966956fa..2fbd930bcda 100644 --- a/src/redis.h +++ b/src/redis.h @@ -909,6 +909,7 @@ void copyClientOutputBuffer(redisClient *dst, redisClient *src); void *dupClientReplyValue(void *o); void getClientsMaxBuffers(unsigned long *longest_output_list, unsigned long *biggest_input_buffer); +void formatPeerId(char *peerid, size_t peerid_len, char *ip, int port); int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len); sds getClientInfoString(redisClient *client); sds getAllClientsInfoString(void); From cdf801d92ef16dacd109f2e58f5c857ee424f0bf Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 9 Jul 2013 16:21:21 +0200 Subject: [PATCH 0725/1752] Use getClientPeerId() for MONITOR implementation. --- src/replication.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/replication.c b/src/replication.c index 4cd5710320b..9f79bb2293f 100644 --- a/src/replication.c +++ b/src/replication.c @@ -290,10 +290,10 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj **argv, int argc) { listNode *ln; listIter li; - int j, port; + int j; sds cmdrepr = sdsnew("+"); robj *cmdobj; - char ip[REDIS_IP_STR_LEN]; + char peerid[REDIS_PEER_ID_LEN]; struct timeval tv; gettimeofday(&tv,NULL); @@ -303,8 +303,8 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** } else if (c->flags & REDIS_UNIX_SOCKET) { cmdrepr = sdscatprintf(cmdrepr,"[%d unix:%s] ",dictid,server.unixsocket); } else { - anetPeerToString(c->fd,ip,sizeof(ip),&port); - cmdrepr = sdscatprintf(cmdrepr,"[%d %s:%d] ",dictid,ip,port); + getClientPeerId(c,peerid,sizeof(peerid)); + cmdrepr = sdscatprintf(cmdrepr,"[%d %s] ",dictid,peerid); } for (j = 0; j < argc; j++) { From 9e089e7c5dbfd4ce94f59d3213fefa31a8a6228c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 10 Jul 2013 14:34:58 +0200 Subject: [PATCH 0726/1752] anet.c: use SO_REUSEADDR when creating listening sockets. It used to be ok, but the socket option was removed when adding IPv6 support. --- src/anet.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/anet.c b/src/anet.c index bf8c925491d..3726b9f6747 100644 --- a/src/anet.c +++ b/src/anet.c @@ -392,11 +392,9 @@ static int _anetTcpServer(char *err, int port, char *bindaddr, int af) if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) continue; - if (af == AF_INET6 && anetV6Only(err,s) == ANET_ERR) - goto error; /* could continue here? */ - - if (anetListen(err,s,p->ai_addr,p->ai_addrlen) == ANET_ERR) - goto error; /* could continue here? */ + if (af == AF_INET6 && anetV6Only(err,s) == ANET_ERR) goto error; + if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; + if (anetListen(err,s,p->ai_addr,p->ai_addrlen) == ANET_ERR) goto error; goto end; } if (p == NULL) { From 8669e70921cf1734db435df51011595973271f1f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 10 Jul 2013 14:37:13 +0200 Subject: [PATCH 0727/1752] anet.c: save some vertical space. --- src/anet.c | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/anet.c b/src/anet.c index 3726b9f6747..257b491e915 100644 --- a/src/anet.c +++ b/src/anet.c @@ -237,18 +237,11 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) continue; /* if we set err then goto cleanup, otherwise next */ - if (anetSetReuseAddr(err,s) == ANET_ERR) { + if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; + if (flags & ANET_CONNECT_NONBLOCK && anetNonBlock(err,s) != ANET_OK) goto error; - } - if (flags & ANET_CONNECT_NONBLOCK) { - if (anetNonBlock(err,s) != ANET_OK) - goto error; - } if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { - if (errno == EINPROGRESS && - flags & ANET_CONNECT_NONBLOCK) - goto end; - + if (errno == EINPROGRESS && flags & ANET_CONNECT_NONBLOCK) goto end; close(s); continue; } From 98d0abcecdf50a15a8aac31c3c9fd995091e6169 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 10 Jul 2013 14:44:38 +0200 Subject: [PATCH 0728/1752] Sentinel: make sure published addr/id buffer is large enough. With ipv6 support we need more space, so we account for the IP address max size plus what we need for the Run ID, port, flags. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index eb729966ad2..594b7e4989b 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1839,7 +1839,7 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { /* PUBLISH hello messages only to masters. */ char ip[REDIS_IP_STR_LEN]; if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { - char myaddr[128]; + char myaddr[REDIS_IP_STR_LEN+128]; // FIXME: IPv6 will break this due to nested : characters -geoffgarside snprintf(myaddr,sizeof(myaddr),"%s:%d:%s:%d", From 4cfd70e90368c4dc3eff511acaca6db4b4f24077 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jul 2013 13:11:27 +0200 Subject: [PATCH 0729/1752] hiredis: minimal IPv6 support. --- deps/hiredis/net.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deps/hiredis/net.c b/deps/hiredis/net.c index 82ab2b468c8..b10eee2e527 100644 --- a/deps/hiredis/net.c +++ b/deps/hiredis/net.c @@ -215,9 +215,17 @@ int redisContextConnectTcp(redisContext *c, const char *addr, int port, struct t hints.ai_family = AF_INET; hints.ai_socktype = SOCK_STREAM; + /* Try with IPv6 if no IPv4 address was found. We do it in this order since + * in a Redis client you can't afford to test if you have IPv6 connectivity + * as this would add latency to every connect. Otherwise a more sensible + * route could be: Use IPv6 if both addresses are available and there is IPv6 + * connectivity. */ if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { - __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); - return REDIS_ERR; + hints.ai_family = AF_INET6; + if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { + __redisSetError(c,REDIS_ERR_OTHER,gai_strerror(rv)); + return REDIS_ERR; + } } for (p = servinfo; p != NULL; p = p->ai_next) { if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) From 076f6395b984d446e2d4b3b47e2f928371144a1a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jul 2013 16:31:39 +0200 Subject: [PATCH 0730/1752] Sentinel: use comma as separator to publish hello messages. We use comma to play well with IPv6 addresses, but the implementation is still able to parse the old messages separated by colons. --- src/sentinel.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 594b7e4989b..62473b5c151 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1741,9 +1741,13 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd { int numtokens, port, removed, canfailover; + /* Separator changed from ":" to "," in recent versions in order to + * play well with IPv6 addresses. For now we make sure to parse both + * correctly detecting if there is "," inside the string. */ + char *sep = strchr(r->element[2]->str,',') ? "," : ":"; char **token = sdssplitlen(r->element[2]->str, r->element[2]->len, - ":",1,&numtokens); + sep,1,&numtokens); sentinelRedisInstance *sentinel; if (numtokens == 4) { @@ -1841,8 +1845,7 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { char myaddr[REDIS_IP_STR_LEN+128]; - // FIXME: IPv6 will break this due to nested : characters -geoffgarside - snprintf(myaddr,sizeof(myaddr),"%s:%d:%s:%d", + snprintf(myaddr,sizeof(myaddr),"%s,%d,%s,%d", ip, server.port, server.runid, (ri->flags & SRI_CAN_FAILOVER) != 0); retval = redisAsyncCommand(ri->cc, From 1e23848ed357f68bfb3dc60937f605cd147cb5f2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 11 Jul 2013 16:38:30 +0200 Subject: [PATCH 0731/1752] Sentinel: embed IPv6 address into [] when naming slave/sentinel instance. --- src/sentinel.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 62473b5c151..659de29b8be 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -835,7 +835,9 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * /* For slaves and sentinel we use ip:port as name. */ if (flags & (SRI_SLAVE|SRI_SENTINEL)) { - snprintf(slavename,sizeof(slavename),"%s:%d",hostname,port); + snprintf(slavename,sizeof(slavename), + strchr(hostname,':') ? "[%s]:%d" : "%s:%d", + hostname,port); name = slavename; } @@ -943,7 +945,9 @@ sentinelRedisInstance *sentinelRedisInstanceLookupSlave( sentinelRedisInstance *slave; redisAssert(ri->flags & SRI_MASTER); - key = sdscatprintf(sdsempty(),"%s:%d",ip,port); + key = sdscatprintf(sdsempty(), + strchr(ip,':') ? "[%s]:%d" : "%s:%d", + ip,port); slave = dictFetchValue(ri->slaves,key); sdsfree(key); return slave; From 1bcbb7a90c7a32da41fa9430f1c20c46ba77e43d Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Thu, 11 Jul 2013 17:47:55 +0200 Subject: [PATCH 0732/1752] Wrap IPv6 in brackets in the prompt. --- src/redis-cli.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index c9eba1b5914..f14b4788239 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -116,7 +116,8 @@ static void cliRefreshPrompt(void) { len = snprintf(config.prompt,sizeof(config.prompt),"redis %s", config.hostsocket); else - len = snprintf(config.prompt,sizeof(config.prompt),"redis %s:%d", + len = snprintf(config.prompt,sizeof(config.prompt), + strchr(config.hostip,':') ? "[%s]:%d" : "%s:%d", config.hostip, config.hostport); /* Add [dbnum] if needed */ if (config.dbnum != 0) From d8fcbb664575d9ad00c853db5389c4c3a2fffbc2 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 12 Jul 2013 11:56:52 +0200 Subject: [PATCH 0733/1752] Fixed compareStringObject() and introduced collateStringObject(). compareStringObject was not always giving the same result when comparing two exact strings, but encoded as integers or as sds strings, since it switched to strcmp() when at least one of the strings were not sds encoded. For instance the two strings "123" and "123\x00456", where the first string was integer encoded, would result into the old implementation of compareStringObject() to return 0 as if the strings were equal, while instead the second string is "greater" than the first in a binary comparison. The same compasion, but with "123" encoded as sds string, would instead return a value < 0, as it is correct. It is not impossible that the above caused some obscure bug, since the comparison was not always deterministic, and compareStringObject() is used in the implementation of skiplists, hash tables, and so forth. At the same time, collateStringObject() was introduced by this commit, so that can be used by SORT command to return sorted strings usign collation instead of binary comparison. See next commit. --- src/object.c | 41 +++++++++++++++++++++++++++++++++-------- src/redis.h | 1 + 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/src/object.c b/src/object.c index 2554656a332..58668da5bf0 100644 --- a/src/object.c +++ b/src/object.c @@ -332,35 +332,60 @@ robj *getDecodedObject(robj *o) { } } -/* Compare two string objects via strcmp() or alike. +/* Compare two string objects via strcmp() or strcoll() depending on flags. * Note that the objects may be integer-encoded. In such a case we * use ll2string() to get a string representation of the numbers on the stack * and compare the strings, it's much faster than calling getDecodedObject(). * - * Important note: if objects are not integer encoded, but binary-safe strings, - * sdscmp() from sds.c will apply memcmp() so this function ca be considered - * binary safe. */ -int compareStringObjects(robj *a, robj *b) { + * Important note: when REDIS_COMPARE_BINARY is used a binary-safe comparison + * is used. */ + +#define REDIS_COMPARE_BINARY (1<<0) +#define REDIS_COMPARE_COLL (1<<1) + +int compareStringObjectsWithFlags(robj *a, robj *b, int flags) { redisAssertWithInfo(NULL,a,a->type == REDIS_STRING && b->type == REDIS_STRING); char bufa[128], bufb[128], *astr, *bstr; + size_t alen, blen, minlen; int bothsds = 1; if (a == b) return 0; if (a->encoding != REDIS_ENCODING_RAW) { - ll2string(bufa,sizeof(bufa),(long) a->ptr); + alen = ll2string(bufa,sizeof(bufa),(long) a->ptr); astr = bufa; bothsds = 0; } else { astr = a->ptr; + alen = sdslen(astr); } if (b->encoding != REDIS_ENCODING_RAW) { - ll2string(bufb,sizeof(bufb),(long) b->ptr); + blen = ll2string(bufb,sizeof(bufb),(long) b->ptr); bstr = bufb; bothsds = 0; } else { bstr = b->ptr; + blen = sdslen(bstr); + } + if (flags & REDIS_COMPARE_COLL) { + return strcoll(astr,bstr); + } else { + int cmp; + + minlen = (alen < blen) ? alen : blen; + cmp = memcmp(astr,bstr,minlen); + if (cmp == 0) return alen-blen; + return cmp; } - return bothsds ? sdscmp(astr,bstr) : strcmp(astr,bstr); +} + +/* Wrapper for compareStringObjectsWithFlags() using binary comparison. */ +int compareStringObjects(robj *a, robj *b) { + return compareStringObjectsWithFlags(a,b,REDIS_COMPARE_BINARY); +} + +/* Wrapper for compareStringObjectsWithFlags() using collation. */ +int collateStringObjects(robj *a, robj *b) { + return compareStringObjectsWithFlags(a,b,REDIS_COMPARE_COLL); } /* Equal string objects return 1 if the two objects are the same from the diff --git a/src/redis.h b/src/redis.h index 2fbd930bcda..043d64c0121 100644 --- a/src/redis.h +++ b/src/redis.h @@ -995,6 +995,7 @@ int getLongDoubleFromObject(robj *o, long double *target); int getLongDoubleFromObjectOrReply(redisClient *c, robj *o, long double *target, const char *msg); char *strEncoding(int encoding); int compareStringObjects(robj *a, robj *b); +int collateStringObjects(robj *a, robj *b); int equalStringObjects(robj *a, robj *b); unsigned long estimateObjectIdleTime(robj *o); From 18fabeb264193cab98869e514d771502d3456d60 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 12 Jul 2013 12:02:36 +0200 Subject: [PATCH 0734/1752] SORT ALPHA: use collation instead of binary comparison. Note that we only do it when STORE is not used, otherwise we want an absolutely locale independent and binary safe sorting in order to ensure AOF / replication consistency. This is probably an unexpected behavior violating the least surprise rule, but there is currently no other simple / good alternative. --- src/redis.h | 1 + src/sort.c | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/src/redis.h b/src/redis.h index 043d64c0121..5860d2aaa34 100644 --- a/src/redis.h +++ b/src/redis.h @@ -734,6 +734,7 @@ struct redisServer { int sort_desc; int sort_alpha; int sort_bypattern; + int sort_store; /* Zip structure config, see redis.conf for more information */ size_t hash_max_ziplist_entries; size_t hash_max_ziplist_value; diff --git a/src/sort.c b/src/sort.c index 4b5040250c6..a4b062645fa 100644 --- a/src/sort.c +++ b/src/sort.c @@ -163,12 +163,22 @@ int sortCompare(const void *s1, const void *s2) { else cmp = 1; } else { - /* We have both the objects, use strcoll */ - cmp = strcoll(so1->u.cmpobj->ptr,so2->u.cmpobj->ptr); + /* We have both the objects, compare them. */ + if (server.sort_store) { + cmp = compareStringObjects(so1->u.cmpobj,so2->u.cmpobj); + } else { + /* Here we can use strcoll() directly as we are sure that + * the objects are decoded string objects. */ + cmp = strcoll(so1->u.cmpobj->ptr,so2->u.cmpobj->ptr); + } } } else { /* Compare elements directly. */ - cmp = compareStringObjects(so1->obj,so2->obj); + if (server.sort_store) { + cmp = compareStringObjects(so1->obj,so2->obj); + } else { + cmp = collateStringObjects(so1->obj,so2->obj); + } } } return server.sort_desc ? -cmp : cmp; @@ -432,6 +442,7 @@ void sortCommand(redisClient *c) { server.sort_desc = desc; server.sort_alpha = alpha; server.sort_bypattern = sortby ? 1 : 0; + server.sort_store = storekey ? 1 : 0; if (sortby && (start != 0 || end != vectorlen-1)) pqsort(vector,vectorlen,sizeof(redisSortObject),sortCompare, start,end); else From 98757b4a6e8c8f98dd0739fc66ad595b2050eed8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 12 Jul 2013 12:06:05 +0200 Subject: [PATCH 0735/1752] Use the environment locale for strcoll() collation. --- src/redis.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/redis.c b/src/redis.c index 7eea63d8369..5d05f24068f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -49,6 +49,7 @@ #include #include #include +#include /* Our shared "common" objects */ @@ -2814,6 +2815,7 @@ int main(int argc, char **argv) { #ifdef INIT_SETPROCTITLE_REPLACEMENT spt_init(argc, argv); #endif + setlocale(LC_COLLATE,""); zmalloc_enable_thread_safeness(); zmalloc_set_oom_handler(redisOutOfMemoryHandler); srand(time(NULL)^getpid()); From fe04710908940da676559ee7c6dd363bf0ce6f85 Mon Sep 17 00:00:00 2001 From: Ted Nyman Date: Fri, 12 Jul 2013 14:06:27 -0700 Subject: [PATCH 0736/1752] Make sure the log standardizes on 'timeout' --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 9f79bb2293f..d44e06ec826 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1534,7 +1534,7 @@ void replicationCron(void) { if (server.masterhost && server.repl_state == REDIS_REPL_CONNECTED && (time(NULL)-server.master->lastinteraction) > server.repl_timeout) { - redisLog(REDIS_WARNING,"MASTER time out: no data nor PING received..."); + redisLog(REDIS_WARNING,"MASTER timeout: no data nor PING received..."); freeClient(server.master); } From 112e76361872fa2a6ee145b90afd153d198f88e4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 16 Jul 2013 15:05:13 +0200 Subject: [PATCH 0737/1752] Make sure that ZADD can accept the full range of double values. This fixes issue #1194, that contains many details. However in short, it was possible for ZADD to not accept as score values that was however possible to obtain with multiple calls to ZINCRBY, like in the following example: redis 127.0.0.1:6379> zadd k 2.5e-308 m (integer) 1 redis 127.0.0.1:6379> zincrby k -2.4e-308 m "9.9999999999999694e-310" redis 127.0.0.1:6379> zscore k m "9.9999999999999694e-310" redis 127.0.0.1:6379> zadd k 9.9999999999999694e-310 m1 (error) ERR value is not a valid float The problem was due to strtod() returning ERANGE in the following case specified by POSIX: "If the correct value would cause an underflow, a value whose magnitude is no greater than the smallest normalized positive number in the return type shall be returned and errno set to [ERANGE].". Now instead the returned value is accepted even when ERANGE is returned as long as the return value of the function is not negative or positive HUGE_VAL or zero. --- src/object.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/object.c b/src/object.c index 58668da5bf0..472d4a34bbb 100644 --- a/src/object.c +++ b/src/object.c @@ -422,8 +422,12 @@ int getDoubleFromObject(robj *o, double *target) { if (o->encoding == REDIS_ENCODING_RAW) { errno = 0; value = strtod(o->ptr, &eptr); - if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || - errno == ERANGE || isnan(value)) + if (isspace(((char*)o->ptr)[0]) || + eptr[0] != '\0' || + (errno == ERANGE && + (value == HUGE_VAL || value == -HUGE_VAL || value == 0)) || + errno == EINVAL || + isnan(value)) return REDIS_ERR; } else if (o->encoding == REDIS_ENCODING_INT) { value = (long)o->ptr; From 9f6f436a51570fa989ca52d250e592547214596c Mon Sep 17 00:00:00 2001 From: yoav Date: Wed, 12 Dec 2012 15:59:22 +0200 Subject: [PATCH 0738/1752] Chunked loading of RDB to prevent redis from stalling reading very large keys. --- src/rdb.c | 23 ++++++++++++++--------- src/redis.c | 1 + src/redis.h | 1 + src/rio.c | 4 ++++ src/rio.h | 31 +++++++++++++++++++++++++------ 5 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index e27efd15aeb..2581c007496 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1057,21 +1057,32 @@ void stopLoading(void) { server.loading = 0; } +/* Track loading progress in order to serve client's from time to time + and if needed calculate rdb checksum */ +void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { + if (server.rdb_checksum) + rioGenericUpdateChecksum(r, buf, len); + if (server.loading_process_events_interval_bytes && + (r->processed_bytes + len)/server.loading_process_events_interval_bytes > r->processed_bytes/server.loading_process_events_interval_bytes) { + loadingProgress(r->processed_bytes); + aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); + } +} + int rdbLoad(char *filename) { uint32_t dbid; int type, rdbver; redisDb *db = server.db+0; char buf[1024]; long long expiretime, now = mstime(); - long loops = 0; FILE *fp; rio rdb; if ((fp = fopen(filename,"r")) == NULL) return REDIS_ERR; rioInitWithFile(&rdb,fp); - if (server.rdb_checksum) - rdb.update_cksum = rioGenericUpdateChecksum; + rdb.update_cksum = rdbLoadProgressCallback; + rdb.max_processing_chunk = server.loading_process_events_interval_bytes; if (rioRead(&rdb,buf,9) == 0) goto eoferr; buf[9] = '\0'; if (memcmp(buf,"REDIS",5) != 0) { @@ -1093,12 +1104,6 @@ int rdbLoad(char *filename) { robj *key, *val; expiretime = -1; - /* Serve the clients from time to time */ - if (!(loops++ % 1000)) { - loadingProgress(rioTell(&rdb)); - aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); - } - /* Read type. */ if ((type = rdbLoadType(&rdb)) == -1) goto eoferr; if (type == REDIS_RDB_OPCODE_EXPIRETIME) { diff --git a/src/redis.c b/src/redis.c index 5d05f24068f..791e77de9a5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1250,6 +1250,7 @@ void initServerConfig() { server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; server.lua_timedout = 0; + server.loading_process_events_interval_bytes = (1024*1024*2); updateLRUClock(); resetServerSaveParams(); diff --git a/src/redis.h b/src/redis.h index 5860d2aaa34..5fd7f0ec808 100644 --- a/src/redis.h +++ b/src/redis.h @@ -596,6 +596,7 @@ struct redisServer { off_t loading_total_bytes; off_t loading_loaded_bytes; time_t loading_start_time; + off_t loading_process_events_interval_bytes; /* Fast pointers to often looked up command */ struct redisCommand *delCommand, *multiCommand, *lpushCommand, *lpopCommand, *rpopCommand; diff --git a/src/rio.c b/src/rio.c index b2f46a08bc1..405e789e6ea 100644 --- a/src/rio.c +++ b/src/rio.c @@ -108,6 +108,8 @@ static const rio rioBufferIO = { rioBufferTell, NULL, /* update_checksum */ 0, /* current checksum */ + 0, /* bytes read or written */ + 0, /* read/write chunk size */ { { NULL, 0 } } /* union for io-specific vars */ }; @@ -117,6 +119,8 @@ static const rio rioFileIO = { rioFileTell, NULL, /* update_checksum */ 0, /* current checksum */ + 0, /* bytes read or written */ + 0, /* read/write chunk size */ { { NULL, 0 } } /* union for io-specific vars */ }; diff --git a/src/rio.h b/src/rio.h index 3cab66af0f9..c28b47dc44e 100644 --- a/src/rio.h +++ b/src/rio.h @@ -53,6 +53,12 @@ struct _rio { /* The current checksum */ uint64_t cksum; + /* number of bytes read or written */ + size_t processed_bytes; + + /* maximum simgle read or write chunk size */ + size_t max_processing_chunk; + /* Backend-specific vars. */ union { struct { @@ -74,16 +80,29 @@ typedef struct _rio rio; * if needed. */ static inline size_t rioWrite(rio *r, const void *buf, size_t len) { - if (r->update_cksum) r->update_cksum(r,buf,len); - return r->write(r,buf,len); + while (len) { + size_t bytes_to_write = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; + if (r->update_cksum) r->update_cksum(r,buf,bytes_to_write); + if (r->write(r,buf,bytes_to_write) == 0) + return 0; + buf = (char*)buf + bytes_to_write; + len -= bytes_to_write; + r->processed_bytes += bytes_to_write; + } + return 1; } static inline size_t rioRead(rio *r, void *buf, size_t len) { - if (r->read(r,buf,len) == 1) { - if (r->update_cksum) r->update_cksum(r,buf,len); - return 1; + while (len) { + size_t bytes_to_read = (r->max_processing_chunk && r->max_processing_chunk < len) ? r->max_processing_chunk : len; + if (r->read(r,buf,bytes_to_read) == 0) + return 0; + if (r->update_cksum) r->update_cksum(r,buf,bytes_to_read); + buf = (char*)buf + bytes_to_read; + len -= bytes_to_read; + r->processed_bytes += bytes_to_read; } - return 0; + return 1; } static inline off_t rioTell(rio *r) { From 2e449d42e0f4dda4d2e71f04a2973b21f8fb6e8c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 16 Jul 2013 15:43:36 +0200 Subject: [PATCH 0739/1752] Fixed typo in rio.h, simgle -> single. --- src/rio.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rio.h b/src/rio.h index c28b47dc44e..2d12c6cc775 100644 --- a/src/rio.h +++ b/src/rio.h @@ -56,7 +56,7 @@ struct _rio { /* number of bytes read or written */ size_t processed_bytes; - /* maximum simgle read or write chunk size */ + /* maximum single read or write chunk size */ size_t max_processing_chunk; /* Backend-specific vars. */ From a50635bb6c6cce99885418b8752b4939cd4fd62e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 17 Jul 2013 15:04:22 +0200 Subject: [PATCH 0740/1752] addReplyDouble(): format infinite in a libc agnostic way. There are systems that when printing +/- infinte with printf-family functions will not use the usual "inf" "-inf", but different strings. Handle that explicitly. Fixes issue #930. --- src/networking.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/networking.c b/src/networking.c index 84895262696..23ada4541de 100644 --- a/src/networking.c +++ b/src/networking.c @@ -29,6 +29,7 @@ #include "redis.h" #include +#include static void setProtocolError(redisClient *c, int pos); @@ -415,9 +416,15 @@ void setDeferredMultiBulkLength(redisClient *c, void *node, long length) { void addReplyDouble(redisClient *c, double d) { char dbuf[128], sbuf[128]; int dlen, slen; - dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); - slen = snprintf(sbuf,sizeof(sbuf),"$%d\r\n%s\r\n",dlen,dbuf); - addReplyString(c,sbuf,slen); + if (isinf(d)) { + /* Libc in odd systems (Hi Solaris!) will format infinite in a + * different way, so better to handle it in an explicit way. */ + addReplyBulkCString(c, d > 0 ? "inf" : "-inf"); + } else { + dlen = snprintf(dbuf,sizeof(dbuf),"%.17g",d); + slen = snprintf(sbuf,sizeof(sbuf),"$%d\r\n%s\r\n",dlen,dbuf); + addReplyString(c,sbuf,slen); + } } /* Add a long long as integer reply or bulk len / multi bulk count. From 3cb3714e999812fce413b67fbaf760c65841273e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 18 Jul 2013 11:26:53 +0200 Subject: [PATCH 0741/1752] Redis 2.7.101 (2.8 Release Candidate 1). --- 00-RELEASENOTES | 90 +++- Changelog | 1032 --------------------------------------------- src/config.c | 34 +- src/redis.c | 2 +- src/replication.c | 4 +- src/version.h | 2 +- 6 files changed, 94 insertions(+), 1070 deletions(-) delete mode 100644 Changelog diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 8910ad91854..77a9c965167 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -1,6 +1,90 @@ -Redis 2.7 release notes +Redis 2.8 release notes ======================= -This is the unstable version of Redis 2.8, currently work in progress. +** IMPORTANT ** Check the 'Migrating from 2.6 to 2.8' section at the end of + this file for information about what changed between 2.4 and + 2.6 and how this may affect your application. -When it will be ready for production, it will be releaed as Redis 2.8.0. +-------------------------------------------------------------------------------- +Upgrade urgency levels: + +LOW: No need to upgrade unless there are new features you want to use. +MODERATE: Program an upgrade of the server, but it's not urgent. +HIGH: There is a critical bug that may affect a subset of users. Upgrade! +CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. +-------------------------------------------------------------------------------- + +--[ Redis 2.8 Release Candidate 1 (2.7.101) ] Release date: 18 Jul 2013 + +This is the first release candidate of Redis 2.8 (official version is 2.7.101). + +The following is a list of improvements in Redis 2.8, compared to Redis 2.6. + +* [NEW] Slaves are now able to partially resynchronize with the master, so most + of the times a full resynchornization with the RDB creation in the master + side is not needed when the master-slave link is disconnected for a short + amount of time. +* [NEW] Experimental IPv6 support.4 +* [NEW] Slaves explicitly ping masters now, a master is able to detect a timedout + slave independently. +* [NEW] Masters can stop accepting writes if not enough slaves with a given + maxium latency are connected. +* [NEW] Keyspace changes notifications via Pub/Sub. +* [NEW] CONFIG SET maxclients is now available. +* [NEW] Ability to bind multiple IP addresses. +* [NEW] Set process names so that you can recognize, in the "ps" command output, + the listening port of an instance, or if it is a saving child. +* [NEW] Automatic memory check on crash. +* [NEW] CONFIG REWRITE is able to materialize the changes in the configuration + operated using CONFIG SET into the redis.conf file. +* [NEW] More NetBSD friendly code base. +* [NEW] PUBSUB command for Pub/Sub introspection capabilities. +* [NEW] EVALSHA can now be replicated as such, without requiring to be expanded + to a full EVAL for the replication link. +* [NEW] Better Lua scripts error reporting. +* [NEW] SDIFF performances improved. +* [FIX] A number of bugfixes. + +Migrating from 2.6 to 2.8 +========================= + +Redis 2.6 is mostly a strict subset of 2.8. However there are a few things +that you should be aware of: + +The following commands changed behavior: + + * SORT with ALPHA now sort according to local collation locale if no STORE + option is used. + * ZADD/ZINCRBY are now able to accept a bigger range of values as valid + scores, that is, all the values you may end having as a result of + calling ZINCRBY multiple times. + * Many errors are now prefixed by a more specific error code instead of + the generic -ERR, for example -WRONGTYPE, -NOAUTH, ... + * PUBLISH called inside lua scripts is now correctly propagated to slaves. + +The following redis.conf and CONFIG GET / SET parameters changed: + + * logfile now uses the empty string in order to log to standard output, + so 'logfile stdout' is now invalid, use 'logfile ""' instead. + +The following INFO fields changed format in a non-backward compatible way: + + * The list of slaves in INFO is now in field=value format. + +Replication: + + Redis 2.8 can be used as slave for Redis 2.6, but doing this is only + a good idea for a short amonut of time needed to upgrade your servers. + We suggest to update both master and slaves about at the same time. + +-------------------------------------------------------------------------------- + +Credits: Where not specified the implementation and design is done by +Salvatore Sanfilippo. Thanks to VMware and Pivotal for making all +this possible. Also many thanks to all the other contributors and the amazing +community we have. + +See commit messages for more credits. + +Cheers, +Salvatore diff --git a/Changelog b/Changelog deleted file mode 100644 index f72746663d6..00000000000 --- a/Changelog +++ /dev/null @@ -1,1032 +0,0 @@ -2010-07-01 gitignore modified (antirez) -2010-06-22 redis.c split into many different C files. (antirez) -2010-06-16 more pub/sub tests (Pieter Noordhuis) -2010-06-15 initial basic pub/sub tests (Pieter Noordhuis) -2010-06-15 fix BLPOP/BRPOP to use the wrapped function for list length (Pieter Noordhuis) -2010-06-15 tests for BLPOP/BRPOP via an option in the tcl client that defers reading the reply (Pieter Noordhuis) -2010-06-14 TODO updated (antirez) -2010-06-14 Merge branch 'ltrim-tests' of git://github.com/pietern/redis (antirez) -2010-06-14 rename "list" to "linkedlist" to be more verbose (Pieter Noordhuis) -2010-06-14 allow running the test suite against an external Redis instance, without auto spawning (antirez) -2010-06-14 change ltrim tests to cover all min/max cases and add stronger stresser (Pieter Noordhuis) -2010-06-13 Fixed deps in makefile and mkreleasehdr.sh script to really take advantage of the new trick to avoid recompilation of redis.c on git sha1 or dirty status change (antirez) -2010-06-13 hopefully faster recompiling with a trick (antirez) -2010-06-13 fixed a bug in rdbLoadObject abount specially encoded objects (antirez) -2010-06-13 use raw strings when loading a hash from the rdb into a zipmap (Pieter Noordhuis) -2010-06-12 Merge branch 'expire' of git://github.com/pietern/redis (antirez) -2010-06-11 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-11 LPUSHX, RPUSHX, LINSERT only work on non-empty lists, so there are no clients waiting for a push (Pieter Noordhuis) -2010-06-11 make LINSERT return -1 when the value could not be inserted (Pieter Noordhuis) -2010-06-11 check if the list encoding needs to be changed on LPUSHX, RPUSHX, LINSERT (Pieter Noordhuis) -2010-06-11 make sure the value to insert is string encoded (Pieter Noordhuis) -2010-06-11 rename vars, move arguments, add comments (Pieter Noordhuis) -2010-06-11 always iterate from head to tail on LINSERT (Pieter Noordhuis) -2010-06-11 use REDIS_TAIL to insert AFTER an entry and REDIS_HEAD to insert BEFORE an entry (Pieter Noordhuis) -2010-06-11 move listTypeInsert to be grouped with other wrapper functions (Pieter Noordhuis) -2010-06-11 squashed merge from robey/twitter3: LINSERT BEFORE|AFTER, LPUSHX, RPUSHX (Pieter Noordhuis) -2010-06-09 remove pop function and the sds dependency; can be implemented using get+delete (Pieter Noordhuis) -2010-06-07 compute swappability for ziplist encoded lists (Pieter Noordhuis) -2010-06-07 reuse the sds from the main dictionary in the expiration dictionary (Pieter Noordhuis) -2010-06-07 TODO updated (antirez) -2010-06-07 encode integers while loading an hash (antirez) -2010-06-05 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-05 fixed two leaks for the dual encoded lists (Pieter Noordhuis) -2010-06-04 TODO updated (antirez) -2010-06-04 DISCSARD now unwatches all keys, as it should (antirez) -2010-06-04 generated tests for different encodings to avoid test code duplication (Pieter Noordhuis) -2010-06-04 refactor list tests to test both encodings; implemented assert functions (Pieter Noordhuis) -2010-06-04 renamed hash wrapper functions to match wrapper function naming convention: "Type" (Pieter Noordhuis) -2010-06-04 Merge branch 'lists' of git://github.com/pietern/redis (antirez) -2010-06-04 Merge branch 'smallkeys' (antirez) -2010-06-04 safety assert in listTypeNext (Pieter Noordhuis) -2010-06-04 renamed list wrapper functions to be more verbose (Pieter Noordhuis) -2010-06-04 add thresholds for converting a ziplist to a real list (Pieter Noordhuis) -2010-06-04 merge antirez/smallkeys (Pieter Noordhuis) -2010-06-03 test restored (antirez) -2010-06-03 memory leak introduced in the latest big changes fixed (antirez) -2010-06-03 Fixed VM bugs introduced with the top level keys as sds strings changes (antirez) -2010-06-03 top level keys are no longer redis objects but sds strings. There are still a few bugs to fix when VM is enabled (antirez) -2010-06-03 update Makefile to include ziplist.o (Pieter Noordhuis) -2010-06-03 use ziplists in SORT STORE until the thresholds are determined (Pieter Noordhuis) -2010-06-03 Merge branch 'testsuite' of git://github.com/pietern/redis (antirez) -2010-06-03 Merge branch 'testsuite' of git://github.com/pietern/redis into smallkeys (antirez) -2010-06-03 tag memory leak check on kill server as "leaks" (Pieter Noordhuis) -2010-06-03 tag test with sleep() as slow (Pieter Noordhuis) -2010-06-03 make sure the config it returned when called without code (Pieter Noordhuis) -2010-06-03 tag more slow tests (Pieter Noordhuis) -2010-06-03 change how arguments are passed from the AOF tests (Pieter Noordhuis) -2010-06-03 scope res variable outside test (Pieter Noordhuis) -2010-06-02 tags for existing tests (Pieter Noordhuis) -2010-06-02 pass tags to filter and match via arguments (Pieter Noordhuis) -2010-06-02 basic support to tag tests (Pieter Noordhuis) -2010-06-02 changed how server.tcl accepts options to support more directives without requiring more arguments to the proc (Pieter Noordhuis) -2010-06-02 removed obsolete code (Pieter Noordhuis) -2010-06-02 catch exceptions in the server proc, to be able to kill the entire chain of running servers (Pieter Noordhuis) -2010-06-02 Merge branch 'master' into smallkeys (antirez) -2010-06-02 smarter swapout policy on AOF too (antirez) -2010-06-02 better swapout policy while loading RDB file (antirez) -2010-06-02 minor code comment change (antirez) -2010-06-01 use integer types from stdint.h to be more verbose on the size in bytes of encoded elements. update list length to use 2 bytes instead of 1. (Pieter Noordhuis) -2010-06-01 added stress test for heavy i/o in ziplists (Pieter Noordhuis) -2010-06-01 fix signedness errors in ziplist testing code (Pieter Noordhuis) -2010-06-01 minor code movements and free object pull restored to 1 million (antirez) -2010-06-01 TODO updated with syslog plans for 2.2 (antirez) -2010-06-01 Debug message was printing stuff that are sometimes not initialized/valid (antirez) -2010-06-01 Merge branch 'smallkeys' of github.com:antirez/redis into smallkeys (antirez) -2010-06-01 fixed a few comments (antirez) -2010-06-01 fixed bugs introduced in the rewrite of the new VM engine (antirez) -2010-05-31 support rewriting the AOF with dual list encoding (Pieter Noordhuis) -2010-05-31 small refactor of fwrite* commands for AOF rewrite to allow writing a bulk long long (Pieter Noordhuis) -2010-05-31 use list wrapper functions in computing the dataset digest (Pieter Noordhuis) -2010-05-31 ziplistNext should work as expected when called with a pointer to ZIP_END (Pieter Noordhuis) -2010-05-31 update SORT to work with the dual list encoding (Pieter Noordhuis) -2010-05-31 function to create a new ziplist encoded list (Pieter Noordhuis) -2010-05-31 fixed missing incrRefCount (antirez) -2010-05-31 support rdb saving/loading with dual list encoding (Pieter Noordhuis) -2010-05-31 fixed signedness and disambiguate variable names (Pieter Noordhuis) -2010-05-31 added rdb save function to directly save long long values (Pieter Noordhuis) -2010-05-31 update RPOPLPUSH to support dual encoding (Pieter Noordhuis) -2010-05-31 update list iteration semantic to work as expected (i.e. "while(lNext(..))") (Pieter Noordhuis) -2010-05-31 ziplistDelete no longer needs a direction now ziplistPrev is fixed (Pieter Noordhuis) -2010-05-31 ziplistPrev should return the tail when the argument is ZIP_END (Pieter Noordhuis) -2010-05-31 first step of VM rewrite. blocking VM tests passing, more work needed in the async side (antirez) -2010-05-31 Merge branch 'no-appendfsync-on-rewrite' (antirez) -2010-05-30 fix LREM to remove *all* occurances when a zero argument is given (Pieter Noordhuis) -2010-05-30 fixed LINDEX to always return bulk response (Pieter Noordhuis) -2010-05-30 the tail offset must be an integer pointer to hold a 32-bit offset (Pieter Noordhuis) -2010-05-30 update LREM to support dual encoding via extra iteration primitives (Pieter Noordhuis) -2010-05-30 support dual encoding in LTRIM (Pieter Noordhuis) -2010-05-30 update LRANGE to use basic iteration code to support dual encoding (Pieter Noordhuis) -2010-05-30 inline support for dual encoding in the LINDEX and LSET commands (Pieter Noordhuis) -2010-05-30 generic pop and length function for ziplist encoding (Pieter Noordhuis) -2010-05-30 generic push function that supports the dual encoding (Pieter Noordhuis) -2010-05-30 change delete function to accept a direction argument, so "p" can be properly updated (Pieter Noordhuis) -2010-05-30 expose extra functionality from ziplist.c (Pieter Noordhuis) -2010-05-30 code style consistency fixes (Pieter Noordhuis) -2010-05-29 ziplistIndex now accepts negative indices (Pieter Noordhuis) -2010-05-29 fix compile warnings (Pieter Noordhuis) -2010-05-29 use simpler encoding for the length of the previous entry (Pieter Noordhuis) -2010-05-29 replace functions to get pointers to head and tail by macros (Pieter Noordhuis) -2010-05-29 function to insert an element at an arbitrary position in the list (Pieter Noordhuis) -2010-05-29 extract a generic delete function that can be used in pop and delete(range) (Pieter Noordhuis) -2010-05-29 use the entry struct in zipRawEntryLength (Pieter Noordhuis) -2010-05-29 rename argument names to s* to disambiguate from e* (Pieter Noordhuis) -2010-05-29 change ziplistRepr to use the entry struct (Pieter Noordhuis) -2010-05-29 modify compare function to check if the encoding is equal before comparing (Pieter Noordhuis) -2010-05-29 use a struct to retrieve all details for an entry (Pieter Noordhuis) -2010-05-29 initial implementation for making the ziplist doubly linked (Pieter Noordhuis) -2010-05-29 fix some warnings (Pieter Noordhuis) -2010-05-29 add function to retrieve ziplist size in bytes (Pieter Noordhuis) -2010-05-22 fix compare function of ziplist to only load integer from ziplist when it is encoded as integer (Pieter Noordhuis) -2010-05-22 add function to retrieve length of ziplist (Pieter Noordhuis) -2010-05-22 re-introduce ZIP_BIGLEN for clarity (Pieter Noordhuis) -2010-05-22 added header ziplist.h (Pieter Noordhuis) -2010-05-22 code to compare strings with entries in ziplist, regardless of their encoding (Pieter Noordhuis) -2010-05-22 updated iteration code to work well with different encodings (Pieter Noordhuis) -2010-05-22 move code from zip.c to ziplist.c (Pieter Noordhuis) -2010-05-22 partial revert of c80df5 because ziplist functions are starting to divert too much from zipmap functions (Pieter Noordhuis) -2010-05-22 initial work for integer encoding in ziplists (Pieter Noordhuis) -2010-05-22 move length housekeeping to a macro (Pieter Noordhuis) -2010-05-21 allow entries to be deleted in place when iterating over a ziplist (Pieter Noordhuis) -2010-05-21 allow pointer to be stored to current element when iterating over ziplist (Pieter Noordhuis) -2010-05-21 rename ziplistDelete to ziplistDeleteRange (Pieter Noordhuis) -2010-05-21 code to delete an inner range from the ziplist (Pieter Noordhuis) -2010-05-21 check if *value is non-NULL before setting it (Pieter Noordhuis) -2010-05-21 change iteration code to avoid allocating a new sds for each traversed entry (Pieter Noordhuis) -2010-05-21 code to iterate over a ziplist (Pieter Noordhuis) -2010-05-21 implementation for a ziplist with push and pop support (Pieter Noordhuis) -2010-05-21 extracted general methods to zip.c for reuse in other zip* structures (Pieter Noordhuis) -2010-05-28 command table size calculated with sizeof (antirez) -2010-05-28 use qsort and bsearch to lookup commands in O(log(N)) instead of O(N) (Pieter Noordhuis) -2010-05-28 Merge branch 'cli-stdin' of git://github.com/pietern/redis (antirez) -2010-05-28 Fixed ZINCR Nan bugs leading to server crash and added tests (antirez) -2010-05-28 redis.conf new features the new option, a minor typo preventing the compilation fixed (antirez) -2010-05-28 don't fsync after a rewrite if appendfsync is set to no. use aof_fsycn instead of fsync where appropriate (antirez) -2010-05-28 added new option no-appendfsync-on-rewrite to avoid blocking on fsync() in the main thread while a background process is doing big I/O (antirez) -2010-05-28 Added Git sha1 and dirty status in redis-server -v output (antirez) -2010-05-28 changed the message in the Makefile with the new command like to run the test suite (antirez) -2010-05-27 Fixed typo. (Vincent Palmer) -2010-05-27 new multi/exec tests (antirez) -2010-05-26 build command outside while loop (Pieter Noordhuis) -2010-05-26 require the flag "-c" to be used for redis-cli to read the last argument from stdin (Pieter Noordhuis) -2010-05-26 Merge branch 'master' into nested-multi (antirez) -2010-05-26 Fix EXEC bug that was leaving the client in dirty status when used with WATCH (antirez) -2010-05-26 raise error on nested MULTI and WATCH inside multi (antirez) -2010-05-25 allow regular sets to be passed to zunionstore/zinterstore (Pieter Noordhuis) -2010-05-25 Version is now 2.1.1 (antirez) -2010-05-25 RENAME is now WATCH-aware (antirez) -2010-05-25 TODO updated (antirez) -2010-05-25 WATCH is now able to detect keys removed by FLUSHALL and FLUSHDB (antirez) -2010-05-25 WATCH tests (antirez) -2010-05-25 minor bug fixed in WATCH (antirez) -2010-05-25 WATCH for MULTI/EXEC (CAS alike concurrency) (antirez) -2010-05-25 gitignore updated (antirez) -2010-05-21 Master is now already unfreezed, unstable, and ready to hacking sessions! (antirez) -2010-05-21 Merge branch 'solaris' of git://github.com/pietern/redis (antirez) -2010-05-21 Changelog updated (antirez) -2010-05-21 redis version is now 1.3.14 (aka 2.0.0 RC1) (antirez) -2010-05-21 html doc updated (antirez) -2010-05-21 by default test with valgrind does not show full leak info (antirez) -2010-05-21 minor fix for the skiplist code, resulting in a false positive with valgrind, and in general into a useless small allocation (antirez) -2010-05-21 Merge branch 'master' of git@github.com:antirez/redis (antirez) -2010-05-21 tests suite initial support for valgrind, fixed the old test suite until the new one is able to target a specific host/port (antirez) -2010-05-21 include solaris fixes in sha1.c (Pieter Noordhuis) -2010-05-20 Don't exit with error in tests temp file cleanup if there are no files to clean (antirez) -2010-05-20 fix memory leak on 32-bit builds (Pieter Noordhuis) -2010-05-20 Merge branch 'master' of github.com:antirez/redis (antirez) -2010-05-20 Fix for DEBUG DIGEST (antirez) -2010-05-20 Merge branch 'test_vm' of git://github.com/pietern/redis (antirez) -2010-05-20 code to enable running tests with the vm enabled (Pieter Noordhuis) -2010-05-20 minor change to shutdown (antirez) -2010-05-20 shutdown on SIGTERM (antirez) -2010-05-20 Merge http://github.com/ngmoco/redis (antirez) -2010-05-20 fix compile error on solaris (Pieter Noordhuis) -2010-05-20 added regression for zipmap bug (antirez) -2010-05-20 fix lookup of keys with length larger than ZIPMAP_BIGLEN (Pieter Noordhuis) -2010-05-19 TODO updated (antirez) -2010-05-19 initial tests for AOF (and small changes to server.tcl to support these) (Pieter Noordhuis) -2010-05-19 Merge branch 'master' into integration (Pieter Noordhuis) -2010-05-19 Fix for 'CONFIG SET appendonly no' (antirez) -2010-05-19 It's now possible to turn off and on the AOF via CONFIG (antirez) -2010-05-18 git hash 00000000 in reelase.h when git is not found enabled again after some shell scripting fix that is now compatible with most shells (antirez) -2010-05-18 build fixed when simpler shells are used to create release.h (antirez) -2010-05-18 use git diff when generating release.h to check for dirty status (antirez) -2010-05-18 Solaris fixes (antirez) -2010-05-18 html doc rebuild (antirez) -2010-05-18 buliding of release.h moved into an external script. Avoided recompialtion of redis.c if git sha1 is the same as the previous one (antirez) -2010-05-17 create release.h in make process and add this information to INFO listing (Pieter Noordhuis) -2010-05-16 Redis version is now 1.3.12 (antirez) -2010-05-16 redis version is now 1.3.11 (antirez) -2010-05-16 random refactoring and speedups (antirez) -2010-05-16 faster INCR with very little efforts... (antirez) -2010-05-15 print warnings in redis log when a test raises an exception (very likely to be caused by something like a failed assertion) (Pieter Noordhuis) -2010-05-15 Merge branch 'redis-cli-fix' of http://github.com/tizoc/redis (antirez) -2010-05-15 added pid info to the check memory leaks test, so that those tests don't appear to be duplicated (antirez) -2010-05-15 Merge branch 'integration' of git://github.com/pietern/redis (antirez) -2010-05-14 more endianess detection fix for SHA1 (antirez) -2010-05-14 fixed a warning seen with some GCC version under Linux (antirez) -2010-05-14 initial rough integration test for replication (Pieter Noordhuis) -2010-05-14 store entire server object on the stack instead of just the client (Pieter Noordhuis) -2010-05-14 proc to retrieve values from INFO properties (Pieter Noordhuis) -2010-05-14 one more fix for endianess detection (antirez) -2010-05-14 Fixed sha1.c compilation on Linux, due to endianess detection lameness (antirez) -2010-05-14 ZUNION,ZINTER -> ZUNIONSTORE,ZINTERSTORE (antirez) -2010-05-14 minor fixes to the new test suite, html doc updated (antirez) -2010-05-14 wait for redis-server to be settled and ready for connections (Pieter Noordhuis) -2010-05-14 fix cleaning up tmp folder (Pieter Noordhuis) -2010-05-14 update makefile to use the new test suite (Pieter Noordhuis) -2010-05-14 check for memory leaks before killing a server (Pieter Noordhuis) -2010-05-14 extract code to kill a server to a separate proc (Pieter Noordhuis) -2010-05-14 start servers on different ports to prevent conflicts (Pieter Noordhuis) -2010-05-14 use DEBUG DIGEST in new test suite (Pieter Noordhuis) -2010-05-14 split test suite into multiple files; runs redis-server in isolation (Pieter Noordhuis) -2010-05-14 use DEBUG DIGEST in the test instead of a function that was doing a similar work, but in a much slower and buggy way (antirez) -2010-05-14 Don't rely on cliReadReply being able to return on shutdown (Bruno Deferrari) -2010-05-14 If command is a shutdown, ignore errors on reply (Bruno Deferrari) -2010-05-14 DEBUG DIGEST implemented, in order to improve the ability to test persistence and replication consistency (antirez) -2010-05-13 Add SIGTERM shutdown handling. (Ashley Martens) -2010-05-13 makefile deps updated (antirez) -2010-05-13 conflicts resolved (antirez) -2010-05-13 feed SETEX as SET and EXPIREAT to AOF (Pieter Noordhuis) -2010-05-13 very strong speedup in saving time performance when there are many integers in the dataset. Instead of decoding the object before to pass them to the rdbSaveObject layer we check asap if the object is integer encoded and can be written on disk as an integer. (antirez) -2010-05-13 include limits.h otherwise no double precison macros (antirez) -2010-05-13 explicitly checks with ifdefs if our floating point and long long assumptions are verified (antirez) -2010-05-13 Yet another version of the double saving code, with comments explaining what's happening there (antirez) -2010-05-12 added overflow check in the double -> long long conversion trick to avoid integer overflows. I think this was not needed in practical terms, but it is safer (antirez) -2010-05-12 use withscores when performing the dataset digest (antirez) -2010-05-12 If a float can be casted to a long long without rounding loss, we can use the integer conversion function to write the score on disk. This is a seriuous speedup (antirez) -2010-05-12 fixed compilation warnings in the AOF sanity check tool (antirez) -2010-05-12 Merge branch 'vm-speedup' (antirez) -2010-05-11 fix to return error when calling INCR on a non-string type (Pieter Noordhuis) -2010-05-11 load objects encoded from disk directly without useless conversion (antirez) -2010-05-11 fixed a problem leading to crashes, as keys can't be currently specially encoded, so we can't encode integers at object loading time... For now this can be fixed passing a few flags, or later can be fixed allowing encoded keys as well (antirez) -2010-05-11 long long to string conversion speedup applied in other places as well. Still the code has bugs, fixing right now... (antirez) -2010-05-11 hand written code to turn a long long into a string -> very big speed win (antirez) -2010-05-11 added specialized function to compare string objects for perfect match that is optimized for this task (antirez) -2010-05-11 better use of encoding inforamtion in dictEncObjKeyCompare (antirez) -2010-05-10 CONFIG now can change appendfsync policy at run time (antirez) -2010-05-10 CONFIG command now supports hot modification of RDB saving parameters. (antirez) -2010-05-10 while loading the rdb file don't add the key to the dictionary at all if it's already expired, instead of removing it just after the insertion. (antirez) -2010-05-10 Merge branch 'check-aof' of git://github.com/pietern/redis (antirez) -2010-05-08 minor changes to improve code readability (antirez) -2010-05-08 swap objects out directly while loading an RDB file if we detect we can't stay in the vm max memory limits anyway (antirez) -2010-05-07 change command names no longer used to zunion/zinter (Pieter Noordhuis) -2010-05-07 DEBUG POPULATE command for fast creation of test databases (antirez) -2010-05-07 update TODO (Pieter Noordhuis) -2010-05-07 swap arguments in blockClientOnSwappedKeys to be consistent (Pieter Noordhuis) -2010-05-07 added function that preloads all keys needed to execute a MULTI/EXEC block (Pieter Noordhuis) -2010-05-07 add sanity check to zunionInterBlockClientOnSwappedKeys, as the number of keys used is provided as argument to the function (Pieter Noordhuis) -2010-05-07 make prototype of custom function to preload keys from the vm match the prototype of waitForMultipleSwappedKeys (Pieter Noordhuis) -2010-05-07 extract preloading of multiple keys according to the command prototype to a separate function (Pieter Noordhuis) -2010-05-07 make append only filename configurable (Pieter Noordhuis) -2010-05-07 don't load value from VM for EXISTS (Pieter Noordhuis) -2010-05-07 swap file name pid expansion removed. Not suited for mission critical software... (antirez) -2010-05-07 Swap file is now locked (antirez) -2010-05-06 Merge branch 'master' into aof-speedup (antirez) -2010-05-06 log error and quit when the AOF contains an unfinished MULTI (antirez) -2010-05-06 log error and quit when the AOF contains an unfinished MULTI (Pieter Noordhuis) -2010-05-06 Merge branch 'master' into check-aof (Pieter Noordhuis) -2010-05-06 hincrby should report an error when called against a hash key that doesn't contain an integer (Pieter Noordhuis) -2010-05-06 AOF writes are now accumulated into a buffer and flushed into disk just before re-entering the event loop. A lot less writes but still this guarantees that AOF is written before the client gets a positive reply about a write operation, as no reply is trasnmitted before re-entering into the event loop. (antirez) -2010-05-06 clarified a few messages in redis.conf (antirez) -2010-05-05 ask for confirmation before AOF is truncated (Pieter Noordhuis) -2010-05-05 str can be free'd outside readString (Pieter Noordhuis) -2010-05-05 moved argument parsing around (Pieter Noordhuis) -2010-05-05 ignore redis-check-aof binary (Pieter Noordhuis) -2010-05-05 allow AOF to be fixed by truncating to the portion of the file that is valid (Pieter Noordhuis) -2010-05-05 tool to check if AOF is valid (Pieter Noordhuis) -2010-05-02 included fmacros.h in linenose.c to avoid compilation warnings on Linux (antirez) -2010-05-02 compilation fix for mac os x (antirez) -2010-05-02 Merge branch 'master' of git@github.com:antirez/redis (antirez) -2010-05-02 On Linux now fdatasync() is used insetad of fsync() in order to flush the AOF file kernel buffers (antirez) -2010-04-30 More tests for APPEND and tests for SUBSTR (antirez) -2010-04-30 linenoise.c updated, now redis-cli can be used in a pipe (antirez) -2010-04-29 redis-cli minor fix (less segfault is better) (antirez) -2010-04-29 New MONITOR output format with timestamp, every command in a single line, string representations (antirez) -2010-04-29 redis-cli INFO output format is now raw again (antirez) -2010-04-29 Added more information about slave election in Redis Cluster alternative doc (antirez) -2010-04-29 Redis cluster version 2 (antirez) -2010-04-27 Fixed a redis-cli bug, was using free instead of zfree call (antirez) -2010-04-27 AOF is now rewritten on slave after SYNC with master. Thanks to @_km for finding this bug and any others' (antirez) -2010-04-27 redis-cli is now using only the new protocol (antirez) -2010-04-27 Minimal support for subscribe/psubscribe in redis-cli (antirez) -2010-04-26 don't output the newline when stdout is not a tty (antirez) -2010-04-26 redis-cli now is able to also output the string representation instead of the raw string. Much better for debugging (antirez) -2010-04-26 Initial support for quoted strings in redis-cli (antirez) -2010-04-23 SETEX implemented (antirez) -2010-04-23 Pub/Sub API change: now messages received via pattern matching have a different message type and an additional field representing the original pattern the message matched (antirez) -2010-04-22 typo fixed, reloaded (antirez) -2010-04-22 typo fixed (antirez) -2010-04-22 REDIS-CLUSTER doc updated (antirez) -2010-04-22 Virtual memory design document removed, no longer needed as we have a full specification and implementation (antirez) -2010-04-22 new units for bytes specification (antirez) -2010-04-22 Now in redis.conf it is possible to specify units where appropriate instead of amounts of bytes, like 2Gi or 4M and so forth (antirez) -2010-04-21 binary safe keys ready implementation of RANDOMKEYS (antirez) -2010-04-21 Now that's the right 1.3.10 (antirez) -2010-04-21 Revert "fsync always now uses O_DIRECT on Linux" (antirez) -2010-04-21 Revert "define __USE_GNU to get O_DIRECT" (antirez) -2010-04-21 Merge branch 'master' of github.com:antirez/redis (antirez) -2010-04-21 Revert "version 1.3.10" (antirez) -2010-04-21 version 1.3.10 (antirez) -2010-04-20 define __USE_GNU to get O_DIRECT (antirez) -2010-04-20 fsync always now uses O_DIRECT on Linux (antirez) -2010-04-20 More precise memory used guesswork in zmalloc.c (antirez) -2010-04-19 Fix for MULTI/EXEC and Replication/AOF: now the block is correctly sent as MULTI/..writing operations../EXEC. Ok for slaves but more work needed for the AOF as it should be a write-all-or-nothing business (antirez) -2010-04-19 running the test using tcl8.5 directly instead of tclsh that too often it's a symlink to 8.4 (antirez) -2010-04-19 Added package require Tcl 8.5 in redis.tcl so it will show a clear error when the test suit is attempted to run under 8.4 (antirez) -2010-04-18 Fix for a SORT bug introduced with commit 16fa22f1, regression test added (antirez) -2010-04-18 Guru mediation -> meditation (antirez) -2010-04-16 check eptr inline (Pieter Noordhuis) -2010-04-16 refactor code that retrieves value from object or replies to client (Pieter Noordhuis) -2010-04-17 Merge branch 'hash' of git://github.com/pietern/redis (antirez) -2010-04-17 redisAssert(0) => redisPanic("something meaningful") (antirez) -2010-04-17 make sure that the resulting value in hincrby is encoded when possible (Pieter Noordhuis) -2010-04-17 increment dirty counter after hmset (Pieter Noordhuis) -2010-04-17 strip tryObjectEncoding from hashSet, to enable the arguments being encoded in-place (Pieter Noordhuis) -2010-04-17 Added support for Guru Mediation, and raising a guru mediation if refCount <= 0 but decrRefCount is called against such an object (antirez) -2010-04-16 fix small error and memory leaks in SORT (Pieter Noordhuis) -2010-04-16 SORT/GET test added (antirez) -2010-04-16 Added tests for GET/BY against hashes fields (antirez) -2010-04-16 Merge branch 'hash-refactor' of git://github.com/pietern/redis (antirez) -2010-04-16 check object type in lookupKeyByPattern (Pieter Noordhuis) -2010-04-16 make sortCommand aware that lookupKeyByPattern always increased the refcount of the returned value (Pieter Noordhuis) -2010-04-16 revert 0c390a to stop using tricks with o->refcount (Pieter Noordhuis) -2010-04-16 store the hash iterator on the heap instead of the stack (Pieter Noordhuis) -2010-04-16 drop inline directive (Pieter Noordhuis) -2010-04-16 rename hashReplace to hashSet (Pieter Noordhuis) -2010-04-16 added dictFetchValue() to dict.c to make hash table API a bit less verbose in the common cases (antirez) -2010-04-03 Don't set expire to keys with ttl=0, remove them immediately. (antirez) -2010-04-15 make sure that cmpobj is in decoded form when sorting by ALPHA (this solves edge case from previous commit where (!sortby && alpha) == 1) (Pieter Noordhuis) -2010-04-15 enable hash dereference in SORT on BY and GET (Pieter Noordhuis) -2010-04-15 use shared replies for hset (Pieter Noordhuis) -2010-04-15 set refcount of string objects retrieved from zipmaps to 0, so we don't have to touch the refcount of the objects inside dicts (Pieter Noordhuis) -2010-04-15 added HSETNX (Pieter Noordhuis) -2010-04-14 refactor of hash commands to use specialized api that abstracts zipmap and dict apis (Pieter Noordhuis) -2010-04-13 move retrieval of long up to prevent an empty hash from being created (Pieter Noordhuis) -2010-04-15 more advanced leaks detection in test redis (antirez) -2010-04-15 ability to select port/host from make test (antirez) -2010-04-15 Active rehashing (antirez) -2010-04-15 Incrementally rehahsing hash table! Thanks to Derek Collison and Pieter Noordhuis for feedbacks/help (antirez) -2010-04-14 Does not allow commands other than Pub/Sub commands when there is at least one pattern (antirez) -2010-04-13 Fixed a tiny memory leak when loading the configuration file. (Alex McHale) -2010-04-13 Merge branch 'hmget' of git://github.com/pietern/redis (antirez) -2010-03-29 Validate numeric inputs. (Alex McHale) -2010-03-24 Remove trailing whitespace. (Alex McHale) -2010-04-12 Now all the commands returning a multi bulk reply against non existing keys will return an empty multi bulk, not a nil one (antirez) -2010-04-12 implemented HMGET (Pieter Noordhuis) -2010-04-12 implemented HMSET (Pieter Noordhuis) -2010-04-12 Sharing of small integer objects: may save a lot of memory with datasets having many of this (antirez) -2010-04-10 dict.c fixed to play well with enabling/disabling of the hash table (antirez) -2010-04-09 removed a no longer true assert in the VM code (antirez) -2010-04-09 shareobjects feautres killed - no gains most of the time, but VM complexities (antirez) -2010-04-09 use directly the real key object in VM I/O jobs to match by pointer, and to handle different keys with the same name living in different DBs, but being at the same moment in the IO job queues (antirez) -2010-04-08 last change reverted as it was unstable... more testing needed (antirez) -2010-04-08 Prevent hash table resize while there are active child processes in order to play well with copy on write (antirez) -2010-04-08 Merge branch 'issue_218' of git://github.com/pietern/redis (antirez) -2010-04-08 -1 not needed... (antirez) -2010-04-08 Skiplist theoretical fix (antirez) -2010-04-07 Now when a child is terminated by a signal, the signal number is logged as well (antirez) -2010-04-07 First version of evented Redis Tcl client, that will be used for BLPOP and Pub/Sub tests (antirez) -2010-04-05 use long long reply type for HINCRBY (Pieter Noordhuis) -2010-04-05 last argument is never encoded for HINCRBY (Pieter Noordhuis) -2010-04-02 Now PUBLISH commands are replicated to slaves (antirez) -2010-04-01 use the right object when cleaning up after zunion/zinter (fixes issue 216) (Pieter Noordhuis) -2010-04-01 Merge branch 'zipmap' of git://github.com/pietern/redis (antirez) -2010-04-01 reduce code complexity because zipmapLen now is O(1) (Pieter Noordhuis) -2010-04-01 update the zipmap entry in-place instead of appending it (Pieter Noordhuis) -2010-04-01 updated zipmap documentation to match the implementation (Pieter Noordhuis) -2010-04-01 allow 4 free trailing bytes for each value (Pieter Noordhuis) -2010-04-01 Pub/Sub pattern matching capabilities (antirez) -2010-04-01 use function to determine length of a single entry (Pieter Noordhuis) -2010-03-31 Deny EXEC under out of memory (antirez) -2010-03-29 No timeouts nor other commands for clients in a Pub/Sub context (antirez) -2010-03-29 free hash table entries about no longer active classes, so that PUBSUB can be abused with millions of different classes (antirez) -2010-03-29 Fixed a refcount stuff leading to PUBSUB crashes (antirez) -2010-03-29 fmacros added to linenoise, avoiding all the nice warnings... (antirez) -2010-03-29 First pubsub fix (antirez) -2010-03-29 PUBSUB implemented (antirez) -2010-03-29 Redis version is now 1.3.8 (antirez) -2010-03-28 removed references in code to ZIPMAP_EMPTY (Pieter Noordhuis) -2010-03-28 use first byte of zipmap to store length (Pieter Noordhuis) -2010-03-28 implemented strategy that doesn't use free blocks in zipmaps (Pieter Noordhuis) -2010-03-26 Merge branch 'hincrby' of git://github.com/pietern/redis (antirez) -2010-03-26 removed unnecessary refcount increase that caused the HINCRBY memleak (Pieter Noordhuis) -2010-03-26 implements HINCRBY and tests (todo: find and fix small memleak) (Pieter Noordhuis) -2010-03-26 Removed a useless if spotted by Pieter Noordhuis (antirez) -2010-03-26 Fixed a critical replication bug: binary values issued with the multi bulk protocol caused a protocol desync with slaves. (antirez) -2010-03-24 Fixed the reply about denied write commands under maxmemory reached condition: now the error will no longer lead to a client-server protocol desync (antirez) -2010-03-24 CONFIG command implemened -- just a start but already useful (antirez) -2010-03-24 redis-cli prompt is now redis> (antirez) -2010-03-23 with --help states that you can use - as config file name to feed config via stdin (antirez) -2010-03-23 New INFO field: expired_keys (antirez) -2010-03-23 the Cron timer function is now called 10 times per second instead of 1 time per second to make Redis more responsibe to BGSAVE and to delete expired keys more incrementally (antirez) -2010-03-23 Use linenoise for line editing on redis-cli. (Michel Martens) -2010-03-23 Fix authentication for redis-cli on non-interactive mode. (Michel Martens) -2010-03-23 key deletion on empty value fix + some refactoring (antirez) -2010-03-23 Empty value trigger key removal in all the operations (antirez) -2010-03-22 Merged gnrfan patches fixing issues 191, 193, 194 (antirez) -2010-03-22 Merge branch 'issue_193' of git://github.com/gnrfan/redis (antirez) -2010-03-22 Merge branch 'issue_191' of git://github.com/gnrfan/redis (antirez) -2010-03-22 Redis master version is now 1.3.7 (antirez) -2010-03-19 support for include directive in config parser (Jeremy Zawodny) -2010-03-19 Removed a stupid overriding of config values due to a wrong cut&paste (antirez) -2010-03-19 VM hash type swappability implemented. Handling of failed pthread_create() call. (antirez) -2010-03-19 Solving issue #191 on Google Code: -v and --version should print the version of Redis (Antonio Ognio) -2010-03-19 Solves issue #194 on Google Code: --help parameter to redis-srver prints the usage message (Antonio Ognio) -2010-03-19 Fixing issue 193 (Antonio Ognio) -2010-03-18 increment server.dirty on HDEL (antirez) -2010-03-18 Redis 1.3.6 (antirez) -2010-03-18 test-redis.tcl dataset digest function Hash support (antirez) -2010-03-18 zipmap fix for large values (antirez) -2010-03-18 Optimization fixed and re-activated (antirez) -2010-03-18 reverted an optimization that makes Redis not stable (antirez) -2010-03-18 Fixed redis-cli auth code (antirez) -2010-03-17 HDEL fix, an optimization for comparison of objects in hash table lookups when they are integer encoding (antirez) -2010-03-17 Version is now 1.3.5 (antirez) -2010-03-17 Merged Pietern patch for VM key args helper function. Fixed an obvious bug in the redis-cli passwd auth stuff (antirez) -2010-03-17 Merge branch 'aggregates' of git://github.com/pietern/redis (antirez) -2010-03-17 Added Authentication to redis-cli.c using -a switch Update usage fixed Makefile to delete redis-check-dump during make clean (root) -2010-03-17 HEXISTS and tests implemented (antirez) -2010-03-17 More hash tests (antirez) -2010-03-17 better HSET test (antirez) -2010-03-17 Fixed a bug in HSET, a memory leak, and a theoretical bug in dict.c (antirez) -2010-03-17 More Hash tests (antirez) -2010-03-13 added preloading keys from VM when using ZINTER or ZUNION (Pieter Noordhuis) -2010-03-13 added explicit AGGREGATE [SUM|MIN|MAX] option to ZUNION/ZINTER (Pieter Noordhuis) -2010-03-16 HGET fix for integer encoded field against zipmap encoded hash (antirez) -2010-03-16 zrevrank support in redis-cli (antirez) -2010-03-16 HKEYS / HVALS / HGETALL (antirez) -2010-03-16 Solved a memory leak with Hashes (antirez) -2010-03-15 pretty big refactoring (antirez) -2010-03-15 An interesting refactoring + more expressive internal API (antirez) -2010-03-15 Fixed the same problem in ZREVRANK (antirez) -2010-03-15 Fixed a ZRANK bug (antirez) -2010-03-15 zipmap to hash conversion in HSET (antirez) -2010-03-14 max zipmap entries and max zipmap value parameters added into INFO output (antirez) -2010-03-14 HDEL and some improvement in DEBUG OBJECT command (antirez) -2010-03-14 Append only file support for hashes (antirez) -2010-03-13 utility to check rdb files for unprocessable opcodes (Pieter Noordhuis) -2010-03-12 A minor fix and a few debug messages removed (antirez) -2010-03-12 Applied the replication bug patch provided by Jeremy Zawodny, removing temp file collision after the slave got the dump.rdb file in the SYNC stage (antirez) -2010-03-11 Fix for HGET against non Hash type, debug messages used to understand a bit better a corrupted rdb file (antirez) -2010-03-09 fix: use zmalloc instead of malloc (Pieter Noordhuis) -2010-03-09 Merged zsetops branch from Pietern (antirez) -2010-03-09 Merged ZREMBYRANK from Pietern (antirez) -2010-03-09 Merged ZREVRANK from Pietern (antirez) -2010-03-09 use a struct to store both a dict and its weight for ZUNION and ZINTER, so qsort can be applied (Pieter Noordhuis) -2010-03-09 Hash auto conversion from zipmap to hash table, type fixed for hashes, hash loading from disk (antirez) -2010-03-09 replaced ZMERGE by ZUNION and ZINTER. note: key preloading by the VM does not yet work (Pieter Noordhuis) -2010-03-08 Hashes saving / fixes (antirez) -2010-03-08 use ZMERGE as starting point (Pieter Noordhuis) -2010-03-07 HSET fixes, now the new pointer is stored back in the object pointer field (antirez) -2010-03-07 added ZREVRANK (Pieter Noordhuis) -2010-03-06 Fix for replicaiton with over 2GB dump file initial SYNC stage (antirez) -2010-03-06 first implementation of HSET/HSET. More work needed (antirez) -2010-03-05 zipmaps functions to get, iterate, test for existence. Initial works for Hash data type (antirez) -2010-03-04 redis-benchmark now implements Set commands benchmarks (antirez) -2010-03-04 zipmap iteration code (antirez) -2010-03-04 moved code to delete a single node from a zset to a separate function (Pieter Noordhuis) -2010-03-04 rename zslDeleteRange to zslDeleteRangeByScore (to differentiate between deleting using score or rank) (Pieter Noordhuis) -2010-03-04 use 1-based rank across zsl*Rank functions consistently (Pieter Noordhuis) -2010-03-04 implemented ZREMBYRANK (Pieter Noordhuis) -2010-03-04 A fix for initialization of augmented skip lists (antirez) -2010-03-04 A fix for an invalid access when VM is disabled (antirez) -2010-03-04 Merge branch 'zsl-get-rank' of git://github.com/pietern/redis (antirez) -2010-03-04 redis-cli now runs in interactive mode if no command is provided (antirez) -2010-03-04 merged memory reduction patch (Pieter Noordhuis) -2010-03-04 Now list push commands return the length of the new list, thanks to Gustavo Picon (antirez) -2010-03-04 first check if starting point is trivial (head or tail) before applying log(N) search (Pieter Noordhuis) -2010-03-04 use rank to find starting point for ZRANGE and ZREVRANGE (Pieter Noordhuis) -2010-03-04 lookup rank of a zset entry in a different function (Pieter Noordhuis) -2010-03-04 SUBSTR fix for integer encoded vals (antirez) -2010-03-04 fix ZRANK (realize that rank is 1-based due to the skip list header) (Pieter Noordhuis) -2010-03-03 initial implementation of SUBSTR (antirez) -2010-03-03 TODO updated (antirez) -2010-03-03 fpurge call removed from redis-cli (antirez) -2010-03-03 ZRANK stress tester (antirez) -2010-03-03 use less memory as element->span[0] will always be 1; any level 0 skip list is essentially a linked list (Pieter Noordhuis) -2010-03-03 rank is very unlikely to overflow integer range (Pieter Noordhuis) -2010-03-03 x->backward never equals zsl->header (Pieter Noordhuis) -2010-03-03 initial implementation for augmented zsets and the zrank command (Pieter Noordhuis) -2010-03-03 zipampDel() implemented (antirez) -2010-03-03 added quit and exit commands to redis-cli in order to quit the interactive mode (antirez) -2010-03-03 Merge remote branch 'djanowski/interactive' (antirez) -2010-03-02 Add support for MULTI/EXEC. (Damian Janowski & Michel Martens) -2010-03-02 Remove trailing newline in interactive mode. (Damian Janowski & Michel Martens) -2010-03-02 minor fix for a Linux warning (antirez) -2010-03-02 Add interactive mode to redis-cli. (Michel Martens & Damian Janowski) -2010-03-02 Better to increment the version minor number when a VM bug is fixed... it will be simpler to understand what's going on when users will report problems with the INFO trace. (antirez) -2010-03-02 Fixed a subtle VM bug... was not flushing the buffer so the child process read truncated data (antirez) -2010-03-01 KEYS now returns a multi bulk reply (antirez) -2010-02-27 Add DISCARD command to discard queued MULTI commands. (antirez) -2010-03-01 Swappability bug due to a typo fixed thanks to code review by Felix Geisendörfer @felixge (antirez) -2010-02-28 minor fixes for zipmap.c (antirez) -2010-02-27 first zipmap fix of a long sequence in the days to come ;) (antirez) -2010-02-27 initial zipmap.c implementation (antirez) -2010-02-27 Bug #169 fixed (BLOP/BRPOP interrupted connections are not cleared from the queue) (antirez) -2010-02-22 Fixed 32bit make target to work on Linux out of the box (antirez) -2010-02-19 A problem with replication with multiple slaves connectiong to a single master fixed. It was due to a typo, and reported on github by the user micmac. Also the copyright year fixed from many files. (antirez) -2010-02-10 Saner VM defaults for redis.conf (antirez) -2010-02-09 VM now is able to block clients on swapped keys for all the commands (antirez) -2010-02-07 ZCOUNT and ZRANGEBYSCORE new tests (antirez) -2010-02-07 ZRANGEBYSCORE now supports open intervals, prefixing double values with a open paren. Added ZCOUNT that can count the elements inside an interval of scores, this supports open intervals too (antirez) -2010-02-07 WITHSCORES in ZRANGEBYSCORE thanks to Sam Hendley (antirez) -2010-02-06 Added "withscores" option to zrangebyscore command. Based on withscores support in zrange function, ugliest part was the argument parsing to handle using it with the limit option. (Sam Hendley) -2010-02-06 DEBUG OBJECT provide info about serialized object length even when VM is disabled (antirez) -2010-02-06 multi bulk requests in redis-benchmark, default fsync policy changed to everysec, added a prefix character for DEBUG logs (antirez) -2010-02-04 APPEND tests (antirez) -2010-02-04 APPEND command (antirez) -2010-02-02 Faster version of the function hashing possibly encoded objects, leading to a general speed gain when working with Sets of integers (antirez) -2010-02-02 faster Set loading time from .rdb file resizing the hash table to the right size before loading elements (antirez) -2010-02-02 Log time taken to load the DB at startup, in seconds (antirez) -2010-01-31 Fixed VM corruption due to child fclosing the VM file directly or indirectly calling exit(), now replaced with _exit() in all the sensible places. Masked a few signals from IO threads. (antirez) -2010-01-28 loading side of the threaded VM (antirez) -2010-01-26 TODO cahnges (antirez) -2010-01-23 Fixed memory human style memory reporting, removed server.usedmemory, now zmalloc_used_memory() is used always. (antirez) -2010-01-22 VM tuning thanks to redis-stat vmstat. Now it performs much better under high load (antirez) -2010-01-21 Changelog updated (antirez) -2010-01-21 REDIS_MAX_COMPLETED_JOBS_PROCESSED is now in percentage, not number of jobs. Moved a debugging message a few lines forward as it was called where a few logged parameters where invalid, leading to a crash (antirez) -2010-01-20 fixed a deadlock caused by too much finished processes in queue so that I/O clients writing to the wirte side of the pipe used to awake the main thread where blocking. Then a BGSAVE started waiting for the last active thread to finish, condition impossible because all the I/O threads where blocking on threads. Takes this as a note to myself... (antirez) -2010-01-20 ae.c event loop does no longer support exception notifications, as they are fully pointless. Also a theoretical bug that never happens in practice fixed. (antirez) -2010-01-19 commercial tools stuff removed from the Redis makefile. cotools are now migrated into a different repos (antirez) -2010-01-19 removed a bug in the function to cancel an I/O job (antirez) -2010-01-17 static symbols update (antirez) -2010-01-16 removed support for REDIS_HELGRIND_FRIENDLY since Helgrind 3.5.0 is friendly enough even with many threads created and destroyed (antirez) -2010-01-15 now redis-cli understands -h (antirez) -2010-01-15 Create swap file only if not exists (antirez) -2010-01-15 I hate warnings (antirez) -2010-01-15 fixed a minor memory leak in configuration file parsing (antirez) -2010-01-15 minor fix (antirez) -2010-01-15 support for named VM swap file. Fixed a few important interaction issues between the background saving processes and IO threads (antirez) -2010-01-15 fix for the just added new test (antirez) -2010-01-15 useless debugging messages removed (antirez) -2010-01-15 new test added (antirez) -2010-01-15 thread safe zmalloc used memory counter (antirez) -2010-01-15 A define to make Redis more helgrind friendly (antirez) -2010-01-15 removed a few races from threaded VM (antirez) -2010-01-14 Fixed a never experienced, theoretical bug that can actually happen in practice. Basically when a thread is working on a I/O Job we need to wait it to finish before to cancel the Job in vmCancelThreadedIOJob(), otherwise the thread may mess with an object that is being manipulated by the main thread as well. (antirez) -2010-01-14 Set the new threads stack size to a LZF friendly amount (antirez) -2010-01-13 access to already freed job structure fixed by statements reoredering (antirez) -2010-01-13 removed a useless debugging message (antirez) -2010-01-13 Wait zero active threads condition before to fork() for BGSAVE or BGREWRITEAOF (antirez) -2010-01-13 list API is now thread safe (antirez) -2010-01-13 minor TODO and debugging info changes (antirez) -2010-01-12 support for blocking VM in config file (antirez) -2010-01-12 more non blocking VM changes (antirez) -2010-01-12 fix for test #11 (antirez) -2010-01-12 a few more stuff in INFO about VM. Test #11 changed a bit in order to be less lame (antirez) -2010-01-12 Added a define to configure how many completed IO jobs the handler should process at every call. (antirez) -2010-01-11 Fixed a bug in the IO Job canceling funtion (antirez) -2010-01-11 more steps towards a working non blocking VM (antirez) -2010-01-11 converted random printfs in debug logs (antirez) -2010-01-11 removed a bug introduced with non blocking VM (antirez) -2010-01-11 a few non blocking VM bugs fixed (antirez) -2010-01-11 More work on non-blocking VM. Should work in a few days (antirez) -2010-01-11 More threaded I/O VM work + Redis init script (antirez) -2010-01-10 more work on VM threaded I/O. Still nothing of usable (antirez) -2010-01-09 non-blocking VM data structures, just a start (antirez) -2010-01-08 used_memory_human added to INFO output. Human readable amount of memory used. (antirez) -2010-01-07 Now DEBUG OBJECT plays well with swapped out objects (antirez) -2010-01-07 fflush VM swap file after object swapping (antirez) -2010-01-07 added the fmacros to enable support for fseeko() lseeko() with 64bit off_t (antirez) -2010-01-07 VM now swaps objects out while loading datasets not fitting into vm-max-memory bytes of RAM (antirez) -2010-01-07 added process id information in INFO (antirez) -2010-01-06 vm-enabled set to no by default in redis.conf (antirez) -2010-01-06 a new default redis.conf (antirez) -2010-01-06 VM stats in INFO command (antirez) -2010-01-06 Introduced a new log verbosity level, so now DEBUG is really for debugging. Refactored a bit maxmemory. When virtual memory is short in RAM free the objects freelist as well as swapping things out. (antirez) -2010-01-05 fixed a bug in bgsave when VM is off but still it was testing for obj->storage field (antirez) -2010-01-05 converted a few calls to assert() => redisAssert() to print stack trace (antirez) -2010-01-05 BGREWRITEAOF now works with swapping on (antirez) -2010-01-05 A first fix for SET key overwrite (antirez) -2010-01-05 SAVE now works with VM (antirez) -2010-01-05 swapping algorithm a bit more aggressive under low memory (antirez) -2010-01-05 basic VM mostly working! (antirez) -2010-01-05 New object field (one of the unused bytes) to hold the type of the swapped out value object in key objects (antirez) -2010-01-05 VM internals bugfixes, set 1 (antirez) -2010-01-05 load key from swap on key lookup (antirez) -2010-01-05 more object-level VM primitives (antirez) -2010-01-05 Redis objects swapping / loading (antirez) -2010-01-05 rdbLoadObject() as a separated function to load objects from disk. Dropped support for RDB version 0, I guess no longer has this legacy DBs around (antirez) -2010-01-04 VM low level pages handling (antirez) -2010-01-04 vm swap file creation, and some basic configuration (antirez) -2010-01-04 version marked 1.3.2 (antirez) -2010-01-04 saving code refactored a bit, added a function returning the number of bytes an object will use on disk (antirez) -2010-01-02 Now the PUSH side of RPOPLPUSH is able to unblock clients blocked on BLPOP (antirez) -2010-01-02 Version is now 1.3.1 (antirez) -2010-01-02 New vararg BLPOP able to block against multiple keys (antirez) -2009-12-29 fixed a problem with BLPOP timeout of zero, now it blocks forever (antirez) -2009-12-29 BLPOP timeouts implemented (antirez) -2009-12-29 first working implementation of BLPOP and BRPOP, still everything is to test well (antirez) -2009-12-29 a few more fixes, still broken (antirez) -2009-12-29 First fix, still broken (antirez) -2009-12-29 minor fix for Linux 64 bit (antirez) -2009-12-29 not yet working BLPOP implementation (antirez) -2009-12-27 AOFSYNC removed, got a better idea... (antirez) -2009-12-27 AOFSYNC command implemented (antirez) -2009-12-27 Version changed to 1.3.0, welcome to the new unstable (antirez) -2009-12-27 Now MULTI returns +OK as well (antirez) -2009-12-27 MULTI/EXEC first implementation (antirez) -2009-12-24 Fixed a minor bug in GETSET, now the SET part is not performed if the GET fails because the key does not contain a string value (antirez) -2009-12-23 html doc readded (antirez) -2009-12-23 ZRANGE WITHSCORES test added (antirez) -2009-12-23 version is now 1.1.94 (antirez) -2009-12-23 Add the command name in the unknown command error message. (antirez) -2009-12-22 ZRANGE, ZREVRANGE now support WITHSCORES options (antirez) -2009-12-22 html docs update (ZINCRBY added) (antirez) -2009-12-18 TODO list update (antirez) -2009-12-18 the pipelining test was ran against DB 1 for error, now it runs on DB 9 like all the other tests (antirez) -2009-12-18 still more tests (antirez) -2009-12-18 SORT STORE test added (antirez) -2009-12-18 Now SORT returns an empty bulk reply if the key does not exist (antirez) -2009-12-18 modified a bit the ZREVRANGE test to cover a few lines of code more (antirez) -2009-12-18 SHUTDOWN now does the right thing when append only is on, that is, fsync instead to save the snapshot. (antirez) -2009-12-18 Added a missing server.dirty increment in a non critical place, added more tests (antirez) -2009-12-18 LTRIM stress testing test added (antirez) -2009-12-18 LTRIM now returns +OK against non existing keys. More tests in test-redis.tcl (antirez) -2009-12-18 added sdstoupper() declaration in sds.h (antirez) -2009-12-18 Fixed sds.c bug #124 (antirez) -2009-12-16 LZF compression re-enabled by default, but with INIT_HTAB set to 0 to avoid the very costly memset initialization. Note that with this option set valgrind will output some false positive about lzf_c.c (antirez) -2009-12-16 lzf compression switched off by default now, with config file option to enable it in redis.conf (antirez) -2009-12-16 Regression for epoll bug in redis-test.tcl, version is now 1.1.93 (antirez) -2009-12-16 Fixed a lame epoll issue (antirez) -2009-12-15 html doc updated (antirez) -2009-12-15 version is now 1.1.92 (antirez) -2009-12-15 Two important fixes to append only file: zero length values and expires. A pretty neat new test to check consistency of randomly build datasets against snapshotting and AOF. (antirez) -2009-12-15 debug loadaof implemented in order to add more consistency tests in test-redis.tcl (antirez) -2009-12-15 Added a new test able to stress a lot the snapshotting engine (antirez) -2009-12-15 Unified handling of empty queries with normal queries. (antirez) -2009-12-15 Fixed some subtle bug in the command processing code almost impossible to spot in the real world, thanks to gcov (antirez) -2009-12-15 Regression test for SINTERSTORE added (antirez) -2009-12-15 Fixed issue #121 (antirez) -2009-12-14 a few more tests and ability to run a specific test in test-redis.tcl (antirez) -2009-12-13 Changed the reply of BGSAVE and BGREWRITEAOF from +OK to a more meaningful message that makes the user aware of an operation that just started and is not yet finished. (antirez) -2009-12-13 Set the master->slave logical client as authenticated on creation, so that if the slave requires a password replication works anyway (antirez) -2009-12-13 TODO update (antirez) -2009-12-12 bgrewriteaof_in_progress added to INFO (antirez) -2009-12-12 TODO list modified. What's planned for 1.4 is now written in the stone ;) (antirez) -2009-12-12 better handling of non blocking connect on redis-benchmark: EPIPE on read does not print an error message now (antirez) -2009-12-11 some change to redis-sha1.rb utility to make it more robust against non-meaningful changes in the dataset (antirez) -2009-12-10 redis-sha1.rb utility updated (antirez) -2009-12-10 a bit more verbose -ERR wrong number o arguments error, now gives info about the command name causing the error (antirez) -2009-12-10 TODO change and minor SETNX optimization (antirez) -2009-12-06 in rdbLoadDoubleValue now the buffer is nul terminated correctly. Thanks valgrind. (antirez) -2009-12-06 printf format warnings fixed by casting (antirez) -2009-12-06 Regression tests for SETNX and MSETNX bugs added (antirez) -2009-12-06 SETNX and MSETNX now respect the delete-on-write operation of EXPIREing keys (antirez) -2009-12-06 Fixed daemonization when using kqueue/kevent. Now the server initialization is performed *after* the daemonization (antirez) -2009-12-05 more HTML doc changes (antirez) -2009-12-05 HTML doc update (antirez) -2009-12-05 a few redis-cli format specified fixed (antirez) -2009-12-05 use __attribute__ format in sdscatprintf() when the compiler is GCC. Fixed format bugs resulting from the new warnings. (antirez) -2009-12-01 TODO update (antirez) -2009-12-01 compilation problem on 64bit mac os x 10.5 possibly fixed (antirez) -2009-12-01 virtual memory design doc typos (antirez) -2009-12-01 design documents added to the project (antirez) -2009-11-30 Fixed issued #85 (getDecodedObject: Assertion 1 != 1 failed. While sorting a set), added a smarter assert() function to dump the stacktrace, provided a macro to initalize Redis objects on the stack to avoid this kind of bugs. (antirez) -2009-11-30 fixed a subtle bug in redis-cli not having visible effects (antirez) -2009-11-29 TODO updated (antirez) -2009-11-29 Version chagned to 1.100, also known as the first first 2.0 beta version (antirez) -2009-11-29 more tests in test-redis.tcl, some minor fix (antirez) -2009-11-29 SORT support for sorted sets (antirez) -2009-11-28 Implemented LIMIT option in ZRANGEBYSCORE. We now enter feature-freeze (antirez) -2009-11-28 Changelog updated (antirez) -2009-11-28 html doc updated (antirez) -2009-11-28 enable kqueue/kevent only for Mac OS X 10.6.x as it seems that 10.5.x has a broken implementation of this syscalls. (antirez) -2009-11-28 TODO updated (antirez) -2009-11-28 ZRANGEBYSCORE fuzzy test (antirez) -2009-11-28 ZRANGEBYSCORE memory leak fixed, ZRANGEBYSCORE initial test added (antirez) -2009-11-28 INFO refactored. Stack trace on memory corruption now dumps the same information as the INFO command (antirez) -2009-11-28 ifdefs added to use kevent on Free Open and Net BSD as well. INFO and ae.c modified in order to report the multiplexing API in use (antirez) -2009-11-28 Enabled object encoding for multiple keys in MSET. Added a test for memory leaks in test-redis.tcl when running on Mac OS X (antirez) -2009-11-28 Merge branch 'kqueue' of git://github.com/mallipeddi/redis (antirez) -2009-11-28 Changes to TODO list, commented a function in redis.c (antirez) -2009-11-28 Added support for kqueue. (Harish Mallipeddi) -2009-11-27 TODO updated (antirez) -2009-11-26 zero length bulk data reading fixed in loadAppendOnlyFile() (antirez) -2009-11-26 append only file fixes (antirez) -2009-11-26 log rebuilding, random refactoring, work in progress please wait for an OK commit before to use this version (antirez) -2009-11-24 DEBUG RELOAD implemented, and test-redis.tcl modified to use it to check for persistence consistency. (antirez) -2009-11-24 Redis version set to 1.07 (antirez) -2009-11-24 sorted sets saving fixed (antirez) -2009-11-24 minor TODO change (antirez) -2009-11-24 minor fix to avoid a false valgrind warning. (antirez) -2009-11-23 epoll support enabled by default for Linux builds (antirez) -2009-11-23 epoll module for ae.c implemented. Some more testing needed (antirez) -2009-11-23 commented the HAVE_EPOLL test in config.h to allow compilation under Linux now that the epoll module is still missing (antirez) -2009-11-23 ae_select module added (antirez) -2009-11-23 ae.c now supports multiple polling API modules, even if only ae_select.c is implemented currently. Also adding and removing an event is now O(1). (antirez) -2009-11-23 ae.c initial refactoring for epoll implementation (antirez) -2009-11-21 version incremented up to 1.06 (antirez) -2009-11-21 TODO aesthetic changes (antirez) -2009-11-21 TODO updated with plans up to 1.5 (antirez) -2009-11-21 SRANDMEMBER test (antirez) -2009-11-21 Fixed a SORT memory leak that should never happen in practice (antirez) -2009-11-21 SORT GET # implemented, with a test (antirez) -2009-11-21 EXPIREAT test (antirez) -2009-11-20 EXPIRE tests (antirez) -2009-11-20 more RPOPLPUSH tests (antirez) -2009-11-20 RPOPLPUSH tests added (antirez) -2009-11-20 ZINCRBY return value fixed (antirez) -2009-11-20 ZINCRSCOREBY => ZINCRBY (antirez) -2009-11-19 ZINCRSCOREBY implemented (antirez) -2009-11-19 writev() finally uncommented again (antirez) -2009-11-19 redis-benchmark hopefully last bug with multi bulk reply fixed (antirez) -2009-11-19 debug mode in redis-bench (antirez) -2009-11-19 Use writev(2) if glue output buffers is disabled (antirez) -2009-11-19 benchmark.c fixes (antirez) -2009-11-18 more experiments with long replies, glue output buffer, and writev. (antirez) -2009-11-18 benchmarking with different number of LRANGE elements. Ability to change the glue output buffer limit by #define (antirez) -2009-11-18 more writev tests/work (antirez) -2009-11-18 redis-benchmark multi bulk reply support hopefully fixed (antirez) -2009-11-17 support for writev implemented but currently ifdef-ed in order to understan why I can't see the improvements expected. Btw code provided by Stefano Barbato (antirez) -2009-11-17 multi-bulk reply support for redis-bench, and as a result LRANGE is not tested, providing some number for the tuning of multi-bulk requests performances server-side (antirez) -2009-11-12 Solaris fix thanks to Alan Harder (antirez) -2009-11-12 Merge git://github.com/ianxm/redis (antirez) -2009-11-12 ZSCORE fixed, now returns NULL on missing key or missing element (antirez) -2009-11-12 Redis test will not fail the SAVE test even if a background save is in progress (antirez) -2009-11-12 LPOPPUSH renamed into RPOPLPUSH (antirez) -2009-11-11 can select db num (ian) -2009-11-11 Workaround for test-redis.tcl and Tcl 8.4.x about ZSCORE test (antirez) -2009-11-11 Removed a long time warning compiling with recent GCC on Linux (antirez) -2009-11-11 TODO updated (antirez) -2009-11-11 LPUSHPOP first implementation (antirez) -2009-11-10 Tcl script, make target, and redis.c changes to build the static symbol table automagically (antirez) -2009-11-10 Implemented a much better lazy expiring algorithm for EXPIRE (antirez) -2009-11-10 Fixed issue 92 in redis: redis-cli (nil) return value lacks CR/LF (antirez) -2009-11-10 Minor TODO change with new expiring algorithm description. New expiring algorithm moved since it'll go in 1.1 (antirez) -2009-11-04 redis-test is now a better Redis citizen, testing everything against DB 9 and 10 and only if this DBs are empty. (antirez) -2009-11-04 fixed a refcounting bug with SORT ... STORE leading to random crashes (root) -2009-11-04 masterauth option merged, thanks to Anthony Lauzon (antirez) -2009-11-03 ZSets double to string serialization fixed (antirez) -2009-11-03 client-libraries directory readded (antirez) -2009-11-03 redis.tcl put at toplevel since it's uesd for the test-redis.tcl script (antirez) -2009-11-03 client libs removed from Redis git (antirez) -2009-11-03 redis-cli now accepts a -r (repeat) switch. Still there is a memory leaks to fix (antirez) -2009-11-01 TODO updated again (antirez) -2009-11-01 TODO updated (antirez) -2009-11-01 redis-cli now makes clear when the returned string is an integer (antirez) -2009-11-01 SORT STORE option (antirez) -2009-11-01 now Redis prints DB stats just after the startup without to wait a second for the first report (antirez) -2009-11-01 another fix for append only mode, now read-only operations are not appended (antirez) -2009-11-01 appendfsync parsing in config file fixed. If you benchmarked Redis against different appendfsync options is time to try again ;) (antirez) -2009-11-01 append only file loading fixed (antirez) -2009-11-01 first version of append only file loading -- STILL BROKEN don't use it (antirez) -2009-10-31 Fixed Issue 83:Using TYPE on a zset results in a malformed response from the Redis server (antirez) -2009-10-31 Fixed compilation on Linux (antirez) -2009-10-30 append only mode is now able to translate EXPIRE into EXPIREAT transparently (antirez) -2009-10-30 appendfsync is now set to NO by default (antirez) -2009-10-30 support for appendonly mode no, always, everysec (antirez) -2009-10-30 first fix for append only mode (antirez) -2009-10-30 Initial implementation of append-only mode. Loading still not implemented. (antirez) -2009-10-30 EXPIRE behaviour changed a bit, a negative TTL or an EXPIREAT with unix time in the past will now delete the key. It seems saner to me than doing nothing. (antirez) -2009-10-30 EXPIREAT implemented, will be useful for the append-only mode (antirez) -2009-10-29 Fixed Issue 74 (ERR just returned on invalid password), now the error message is -ERR invalid password. (antirez) -2009-10-29 Fixed issue 72 (SLAVEOF shutdowns redis-server on malformed reply) (antirez) -2009-10-29 Fixed issue 77 (Incorrect time in log files) thanks to youwantalex (antirez) -2009-10-29 Fixed Issue 76 (redis-server crashes when it can't connect to MASTER and client connects to SLAVE) (antirez) -2009-10-29 ZREMRANGEBYSCORE implemented. Remove a range of elements with score between min and max (antirez) -2009-10-28 TODO changes and mostly theoretical minor skiplist change (antirez) -2009-10-28 ZLEN renamed ZCARD for consistency with SCARD (antirez) -2009-10-27 TODO reworked to reflect the real roadmap (antirez) -2009-10-27 Fix for 'make 32bit' (antirez) -2009-10-27 a fix for the solaris fix itself ;) (antirez) -2009-10-27 More Solaris fixes (antirez) -2009-10-27 A lot of ZSETs tests implemented, and a bug fixed thanks to this new tests (antirez) -2009-10-27 zmalloc Solaris fixes thanks to Alan Harder (antirez) -2009-10-27 ZSCORE implemented (antirez) -2009-10-26 fix for ZRANGEBYSCORE (antirez) -2009-10-26 ZRANGEBYSCORE implemented. Redis got range queries! (antirez) -2009-10-26 A trivial change makes the new implementation O(log(N)) instead of O(log(N))+O(M) when there are M repeated scores! (antirez) -2009-10-26 ZSET now saved on disk like any other type (antirez) -2009-10-26 double serialization routines implemented (antirez) -2009-10-26 ZSETs random fixes. Now the implementation appears to be pretty stable (antirez) -2009-10-26 another leak fixed. Can't find more for now, but still a bug in ZSETs to fix (antirez) -2009-10-26 ZSETs memory leak #1 solved, another one missing (antirez) -2009-10-26 Fix for skiplists backward link (antirez) -2009-10-26 Merged Solaris patches provided by Alan Harder (antirez) -2009-10-26 backward support to skiplists for ZREVRANGE, still broken, committing since I've to merge the Solaris patches (antirez) -2009-10-26 TODO updated (antirez) -2009-10-26 ZREM implemented (antirez) -2009-10-24 fix for ZADD in score update mode (antirez) -2009-10-24 some work on ZADD against existing element (score update), still broken... (antirez) -2009-10-23 zrange now starts to work. zadd still does not support update and will crash or leak or b000mmmmm (antirez) -2009-10-23 zrange initial hack (not working for now) (antirez) -2009-10-23 first skiplist fix, courtesy of valgrind (antirez) -2009-10-23 zset symbols added to stack trace code. ZSets will simply crash at the moment (antirez) -2009-10-23 more work on ZSETs and a new make target called 32bit to build i386 binaries on mac os x leopard (antirez) -2009-10-23 initial skiplist implementation. Most memory checks removed and zmalloc() modified to fail with an error message and abort. Anyway Redis is not designed to recover from out of memory conditions. (antirez) -2009-10-23 Fixed compilation in mac os x snow leopard when compiling a 32 bit binary. (antirez) -2009-10-22 version incremented to 1.050 to distinguish from 1.001 stable and next stable versions with minor fixes (antirez) -2009-10-21 TODO updated (antirez) -2009-10-21 SRANDMEMBER added (antirez) -2009-10-20 Imporant bug leading to data corruption fixed (NOT affecting stable distribution), Tcl client lib MSET/MSETNX implementation fixed, Added new tests for MSET and MSETNX in test-redis.tcl (antirez) -2009-10-17 added multi-bulk protocol support to redis-cli and support for MSET and MSETNX (antirez) -2009-10-17 MSET fixed, was not able to replace keys already set for a stupid bug (antirez) -2009-10-16 some dead code removed (antirez) -2009-10-16 multi bulk input protocol fixed (antirez) -2009-10-16 MSET and MSETNX commands implemented (antirez) -2009-10-07 undoed all the sds hacking that lead just to random bugs and no memory saving ;) (antirez) -2009-10-07 initial multi-bulk query protocol, this will allow MSET and other interesting features. (antirez) -2009-10-03 benchmark now outputs the right command line to shorten the TIME_WAIT interval on Mac OS X when keep alive is set (antirez) -2009-10-02 Issue 69 fixed. Object integer encoding now works with replication and MONITORing again. (antirez) -2009-09-18 LREM fixed, used to crash since the new object integer encoding is on the stage (antirez) -2009-09-17 maxmemory didn't worked in 64 systems for values > 4GB since it used to be an unsigned int. Fixed (antirez) -2009-09-10 incremented version number to 1.001, AKA Redis edge is no longer stable... (antirez) -2009-09-10 in-memory specialized object encoding (for now 32 signed integers only) (antirez) -2009-09-03 Latest doc changes for 1.0 (antirez) -2009-09-03 Redis 1.0.0 release (antirez) -2009-09-02 Redis version pushed to 1.0 (antirez) -2009-09-02 Ruby client lib updated to the latest git version (antirez) -2009-09-02 update-scala-client script added (antirez) -2009-09-02 Scala client added thanks to Alejanro Crosa (antirez) -2009-09-02 QuickStart added (antirez) -2009-09-01 Fixed crash with only space and newline as command (issue 61), thanks to a guy having as nick "fixxxerrr" (antirez) -2009-08-11 TODO list modified (antirez) -2009-07-24 more snow leopard related fixes (for 32bit systems) (antirez) -2009-07-24 fixed compilation with Snow Leopard, thanks to Lon Baker for providing SSH access to Snow Leopard box (antirez) -2009-07-22 Fixed NetBSD compile problems (antirez) -2009-07-17 now the size of the shared pool can be really modified via config, also the number of objects in the sharing pool is logged when the log level is set to debug. Thanks to Aman Gupta (antirez) -2009-07-05 added utils/redis-copy.rb, a script that is able to copy data from one Redis server to another one on the fly. (antirez) -2009-07-04 Applied three different patches thanks to Chris Lamb, one to fix compilation and get the IP register value on Linux IA64 and other systems. One in order to log the overcommit problem on the logs instead of the standard output when Redis is demonized. The latest in order to suggest a more consistent way in order to switch to 1 the memory overcommit Linux feature. (antirez) -2009-07-03 bugfix: EXPIRE now propagates to the Slave. (antirez) -2009-06-16 Redis version modified to 0.900 (antirez) -2009-06-16 update-ruby-client script already points to ezmobius repo (antirez) -2009-06-16 client libraries updated (antirez) -2009-06-16 Redis release candidate 1 (antirez) -2009-06-16 Better handling of background saving process killed or crashed (antirez) -2009-06-14 number of keys info in INFO command thanks to Diego Rosario Brogna (antirez) -2009-06-14 SPOP documented (antirez) -2009-06-14 Clojure library thanks to Ragnar Dahlén (antirez) -2009-06-10 It is now possible to specify - as config file name to read it from stdin (antirez) -2009-06-10 sync with jodosha redis-rb (antirez) -2009-06-10 Redis-rb sync (antirez) -2009-06-10 max inline request raised again to 1024*1024*256 bytes (antirez) -2009-06-10 max bytes in an inline command raised to 1024*1024 bytes, in order to allow for very large MGETs and still protect from client crashes (antirez) -2009-06-08 SPOP implemented. Hash table resizing for Sets and Expires too. Changed the resize policy to play better with RANDOMKEY and SPOP. (antirez) -2009-06-07 some minor changes to the backtrace code (antirez) -2009-06-07 enable backtrace capabilities only for Linux and MacOSX (antirez) -2009-06-07 Dump a backtrace on sigsegv/sigbus, original coded thanks to Diego Rosario Brogna, modified in order to work on different OSes and to enhance reliability (antirez) -2009-06-06 Merge git://github.com/dierbro/redis (antirez) -2009-06-06 add more output (hrothgar) -2009-06-06 store static function pointer for a useful stack trace (hrothgar) -2009-06-06 TODO updated (antirez) -2009-06-06 Makefile dependencies updated (antirez) -2009-06-05 Avoid a busy loop while sending very large replies against very fast links, this allows to be more responsive with other clients even under a KEY * against the loopback interface (antirez) -2009-06-05 Kill the background saving process before performing SHUTDOWN to avoid races (antirez) -2009-06-05 LREM now returns :0 for non existing keys (antirez) -2009-06-05 - put some order in code - better output (hrothgar) -2009-06-05 added config.h for #ifdef business isolation, added fstat64 for Mac OS X (antirez) -2009-06-04 remove die() :-) (hrothgar) -2009-06-04 add compile options to debug (hrothgar) -2009-06-04 initial commit print stack trace (hrothgar) -2009-06-04 initial commit print stack trace (hrothgar) -2009-06-04 macosx specific zmalloc.c, uses malloc_size function in order to avoid to waste memory and time to put an additional header (antirez) -2009-06-04 DEBUG OBJECT implemented (antirez) -2009-06-04 backtrace support removed: unreliable stack trace :( (antirez) -2009-06-04 initial backtrace dumping on sigsegv/sigbus + debug command (antirez) -2009-06-03 Python lib updated (antirez) -2009-06-03 shareobjectspoolsize implemented in reds.conf, in order to control the pool size when object sharing is on (antirez) -2009-05-30 Erlang client updated (antirez) -2009-05-30 Python client library updated (antirez) -2009-05-29 Redis-rb minor bool convertion fix (antirez) -2009-05-29 ruby library client is not Redis-rb merged with RubyRedis "engine" by Brian McKinney (antirez) -2009-05-28 __P completely removed from pqsort.c/h (antirez) -2009-05-28 another minor fix for Solaris boxes (antirez) -2009-05-28 minor fix for Solaris boxes (antirez) -2009-05-28 minor fix for Solaris boxes (antirez) -2009-05-27 maxmemory implemented (antirez) -2009-05-26 Redis git version modified to 0.101 in order to distinguish that from the latest tar.gz via INFO ;) (antirez) -2009-05-26 Redis 0.100 released (antirez) -2009-05-26 client libraries synched in git (antirez) -2009-05-26 ignore gcc warning about write() return code not checked. It is esplicitily this way since the "max number of clients reached" is a best-effort error (antirez) -2009-05-26 max bytes of a received command enlarged from 1k to 16k (antirez) -2009-05-26 RubyRedis: set TCP_NODELAY TCP socket option to to disable the neagle algorithm. Makes a huge difference under some OS, notably Linux (antirez) -2009-05-25 maxclients implemented, see redis.conf for details (antirez) -2009-05-25 INFO command now reports replication info (antirez) -2009-05-25 minor fix to RubyRedis about bulk commands sent without arguments (antirez) -2009-05-24 Warns if using the default config (antirez) -2009-05-24 Issue with redis-client used in scripts solved, now to check if the latest argument must come from standard input we do not check that stdin is or not a tty but the command arity (antirez) -2009-05-23 RubyRedis: now sets are returned as arrays again, and not as Set objects (antirez) -2009-05-23 SLAVEOF command documented (antirez) -2009-05-23 SLAVEOF command implemented for replication remote control (antirez) -2009-05-22 Fix: no connection timeout for the master! (antirez) -2009-05-22 replication slave timeout when receiving the initial bulk data set to 3600 seconds, now that replication is non-blocking the server must save the db before to start the async replication and this can take a lot of time with huge datasets (antirez) -2009-05-22 README tutorial now reflects the new proto (antirez) -2009-05-22 critical bug about glueoutputbuffers=yes fixed. Under load and with pipelining and clients disconnecting on the middle of the chat with the server, Redis could block. Now it's ok (antirez) -2009-05-22 TTL command doc added (antirez) -2009-05-22 TTL command implemented (antirez) -2009-05-22 S*STORE now return the cardinality of the resulting set (antirez) -2009-05-22 rubyredis more compatible with Redis-rb (antirez) -2009-05-21 minor indentation fix (antirez) -2009-05-21 timeout support and Redis-rb compatibility aliases implemented in RubyRedis (antirez) -2009-05-21 RubyRedis info postprocessor rewritten in a more functional way (antirez) -2009-05-21 dead code removed from RubyRedis (antirez) -2009-05-21 command postprocessing implemented into RubyRedis (antirez) -2009-05-20 Automagically reconnection of RubyRedis (antirez) -2009-05-20 RubyRedis: Array alike operators implemented (antirez) -2009-05-20 random testing code removed (antirez) -2009-05-20 RubyRedis DB selection forced at object creation (antirez) -2009-05-20 Initial version of an alternative Ruby client added (antirez) -2009-05-20 SDIFF / SDIFFSTORE added to doc (antirez) -2009-05-20 Aman Gupta changes merged (antirez) -2009-05-20 Merge git://github.com/tmm1/redis (antirez) -2009-05-19 Allow timeout=0 config to disable client timeouts (Aman Gupta) -2009-05-19 Partial qsort implemented in SORT command, only when both BY and LIMIT is used. minor fix for a warning compiling under Linux. (antirez) -2009-05-19 psort.c/h added. This is a partial qsort implementation that Redis will use when SORT+LIMIT is requested (antirez) -2009-05-17 Fix SINTER/UNIONSTORE to allow for &=/|= style operations (i.e. SINTERSTORE set1 set1 set2) (Aman Gupta) -2009-05-17 Optimize SDIFF to return as soon as the result set is empty (Aman Gupta) -2009-05-17 SDIFF/SDIFFSTORE implemnted unifying it with the implementation of SUNION/SUNIONSTORE (antirez) -2009-05-11 timestamp in log lines (antirez) -2009-05-11 Python client updated pushing from Ludo's repository (antirez) -2009-05-11 disconnect when we cannot read from the socket (Ludovico Magnocavallo) -2009-05-11 benchmark utility now supports random keys (antirez) -2009-05-10 minor doc changes (antirez) -2009-05-09 added tests for vararg DEL (antirez) -2009-05-09 DEL is now a vararg, IMPORTANT: memory leak fixed in loading DB code (antirez) -2009-05-09 doc changes (antirez) -2009-05-09 CPP client added thanks to Brian Hammond (antirez) -2009-05-06 Infinite number of arguments for MGET and all the other commands (antirez) -2009-05-04 Warns if /proc/sys/vm/overcommit_memory is set to 0 on Linux. Also make sure to don't resize the hash tables while the child process is saving in order to avoid copy-on-write of memory pages (antirez) -2009-04-30 zmalloc fix, return NULL or real malloc failure (antirez) -2009-04-30 more fixes for dict.c and the 150 million keys limit (antirez) -2009-04-30 dict.c modified to be able to handle more than 150,000,000 keys (antirez) -2009-04-29 fuzz stresser implemented in redis-test (antirez) -2009-04-29 fixed for HT resize check 32bits overflow (antirez) -2009-04-29 Check for fork() failure in background saving (antirez) -2009-04-29 fix for the LZF off-by-one bug added (antirez) -2009-04-28 print bytes used at exit on SHUTDOWN (antirez) -2009-04-28 SMOVE test added (antirez) -2009-04-28 SMOVE command implemented (antirez) -2009-04-28 less CPU usage in command parsing, case insensitive config directives (antirez) -2009-04-28 GETSET command doc added (antirez) -2009-04-28 GETSET tests (antirez) -2009-04-28 GETSET implemented (antirez) -2009-04-27 ability to specify a different file name for the DB (antirez) -2009-04-27 log file parsing code improved a bit (antirez) -2009-04-27 bgsave_in_progress field in INFO output (antirez) -2009-04-27 INCRBY/DECRBY now support 64bit increments, with tests (antirez) -2009-04-23 RANDOMKEY regression test added (antirez) -2009-04-23 dictGetRandomKey bug fixed, RANDOMKEY will not block the server anymore (antirez) -2009-04-22 FLUSHALL/FLUSHDB no longer sync on disk. Just increment the dirty counter by the number of elements removed, that will probably trigger a background saving operation (antirez) -2009-04-21 forgot to comment testing code in PHP lib. Now it is ok (antirez) -2009-04-21 PHP client ported to PHP5 and fixed (antirez) -2009-04-21 doc update (antirez) -2009-04-20 Non blocking replication (finally!). C-side linked lists API improved. (antirez) -2009-04-19 SUNION, SUNIONSTORE, Initial work on non blocking replication (antirez) -2009-04-10 Redis 0.091 released (antirez) -2009-04-10 SINTER/SINTERSTORE/SLEMENTS fix: misisng keys are now not errors, but just like empty sets (antirez) -2009-04-09 doc changes (antirez) -2009-04-08 TODO changes, minor change to default redis.conf (antirez) -2009-04-08 html doc updated (antirez) -2009-04-08 library clients update scripts (antirez) -2009-04-08 Ruby client updated (antirez) -2009-04-08 Lua client updated (antirez) -2009-04-08 Changelog updated (antirez) -2009-04-08 Merge git://github.com/ludoo/redis (antirez) -2009-04-08 add expire command to the php lib (Ludovico Magnocavallo) -2009-04-08 fix decode bug, add flush and info commands (Ludovico Magnocavallo) -2009-04-07 Rearrange redisObject struct to reduce memory usage in 64bit environments (as recommended http://groups.google.com/group/redis-db/msg/68f5a743f8f4e287) (Bob Potter) -2009-04-07 ruby19 compat: use each_line on string (Bob Potter) -2009-04-07 64bit fixes for usedmemory (Bob Potter) -2009-04-08 RANDOMKEY issue 26 fixed, generic test + regression added (antirez) -2009-04-06 Don't accept SAVE if BGSAVE is in progress (antirez) -2009-04-06 add expire command to the python lib (Ludovico Magnocavallo) -2009-04-03 persistent EXPIRE (antirez) -2009-04-03 dirty increment was missing in two points. TODO updated (antirez) -2009-04-02 LZF configured to initalize the HT in order to be determinsitic and play well with valgrind (antirez) -2009-04-02 fix select test (Ludovico Magnocavallo) -2009-04-02 fix trailing cr+nl in values (Ludovico Magnocavallo) -2009-04-02 compression/decompression of large values on disk now working (antirez) -2009-04-02 disable LZF compression since it's not able to load the DB for now, the load part is missing (antirez) -2009-04-02 new LZF files added (antirez) -2009-04-02 Fixed issue 23 about AUTH (antirez) -2009-04-02 Issue 22 fixed (antirez) -2009-04-01 non-lazy expired keys purging implemented (antirez) -2009-04-01 fastlz dependence removed (antirez) -2009-04-01 Initial implementation of EXPIRE (antirez) -2009-03-30 TODO updated (antirez) -2009-03-30 changelog added (antirez) -2009-03-28 redis-sha1 utility added (antirez) -2009-03-28 Integer encoding implemented in dump file. Doc updated (antirez) -2009-03-27 feature macros defined to play well with C99 (antirez) -2009-03-27 feature macros defined to play well with C99 (antirez) -2009-03-27 now Redis is C99-ok (antirez) -2009-03-27 IMPORTANT FIX: new dump format implementation was broken. Now it's ok but tests for the 32-bit case values are needed (antirez) -2009-03-27 ANSI-C compatibility changes (antirez) -2009-03-27 Ruby client library updated. Important changes in this new version! (antirez) -2009-03-26 Lua client added thanks to Daniele Alessandri (antirez) -2009-03-26 Lua client added thanks to Daniele Alessandri (antirez) -2009-03-26 AUTH merged from Brian Hammond fork, reworked a bit to fix minor problems (antirez) -2009-03-25 Adds AUTH command. (Brian Hammond) -2009-03-25 Nasty bug of the new DB format fixed, objects sharing implemented (antirez) -2009-03-25 doc update (antirez) -2009-03-25 Erlang client synched with Valentiono's repo (antirez) -2009-03-25 New file dump format, perl client library added (antirez) -2009-03-25 New protocol fix for LREM (antirez) -2009-03-24 two typos fixed (antirez) -2009-03-24 Now the Redis test uses the proper Tcl client library (antirez) -2009-03-24 Tcl client library (antirez) -2009-03-24 redis-benchmark sync with the new protocol (antirez) -2009-03-24 git mess :) (Ludovico Magnocavallo) -2009-03-24 sync python client to the new protocol (Ludovico Magnocavallo) -2009-03-24 protocol fix in SORT reply with null elements (antirez) -2009-03-24 protocol doc changed (antirez) -2009-03-24 Server replies now in the new format, test-redis.tcl and redis-cli modified accordingly (antirez) -2009-03-24 Python client library updated, thanks to Ludo! (antirez) -2009-03-24 random tested mode for test-redis.tcl, minor other stuff, version switched to 0.8 (antirez) -2009-03-23 Now MONITOR/SYNC cannot be issued multiple times (antirez) -2009-03-23 MONITOR command implemented. (antirez) -2009-03-23 lucsky changes imported. pid file path can now be configured, redis-cli fixes (antirez) -2009-03-23 Merge git://github.com/lucsky/redis (antirez) -2009-03-23 another missing free->zfree replacement fixed. Thanks to Ludo (antirez) -2009-03-23 Fixed redis-cli readLine loop to correctly handle EOF. (Luc Heinrich) -2009-03-23 Display the port on server startup. (Luc Heinrich) -2009-03-23 Allow to specify the pid file from the config file. (Luc Heinrich) -2009-03-23 Added gitignore file. (Luc Heinrich) -2009-03-22 MGET tests added (antirez) -2009-03-22 doc changes (antirez) -2009-03-22 added doc for MGET (antirez) -2009-03-22 redis-cli now checks the arity of vararg commnads (antirez) -2009-03-22 INFO fixed, MGET implemented, redis-cli implements INFO/MGET (antirez) -2009-03-22 first commit (antirez) \ No newline at end of file diff --git a/src/config.c b/src/config.c index dc9e50ea312..f657776306d 100644 --- a/src/config.c +++ b/src/config.c @@ -1102,34 +1102,6 @@ void configGetCommand(redisClient *c) { * CONFIG REWRITE implementation *----------------------------------------------------------------------------*/ -/* IGNORE: - * - * rename-command - * include - * - * Special handling: - * - * notify-keyspace-events - * client-output-buffer-limit - * save - * appendonly - * appendfsync - * dir - * maxmemory-policy - * loglevel - * unixsocketperm - * slaveof - * - * Type of config directives: - * - * CUSTOM - * VERBATIM - * YESNO - * L - * LL - * - */ - /* We use the following dictionary type to store where a configuration * option is mentioned in the old configuration file, so it's * like "maxmemory" -> list of line numbers (first line is zero). */ @@ -1363,7 +1335,7 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int char *enum_name, *matching_name = NULL; int enum_val, def_val, force; sds line; - + va_start(ap, value); while(1) { enum_name = va_arg(ap,char*); @@ -1375,7 +1347,7 @@ void rewriteConfigEnumOption(struct rewriteConfigState *state, char *option, int if (value == enum_val) matching_name = enum_name; } va_end(ap); - + force = value != def_val; line = sdscatprintf(sdsempty(),"%s %s",option,matching_name); rewriteConfigRewriteLine(state,option,line,force); @@ -1439,7 +1411,7 @@ void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { int force = server.aof_state != REDIS_AOF_OFF; char *option = "appendonly"; sds line; - + line = sdscatprintf(sdsempty(),"%s %s", option, (server.aof_state == REDIS_AOF_OFF) ? "no" : "yes"); rewriteConfigRewriteLine(state,option,line,force); diff --git a/src/redis.c b/src/redis.c index 791e77de9a5..3d0c6a5499a 100644 --- a/src/redis.c +++ b/src/redis.c @@ -320,7 +320,7 @@ void redisLogFromHandler(int level, const char *msg) { if ((level&0xff) < server.verbosity || (log_to_stdout && server.daemonize)) return; - fd = log_to_stdout ? STDOUT_FILENO : + fd = log_to_stdout ? STDOUT_FILENO : open(server.logfile, O_APPEND|O_CREAT|O_WRONLY, 0644); if (fd == -1) return; ll2string(buf,sizeof(buf),getpid()); diff --git a/src/replication.c b/src/replication.c index d44e06ec826..20b03e00a86 100644 --- a/src/replication.c +++ b/src/replication.c @@ -182,7 +182,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { } } server.slaveseldb = dictid; - + /* Add the multi bulk reply size to the static buffer, that is, the number * of arguments of the command to send to every slave. */ b[0] = '*'; @@ -498,7 +498,7 @@ void syncCommand(redisClient *c) { return; /* No full resync needed, return. */ } else { char *master_runid = c->argv[1]->ptr; - + /* Increment stats for failed PSYNCs, but only if the * runid is not "?", as this is used by slaves to force a full * resync on purpose when they are not albe to partially diff --git a/src/version.h b/src/version.h index da35b43e159..5b8d18d05e6 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.3" +#define REDIS_VERSION "2.7.101" From 5b19e9d4332839d7c3148bb1ed89f0699d75bf26 Mon Sep 17 00:00:00 2001 From: Ronan Amicel Date: Thu, 18 Jul 2013 12:23:47 +0200 Subject: [PATCH 0742/1752] Fix a few typos in release notes --- 00-RELEASENOTES | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 77a9c965167..29cd2b547d2 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -21,14 +21,14 @@ This is the first release candidate of Redis 2.8 (official version is 2.7.101). The following is a list of improvements in Redis 2.8, compared to Redis 2.6. * [NEW] Slaves are now able to partially resynchronize with the master, so most - of the times a full resynchornization with the RDB creation in the master + of the times a full resynchronization with the RDB creation in the master side is not needed when the master-slave link is disconnected for a short amount of time. -* [NEW] Experimental IPv6 support.4 -* [NEW] Slaves explicitly ping masters now, a master is able to detect a timedout +* [NEW] Experimental IPv6 support. +* [NEW] Slaves explicitly ping masters now, a master is able to detect a timed out slave independently. * [NEW] Masters can stop accepting writes if not enough slaves with a given - maxium latency are connected. + maximum latency are connected. * [NEW] Keyspace changes notifications via Pub/Sub. * [NEW] CONFIG SET maxclients is now available. * [NEW] Ability to bind multiple IP addresses. @@ -42,7 +42,7 @@ The following is a list of improvements in Redis 2.8, compared to Redis 2.6. * [NEW] EVALSHA can now be replicated as such, without requiring to be expanded to a full EVAL for the replication link. * [NEW] Better Lua scripts error reporting. -* [NEW] SDIFF performances improved. +* [NEW] SDIFF performance improved. * [FIX] A number of bugfixes. Migrating from 2.6 to 2.8 @@ -53,14 +53,14 @@ that you should be aware of: The following commands changed behavior: - * SORT with ALPHA now sort according to local collation locale if no STORE + * SORT with ALPHA now sorts according to local collation locale if no STORE option is used. * ZADD/ZINCRBY are now able to accept a bigger range of values as valid scores, that is, all the values you may end having as a result of calling ZINCRBY multiple times. * Many errors are now prefixed by a more specific error code instead of the generic -ERR, for example -WRONGTYPE, -NOAUTH, ... - * PUBLISH called inside lua scripts is now correctly propagated to slaves. + * PUBLISH called inside Lua scripts is now correctly propagated to slaves. The following redis.conf and CONFIG GET / SET parameters changed: @@ -74,8 +74,8 @@ The following INFO fields changed format in a non-backward compatible way: Replication: Redis 2.8 can be used as slave for Redis 2.6, but doing this is only - a good idea for a short amonut of time needed to upgrade your servers. - We suggest to update both master and slaves about at the same time. + a good idea for the short amount of time needed to upgrade your servers. + We suggest to update both master and slaves at about the same time. -------------------------------------------------------------------------------- From 5d0ad2f983c93a58a6203b479d7e18e78d33091b Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 18 Jul 2013 16:10:31 +0200 Subject: [PATCH 0743/1752] Fixed typo in 2.8 release notes. --- 00-RELEASENOTES | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 77a9c965167..48ad8e49751 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -2,8 +2,8 @@ Redis 2.8 release notes ======================= ** IMPORTANT ** Check the 'Migrating from 2.6 to 2.8' section at the end of - this file for information about what changed between 2.4 and - 2.6 and how this may affect your application. + this file for information about what changed between 2.6 and + 2.8 and how this may affect your application. -------------------------------------------------------------------------------- Upgrade urgency levels: From 57c8e026b97ef6346e96bd77dcf420a85ebdbf07 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 23 Jul 2013 16:35:55 +0200 Subject: [PATCH 0744/1752] Every function inside sds.c is now commented. --- src/sds.c | 152 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 145 insertions(+), 7 deletions(-) diff --git a/src/sds.c b/src/sds.c index aa45fc4f516..b63f44d940b 100644 --- a/src/sds.c +++ b/src/sds.c @@ -36,6 +36,18 @@ #include "sds.h" #include "zmalloc.h" +/* Create a new sds string with the content specified by the 'init' pointer + * and 'initlen'. + * If NULL is used for 'init' the string is initialized with zero bytes. + * + * The string is always null-termined (all the sds strings are, always) so + * even if you create an sds string with: + * + * mystring = sdsnewlen("abc",3"); + * + * You can print the string with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle, as the length is stored in the sds header. */ sds sdsnewlen(const void *init, size_t initlen) { struct sdshdr *sh; @@ -53,24 +65,43 @@ sds sdsnewlen(const void *init, size_t initlen) { return (char*)sh->buf; } +/* Create an empty (zero length) sds string. Even in this case the string + * always has an implicit null term. */ sds sdsempty(void) { return sdsnewlen("",0); } +/* Create a new sds string starting from a null termined C string. */ sds sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); return sdsnewlen(init, initlen); } +/* Duplicate an sds string. */ sds sdsdup(const sds s) { return sdsnewlen(s, sdslen(s)); } +/* Free an sds string. No operation is performed if 's' is NULL. */ void sdsfree(sds s) { if (s == NULL) return; zfree(s-sizeof(struct sdshdr)); } +/* Set the sds string length to the length as obtained with strlen(), so + * considering as content only up to the first null term character. + * + * This function is useful when the sds string is hacked manually in some + * way, like in the following example: + * + * s = sdsnew("foobar"); + * s[2] = '\0'; + * sdsupdatelen(s); + * printf("%d\n", sdslen(s)); + * + * The output will be "2", but if we comment out the call to sdsupdatelen() + * the output will be "6" as the string was modified but the logical length + * remains 6 bytes. */ void sdsupdatelen(sds s) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); int reallen = strlen(s); @@ -78,6 +109,10 @@ void sdsupdatelen(sds s) { sh->len = reallen; } +/* Modify an sds string on-place to make it empty (zero length). + * However all the existing buffer is not discarded but set as free space + * so that next append operations will not require allocations up to the + * number of bytes previously available. */ void sdsclear(sds s) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); sh->free += sh->len; @@ -89,7 +124,7 @@ void sdsclear(sds s) { * is sure that after calling this function can overwrite up to addlen * bytes after the end of the string, plus one more byte for nul term. * - * Note: this does not change the *size* of the sds string as returned + * Note: this does not change the *length* of the sds string as returned * by sdslen(), but only the free buffer space we have. */ sds sdsMakeRoomFor(sds s, size_t addlen) { struct sdshdr *sh, *newsh; @@ -113,7 +148,10 @@ sds sdsMakeRoomFor(sds s, size_t addlen) { /* Reallocate the sds string so that it has no free space at the end. The * contained string remains not altered, but next concatenation operations - * will require a reallocation. */ + * will require a reallocation. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdsRemoveFreeSpace(sds s) { struct sdshdr *sh; @@ -123,6 +161,13 @@ sds sdsRemoveFreeSpace(sds s) { return sh->buf; } +/* Return the total size of the allocation of the specifed sds string, + * including: + * 1) The sds header before the pointer. + * 2) The string. + * 3) The free buffer at the end if any. + * 4) The implicit null term. + */ size_t sdsAllocSize(sds s) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); @@ -130,7 +175,7 @@ size_t sdsAllocSize(sds s) { } /* Increment the sds length and decrements the left free space at the - * end of the string accordingly to 'incr'. Also set the null term + * end of the string according to 'incr'. Also set the null term * in the new end of the string. * * This function is used in order to fix the string length after the @@ -140,15 +185,17 @@ size_t sdsAllocSize(sds s) { * Note: it is possible to use a negative increment in order to * right-trim the string. * + * Usage example: + * * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the - * following schema to cat bytes coming from the kernel to the end of an - * sds string new things without copying into an intermediate buffer: + * following schema, to cat bytes coming from the kernel to the end of an + * sds string without copying into an intermediate buffer: * * oldlen = sdslen(s); * s = sdsMakeRoomFor(s, BUFFER_SIZE); * nread = read(fd, s+oldlen, BUFFER_SIZE); * ... check for nread <= 0 and handle it ... - * sdsIncrLen(s, nhread); + * sdsIncrLen(s, nread); */ void sdsIncrLen(sds s, int incr) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); @@ -161,7 +208,10 @@ void sdsIncrLen(sds s, int incr) { } /* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. */ + * the original length of the sds will be set to zero. + * + * if the specified length is smaller than the current length, no operation + * is performed. */ sds sdsgrowzero(sds s, size_t len) { struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); size_t totlen, curlen = sh->len; @@ -179,6 +229,11 @@ sds sdsgrowzero(sds s, size_t len) { return s; } +/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the + * end of the specified sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscatlen(sds s, const void *t, size_t len) { struct sdshdr *sh; size_t curlen = sdslen(s); @@ -193,14 +248,24 @@ sds sdscatlen(sds s, const void *t, size_t len) { return s; } +/* Append the specified null termianted C string to the sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscat(sds s, const char *t) { return sdscatlen(s, t, strlen(t)); } +/* Append the specified sds 't' to the existing sds 's'. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscatsds(sds s, const sds t) { return sdscatlen(s, t, sdslen(t)); } +/* Destructively modify the sds string 's' to hold the specified binary + * safe string pointed by 't' of length 'len' bytes. */ sds sdscpylen(sds s, const char *t, size_t len) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t totlen = sh->free+sh->len; @@ -218,10 +283,13 @@ sds sdscpylen(sds s, const char *t, size_t len) { return s; } +/* Like sdscpylen() but 't' must be a null-termined string so that the length + * of the string is obtained with strlen(). */ sds sdscpy(sds s, const char *t) { return sdscpylen(s, t, strlen(t)); } +/* Like sdscatpritf() but gets va_list instead of being variadic. */ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { va_list cpy; char *buf, *t; @@ -245,6 +313,22 @@ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { return t; } +/* Append to the sds string 's' a string obtained using printf-alike format + * specifier. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsempty("Sum is: "); + * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * + * Often you need to create a string from scratch with the printf-alike + * format. When this is the need, just use sdsempty() as the target string: + * + * s = sdscatprintf(sdsempty(), "... your format ...", args); + */ sds sdscatprintf(sds s, const char *fmt, ...) { va_list ap; char *t; @@ -254,6 +338,20 @@ sds sdscatprintf(sds s, const char *fmt, ...) { return t; } +/* Remove the part of the string from left and from right composed just of + * contiguous characters found in 'cset', that is a null terminted C string. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = sdstrim(s,"A. :"); + * printf("%s\n", s); + * + * Output will be just "Hello World". + */ sds sdstrim(sds s, const char *cset) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); char *start, *end, *sp, *ep; @@ -271,6 +369,22 @@ sds sdstrim(sds s, const char *cset) { return s; } +/* Turn the string into a smaller (or equal) string containing only the + * substring specified by the 'start' and 'end' indexes. + * + * start and end can be negative, where -1 means the last character of the + * string, -2 the penultimate character, and so forth. + * + * The interval is inclusive, so the start and end characters will be part + * of the resulting string. + * + * The string is modified in-place. + * + * Example: + * + * s = sdsnew("Hello World"); + * sdstrim(s,1,-1); => "ello Worl" + */ sds sdsrange(sds s, int start, int end) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t newlen, len = sdslen(s); @@ -302,18 +416,31 @@ sds sdsrange(sds s, int start, int end) { return s; } +/* Apply tolower() to every character of the sds string 's'. */ void sdstolower(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } +/* Apply toupper() to every character of the sds string 's'. */ void sdstoupper(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } +/* Compare two sds strings s1 and s2 with memcmp(). + * + * Return value: + * + * 1 if s1 > s2. + * -1 if s1 < s2. + * 0 if s1 and s2 are exactly the same binary string. + * + * If two strings share exactly the same prefix, but one of the two has + * additional characters, the longer string is considered to be greater than + * the smaller one. */ int sdscmp(const sds s1, const sds s2) { size_t l1, l2, minlen; int cmp; @@ -391,6 +518,7 @@ sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count } } +/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ void sdsfreesplitres(sds *tokens, int count) { if (!tokens) return; while(count--) @@ -398,6 +526,10 @@ void sdsfreesplitres(sds *tokens, int count) { zfree(tokens); } +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ sds sdsfromlonglong(long long value) { char buf[32], *p; unsigned long long v; @@ -413,6 +545,12 @@ sds sdsfromlonglong(long long value) { return sdsnewlen(p,32-(p-buf)); } +/* Append to the sds string "s" an escaped string representation where + * all the non-printable characters (tested with isprint()) are turned into + * escapes in the form "\n\r\a...." or "\x". + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscatrepr(sds s, const char *p, size_t len) { s = sdscatlen(s,"\"",1); while(len--) { From a1d37ba46971ed953a0b89148c45d4d1cbe52ca3 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2013 10:37:55 +0200 Subject: [PATCH 0745/1752] Inline protocol improved to accept quoted strings. --- src/networking.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 23ada4541de..992de156bd4 100644 --- a/src/networking.c +++ b/src/networking.c @@ -829,7 +829,7 @@ void resetClient(redisClient *c) { int processInlineBuffer(redisClient *c) { char *newline = strstr(c->querybuf,"\r\n"); int argc, j; - sds *argv; + sds *argv, aux; size_t querylen; /* Nothing to do without a \r\n */ @@ -843,7 +843,9 @@ int processInlineBuffer(redisClient *c) { /* Split the input buffer up to the \r\n */ querylen = newline-(c->querybuf); - argv = sdssplitlen(c->querybuf,querylen," ",1,&argc); + aux = sdsnewlen(c->querybuf,querylen); + argv = sdssplitargs(aux,&argc); + sdsfree(aux); /* Leave data after the first line of the query in the buffer */ c->querybuf = sdsrange(c->querybuf,querylen+2,-1); From f899ab55ca2862246eb0d76bb1f1f0616bb0190e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2013 11:21:39 +0200 Subject: [PATCH 0746/1752] sdsrange() does not need to return a value. Actaully the string is modified in-place and a reallocation is never needed, so there is no need to return the new sds string pointer as return value of the function, that is now just "void". --- src/networking.c | 10 +++++----- src/sds.c | 5 ++--- src/sds.h | 2 +- src/sentinel.c | 2 +- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/networking.c b/src/networking.c index 992de156bd4..22b7dd8e742 100644 --- a/src/networking.c +++ b/src/networking.c @@ -848,7 +848,7 @@ int processInlineBuffer(redisClient *c) { sdsfree(aux); /* Leave data after the first line of the query in the buffer */ - c->querybuf = sdsrange(c->querybuf,querylen+2,-1); + sdsrange(c->querybuf,querylen+2,-1); /* Setup argv array on client structure */ if (c->argv) zfree(c->argv); @@ -877,7 +877,7 @@ static void setProtocolError(redisClient *c, int pos) { sdsfree(client); } c->flags |= REDIS_CLOSE_AFTER_REPLY; - c->querybuf = sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,pos,-1); } int processMultibulkBuffer(redisClient *c) { @@ -915,7 +915,7 @@ int processMultibulkBuffer(redisClient *c) { pos = (newline-c->querybuf)+2; if (ll <= 0) { - c->querybuf = sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,pos,-1); return REDIS_OK; } @@ -964,7 +964,7 @@ int processMultibulkBuffer(redisClient *c) { * try to make it likely that it will start at c->querybuf * boundary so that we can optimized object creation * avoiding a large copy of data. */ - c->querybuf = sdsrange(c->querybuf,pos,-1); + sdsrange(c->querybuf,pos,-1); pos = 0; /* Hint the sds library about the amount of bytes this string is * going to contain. */ @@ -1003,7 +1003,7 @@ int processMultibulkBuffer(redisClient *c) { } /* Trim to pos */ - if (pos) c->querybuf = sdsrange(c->querybuf,pos,-1); + if (pos) sdsrange(c->querybuf,pos,-1); /* We're done when c->multibulk == 0 */ if (c->multibulklen == 0) return REDIS_OK; diff --git a/src/sds.c b/src/sds.c index b63f44d940b..d66c1d7301e 100644 --- a/src/sds.c +++ b/src/sds.c @@ -385,11 +385,11 @@ sds sdstrim(sds s, const char *cset) { * s = sdsnew("Hello World"); * sdstrim(s,1,-1); => "ello Worl" */ -sds sdsrange(sds s, int start, int end) { +void sdsrange(sds s, int start, int end) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t newlen, len = sdslen(s); - if (len == 0) return s; + if (len == 0) return; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -413,7 +413,6 @@ sds sdsrange(sds s, int start, int end) { sh->buf[newlen] = 0; sh->free = sh->free+(sh->len-newlen); sh->len = newlen; - return s; } /* Apply tolower() to every character of the sds string 's'. */ diff --git a/src/sds.h b/src/sds.h index 46d914fd145..6f320113060 100644 --- a/src/sds.h +++ b/src/sds.h @@ -77,7 +77,7 @@ sds sdscatprintf(sds s, const char *fmt, ...); #endif sds sdstrim(sds s, const char *cset); -sds sdsrange(sds s, int start, int end); +void sdsrange(sds s, int start, int end); void sdsupdatelen(sds s); void sdsclear(sds s); int sdscmp(const sds s1, const sds s2); diff --git a/src/sentinel.c b/src/sentinel.c index 659de29b8be..b257ad685ed 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1920,7 +1920,7 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { if (ri->flags & SRI_RECONF_DONE) flags = sdscat(flags,"reconf_done,"); if (ri->flags & SRI_DEMOTE) flags = sdscat(flags,"demote,"); - if (sdslen(flags) != 0) flags = sdsrange(flags,0,-2); /* remove last "," */ + if (sdslen(flags) != 0) sdsrange(flags,0,-2); /* remove last "," */ addReplyBulkCString(c,flags); sdsfree(flags); fields++; From c3e1724a573d7632473cb2e3ff19cdd69d46e534 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 24 Jul 2013 18:59:36 +0200 Subject: [PATCH 0747/1752] Ignore sdsrange return value. --- src/util.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/util.c b/src/util.c index 022a6adf4cc..37534dfb938 100644 --- a/src/util.c +++ b/src/util.c @@ -438,7 +438,7 @@ sds getAbsolutePath(char *filename) { while (sdslen(relpath) >= 3 && relpath[0] == '.' && relpath[1] == '.' && relpath[2] == '/') { - relpath = sdsrange(relpath,3,-1); + sdsrange(relpath,3,-1); if (sdslen(abspath) > 1) { char *p = abspath + sdslen(abspath)-2; int trimlen = 1; @@ -447,7 +447,7 @@ sds getAbsolutePath(char *filename) { p--; trimlen++; } - abspath = sdsrange(abspath,0,-(trimlen+1)); + sdsrange(abspath,0,-(trimlen+1)); } } From aea8020554feceafc831b18273cbe4e512a5094e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 25 Jul 2013 10:32:19 +0200 Subject: [PATCH 0748/1752] Use latest sds.c in the hiredis library under deps. --- deps/hiredis/hiredis.c | 4 +- deps/hiredis/sds.c | 495 ++++++++++++++++++++++++++++++++--------- deps/hiredis/sds.h | 30 ++- deps/hiredis/zmalloc.h | 13 ++ 4 files changed, 422 insertions(+), 120 deletions(-) create mode 100644 deps/hiredis/zmalloc.h diff --git a/deps/hiredis/hiredis.c b/deps/hiredis/hiredis.c index 4709ee32530..0b04935a1b2 100644 --- a/deps/hiredis/hiredis.c +++ b/deps/hiredis/hiredis.c @@ -650,7 +650,7 @@ int redisReaderGetReply(redisReader *r, void **reply) { /* Discard part of the buffer when we've consumed at least 1k, to avoid * doing unnecessary calls to memmove() in sds.c. */ if (r->pos >= 1024) { - r->buf = sdsrange(r->buf,r->pos,-1); + sdsrange(r->buf,r->pos,-1); r->pos = 0; r->len = sdslen(r->buf); } @@ -1125,7 +1125,7 @@ int redisBufferWrite(redisContext *c, int *done) { sdsfree(c->obuf); c->obuf = sdsempty(); } else { - c->obuf = sdsrange(c->obuf,nwritten,-1); + sdsrange(c->obuf,nwritten,-1); } } } diff --git a/deps/hiredis/sds.c b/deps/hiredis/sds.c index 0af9c672093..d66c1d7301e 100644 --- a/deps/hiredis/sds.c +++ b/deps/hiredis/sds.c @@ -1,6 +1,6 @@ /* SDSLib, A C dynamic strings library * - * Copyright (c) 2006-2010, Salvatore Sanfilippo + * Copyright (c) 2006-2012, Salvatore Sanfilippo * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,52 +32,76 @@ #include #include #include +#include #include "sds.h" +#include "zmalloc.h" -#ifdef SDS_ABORT_ON_OOM -static void sdsOomAbort(void) { - fprintf(stderr,"SDS: Out Of Memory (SDS_ABORT_ON_OOM defined)\n"); - abort(); -} -#endif - +/* Create a new sds string with the content specified by the 'init' pointer + * and 'initlen'. + * If NULL is used for 'init' the string is initialized with zero bytes. + * + * The string is always null-termined (all the sds strings are, always) so + * even if you create an sds string with: + * + * mystring = sdsnewlen("abc",3"); + * + * You can print the string with printf() as there is an implicit \0 at the + * end of the string. However the string is binary safe and can contain + * \0 characters in the middle, as the length is stored in the sds header. */ sds sdsnewlen(const void *init, size_t initlen) { struct sdshdr *sh; - sh = malloc(sizeof(struct sdshdr)+initlen+1); -#ifdef SDS_ABORT_ON_OOM - if (sh == NULL) sdsOomAbort(); -#else + if (init) { + sh = zmalloc(sizeof(struct sdshdr)+initlen+1); + } else { + sh = zcalloc(sizeof(struct sdshdr)+initlen+1); + } if (sh == NULL) return NULL; -#endif sh->len = initlen; sh->free = 0; - if (initlen) { - if (init) memcpy(sh->buf, init, initlen); - else memset(sh->buf,0,initlen); - } + if (initlen && init) + memcpy(sh->buf, init, initlen); sh->buf[initlen] = '\0'; return (char*)sh->buf; } +/* Create an empty (zero length) sds string. Even in this case the string + * always has an implicit null term. */ sds sdsempty(void) { return sdsnewlen("",0); } +/* Create a new sds string starting from a null termined C string. */ sds sdsnew(const char *init) { size_t initlen = (init == NULL) ? 0 : strlen(init); return sdsnewlen(init, initlen); } +/* Duplicate an sds string. */ sds sdsdup(const sds s) { return sdsnewlen(s, sdslen(s)); } +/* Free an sds string. No operation is performed if 's' is NULL. */ void sdsfree(sds s) { if (s == NULL) return; - free(s-sizeof(struct sdshdr)); + zfree(s-sizeof(struct sdshdr)); } +/* Set the sds string length to the length as obtained with strlen(), so + * considering as content only up to the first null term character. + * + * This function is useful when the sds string is hacked manually in some + * way, like in the following example: + * + * s = sdsnew("foobar"); + * s[2] = '\0'; + * sdsupdatelen(s); + * printf("%d\n", sdslen(s)); + * + * The output will be "2", but if we comment out the call to sdsupdatelen() + * the output will be "6" as the string was modified but the logical length + * remains 6 bytes. */ void sdsupdatelen(sds s) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); int reallen = strlen(s); @@ -85,7 +109,24 @@ void sdsupdatelen(sds s) { sh->len = reallen; } -static sds sdsMakeRoomFor(sds s, size_t addlen) { +/* Modify an sds string on-place to make it empty (zero length). + * However all the existing buffer is not discarded but set as free space + * so that next append operations will not require allocations up to the + * number of bytes previously available. */ +void sdsclear(sds s) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + sh->free += sh->len; + sh->len = 0; + sh->buf[0] = '\0'; +} + +/* Enlarge the free space at the end of the sds string so that the caller + * is sure that after calling this function can overwrite up to addlen + * bytes after the end of the string, plus one more byte for nul term. + * + * Note: this does not change the *length* of the sds string as returned + * by sdslen(), but only the free buffer space we have. */ +sds sdsMakeRoomFor(sds s, size_t addlen) { struct sdshdr *sh, *newsh; size_t free = sdsavail(s); size_t len, newlen; @@ -93,20 +134,84 @@ static sds sdsMakeRoomFor(sds s, size_t addlen) { if (free >= addlen) return s; len = sdslen(s); sh = (void*) (s-(sizeof(struct sdshdr))); - newlen = (len+addlen)*2; - newsh = realloc(sh, sizeof(struct sdshdr)+newlen+1); -#ifdef SDS_ABORT_ON_OOM - if (newsh == NULL) sdsOomAbort(); -#else + newlen = (len+addlen); + if (newlen < SDS_MAX_PREALLOC) + newlen *= 2; + else + newlen += SDS_MAX_PREALLOC; + newsh = zrealloc(sh, sizeof(struct sdshdr)+newlen+1); if (newsh == NULL) return NULL; -#endif newsh->free = newlen - len; return newsh->buf; } +/* Reallocate the sds string so that it has no free space at the end. The + * contained string remains not altered, but next concatenation operations + * will require a reallocation. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdsRemoveFreeSpace(sds s) { + struct sdshdr *sh; + + sh = (void*) (s-(sizeof(struct sdshdr))); + sh = zrealloc(sh, sizeof(struct sdshdr)+sh->len+1); + sh->free = 0; + return sh->buf; +} + +/* Return the total size of the allocation of the specifed sds string, + * including: + * 1) The sds header before the pointer. + * 2) The string. + * 3) The free buffer at the end if any. + * 4) The implicit null term. + */ +size_t sdsAllocSize(sds s) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + + return sizeof(*sh)+sh->len+sh->free+1; +} + +/* Increment the sds length and decrements the left free space at the + * end of the string according to 'incr'. Also set the null term + * in the new end of the string. + * + * This function is used in order to fix the string length after the + * user calls sdsMakeRoomFor(), writes something after the end of + * the current string, and finally needs to set the new length. + * + * Note: it is possible to use a negative increment in order to + * right-trim the string. + * + * Usage example: + * + * Using sdsIncrLen() and sdsMakeRoomFor() it is possible to mount the + * following schema, to cat bytes coming from the kernel to the end of an + * sds string without copying into an intermediate buffer: + * + * oldlen = sdslen(s); + * s = sdsMakeRoomFor(s, BUFFER_SIZE); + * nread = read(fd, s+oldlen, BUFFER_SIZE); + * ... check for nread <= 0 and handle it ... + * sdsIncrLen(s, nread); + */ +void sdsIncrLen(sds s, int incr) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + + assert(sh->free >= incr); + sh->len += incr; + sh->free -= incr; + assert(sh->free >= 0); + s[sh->len] = '\0'; +} + /* Grow the sds to have the specified length. Bytes that were not part of - * the original length of the sds will be set to zero. */ + * the original length of the sds will be set to zero. + * + * if the specified length is smaller than the current length, no operation + * is performed. */ sds sdsgrowzero(sds s, size_t len) { struct sdshdr *sh = (void*)(s-(sizeof(struct sdshdr))); size_t totlen, curlen = sh->len; @@ -124,6 +229,11 @@ sds sdsgrowzero(sds s, size_t len) { return s; } +/* Append the specified binary-safe string pointed by 't' of 'len' bytes to the + * end of the specified sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscatlen(sds s, const void *t, size_t len) { struct sdshdr *sh; size_t curlen = sdslen(s); @@ -138,11 +248,25 @@ sds sdscatlen(sds s, const void *t, size_t len) { return s; } +/* Append the specified null termianted C string to the sds string 's'. + * + * After the call, the passed sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ sds sdscat(sds s, const char *t) { return sdscatlen(s, t, strlen(t)); } -sds sdscpylen(sds s, char *t, size_t len) { +/* Append the specified sds 't' to the existing sds 's'. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatsds(sds s, const sds t) { + return sdscatlen(s, t, sdslen(t)); +} + +/* Destructively modify the sds string 's' to hold the specified binary + * safe string pointed by 't' of length 'len' bytes. */ +sds sdscpylen(sds s, const char *t, size_t len) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t totlen = sh->free+sh->len; @@ -159,37 +283,52 @@ sds sdscpylen(sds s, char *t, size_t len) { return s; } -sds sdscpy(sds s, char *t) { +/* Like sdscpylen() but 't' must be a null-termined string so that the length + * of the string is obtained with strlen(). */ +sds sdscpy(sds s, const char *t) { return sdscpylen(s, t, strlen(t)); } +/* Like sdscatpritf() but gets va_list instead of being variadic. */ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { va_list cpy; char *buf, *t; size_t buflen = 16; while(1) { - buf = malloc(buflen); -#ifdef SDS_ABORT_ON_OOM - if (buf == NULL) sdsOomAbort(); -#else + buf = zmalloc(buflen); if (buf == NULL) return NULL; -#endif buf[buflen-2] = '\0'; va_copy(cpy,ap); vsnprintf(buf, buflen, fmt, cpy); if (buf[buflen-2] != '\0') { - free(buf); + zfree(buf); buflen *= 2; continue; } break; } t = sdscat(s, buf); - free(buf); + zfree(buf); return t; } +/* Append to the sds string 's' a string obtained using printf-alike format + * specifier. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsempty("Sum is: "); + * s = sdscatprintf(s,"%d+%d = %d",a,b,a+b). + * + * Often you need to create a string from scratch with the printf-alike + * format. When this is the need, just use sdsempty() as the target string: + * + * s = sdscatprintf(sdsempty(), "... your format ...", args); + */ sds sdscatprintf(sds s, const char *fmt, ...) { va_list ap; char *t; @@ -199,6 +338,20 @@ sds sdscatprintf(sds s, const char *fmt, ...) { return t; } +/* Remove the part of the string from left and from right composed just of + * contiguous characters found in 'cset', that is a null terminted C string. + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. + * + * Example: + * + * s = sdsnew("AA...AA.a.aa.aHelloWorld :::"); + * s = sdstrim(s,"A. :"); + * printf("%s\n", s); + * + * Output will be just "Hello World". + */ sds sdstrim(sds s, const char *cset) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); char *start, *end, *sp, *ep; @@ -216,11 +369,27 @@ sds sdstrim(sds s, const char *cset) { return s; } -sds sdsrange(sds s, int start, int end) { +/* Turn the string into a smaller (or equal) string containing only the + * substring specified by the 'start' and 'end' indexes. + * + * start and end can be negative, where -1 means the last character of the + * string, -2 the penultimate character, and so forth. + * + * The interval is inclusive, so the start and end characters will be part + * of the resulting string. + * + * The string is modified in-place. + * + * Example: + * + * s = sdsnew("Hello World"); + * sdstrim(s,1,-1); => "ello Worl" + */ +void sdsrange(sds s, int start, int end) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); size_t newlen, len = sdslen(s); - if (len == 0) return s; + if (len == 0) return; if (start < 0) { start = len+start; if (start < 0) start = 0; @@ -244,22 +413,34 @@ sds sdsrange(sds s, int start, int end) { sh->buf[newlen] = 0; sh->free = sh->free+(sh->len-newlen); sh->len = newlen; - return s; } +/* Apply tolower() to every character of the sds string 's'. */ void sdstolower(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = tolower(s[j]); } +/* Apply toupper() to every character of the sds string 's'. */ void sdstoupper(sds s) { int len = sdslen(s), j; for (j = 0; j < len; j++) s[j] = toupper(s[j]); } -int sdscmp(sds s1, sds s2) { +/* Compare two sds strings s1 and s2 with memcmp(). + * + * Return value: + * + * 1 if s1 > s2. + * -1 if s1 < s2. + * 0 if s1 and s2 are exactly the same binary string. + * + * If two strings share exactly the same prefix, but one of the two has + * additional characters, the longer string is considered to be greater than + * the smaller one. */ +int sdscmp(const sds s1, const sds s2) { size_t l1, l2, minlen; int cmp; @@ -287,14 +468,15 @@ int sdscmp(sds s1, sds s2) { * requires length arguments. sdssplit() is just the * same function but for zero-terminated strings. */ -sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count) { int elements = 0, slots = 5, start = 0, j; + sds *tokens; + + if (seplen < 1 || len < 0) return NULL; + + tokens = zmalloc(sizeof(sds)*slots); + if (tokens == NULL) return NULL; - sds *tokens = malloc(sizeof(sds)*slots); -#ifdef SDS_ABORT_ON_OOM - if (tokens == NULL) sdsOomAbort(); -#endif - if (seplen < 1 || len < 0 || tokens == NULL) return NULL; if (len == 0) { *count = 0; return tokens; @@ -305,26 +487,14 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { sds *newtokens; slots *= 2; - newtokens = realloc(tokens,sizeof(sds)*slots); - if (newtokens == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + newtokens = zrealloc(tokens,sizeof(sds)*slots); + if (newtokens == NULL) goto cleanup; tokens = newtokens; } /* search the separator */ if ((seplen == 1 && *(s+j) == sep[0]) || (memcmp(s+j,sep,seplen) == 0)) { tokens[elements] = sdsnewlen(s+start,j-start); - if (tokens[elements] == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + if (tokens[elements] == NULL) goto cleanup; elements++; start = j+seplen; j = j+seplen-1; /* skip the separator */ @@ -332,35 +502,33 @@ sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count) { } /* Add the final element. We are sure there is room in the tokens array. */ tokens[elements] = sdsnewlen(s+start,len-start); - if (tokens[elements] == NULL) { -#ifdef SDS_ABORT_ON_OOM - sdsOomAbort(); -#else - goto cleanup; -#endif - } + if (tokens[elements] == NULL) goto cleanup; elements++; *count = elements; return tokens; -#ifndef SDS_ABORT_ON_OOM cleanup: { int i; for (i = 0; i < elements; i++) sdsfree(tokens[i]); - free(tokens); + zfree(tokens); + *count = 0; return NULL; } -#endif } +/* Free the result returned by sdssplitlen(), or do nothing if 'tokens' is NULL. */ void sdsfreesplitres(sds *tokens, int count) { if (!tokens) return; while(count--) sdsfree(tokens[count]); - free(tokens); + zfree(tokens); } +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ sds sdsfromlonglong(long long value) { char buf[32], *p; unsigned long long v; @@ -376,10 +544,14 @@ sds sdsfromlonglong(long long value) { return sdsnewlen(p,32-(p-buf)); } -sds sdscatrepr(sds s, char *p, size_t len) { +/* Append to the sds string "s" an escaped string representation where + * all the non-printable characters (tested with isprint()) are turned into + * escapes in the form "\n\r\a...." or "\x". + * + * After the call, the modified sds string is no longer valid and all the + * references must be substituted with the new pointer returned by the call. */ +sds sdscatrepr(sds s, const char *p, size_t len) { s = sdscatlen(s,"\"",1); - if (s == NULL) return NULL; - while(len--) { switch(*p) { case '\\': @@ -399,27 +571,64 @@ sds sdscatrepr(sds s, char *p, size_t len) { break; } p++; - if (s == NULL) return NULL; } return sdscatlen(s,"\"",1); } +/* Helper function for sdssplitargs() that returns non zero if 'c' + * is a valid hex digit. */ +int is_hex_digit(char c) { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F'); +} + +/* Helper function for sdssplitargs() that converts an hex digit into an + * integer from 0 to 15 */ +int hex_digit_to_int(char c) { + switch(c) { + case '0': return 0; + case '1': return 1; + case '2': return 2; + case '3': return 3; + case '4': return 4; + case '5': return 5; + case '6': return 6; + case '7': return 7; + case '8': return 8; + case '9': return 9; + case 'a': case 'A': return 10; + case 'b': case 'B': return 11; + case 'c': case 'C': return 12; + case 'd': case 'D': return 13; + case 'e': case 'E': return 14; + case 'f': case 'F': return 15; + default: return 0; + } +} + /* Split a line into arguments, where every argument can be in the * following programming-language REPL-alike form: * * foo bar "newline are supported\n" and "\xff\x00otherstuff" * * The number of arguments is stored into *argc, and an array - * of sds is returned. The caller should sdsfree() all the returned - * strings and finally free() the array itself. + * of sds is returned. + * + * The caller should free the resulting array of sds strings with + * sdsfreesplitres(). * * Note that sdscatrepr() is able to convert back a string into * a quoted string in the same format sdssplitargs() is able to parse. + * + * The function returns the allocated tokens on success, even when the + * input string is empty, or NULL if the input contains unbalanced + * quotes or closed quotes followed by non space characters + * as in: "foo"bar or "foo' */ -sds *sdssplitargs(char *line, int *argc) { - char *p = line; +sds *sdssplitargs(const char *line, int *argc) { + const char *p = line; char *current = NULL; - char **vector = NULL, **_vector = NULL; + char **vector = NULL; *argc = 0; while(1) { @@ -427,17 +636,24 @@ sds *sdssplitargs(char *line, int *argc) { while(*p && isspace(*p)) p++; if (*p) { /* get a token */ - int inq=0; /* set to 1 if we are in "quotes" */ + int inq=0; /* set to 1 if we are in "quotes" */ + int insq=0; /* set to 1 if we are in 'single quotes' */ int done=0; - if (current == NULL) { - current = sdsempty(); - if (current == NULL) goto err; - } - + if (current == NULL) current = sdsempty(); while(!done) { if (inq) { - if (*p == '\\' && *(p+1)) { + if (*p == '\\' && *(p+1) == 'x' && + is_hex_digit(*(p+2)) && + is_hex_digit(*(p+3))) + { + unsigned char byte; + + byte = (hex_digit_to_int(*(p+2))*16)+ + hex_digit_to_int(*(p+3)); + current = sdscatlen(current,(char*)&byte,1); + p += 3; + } else if (*p == '\\' && *(p+1)) { char c; p++; @@ -451,7 +667,23 @@ sds *sdssplitargs(char *line, int *argc) { } current = sdscatlen(current,&c,1); } else if (*p == '"') { - /* closing quote must be followed by a space */ + /* closing quote must be followed by a space or + * nothing at all. */ + if (*(p+1) && !isspace(*(p+1))) goto err; + done=1; + } else if (!*p) { + /* unterminated quotes */ + goto err; + } else { + current = sdscatlen(current,p,1); + } + } else if (insq) { + if (*p == '\\' && *(p+1) == '\'') { + p++; + current = sdscatlen(current,"'",1); + } else if (*p == '\'') { + /* closing quote must be followed by a space or + * nothing at all. */ if (*(p+1) && !isspace(*(p+1))) goto err; done=1; } else if (!*p) { @@ -472,23 +704,24 @@ sds *sdssplitargs(char *line, int *argc) { case '"': inq=1; break; + case '\'': + insq=1; + break; default: current = sdscatlen(current,p,1); break; } } if (*p) p++; - if (current == NULL) goto err; } /* add the token to the vector */ - _vector = realloc(vector,((*argc)+1)*sizeof(char*)); - if (_vector == NULL) goto err; - - vector = _vector; + vector = zrealloc(vector,((*argc)+1)*sizeof(char*)); vector[*argc] = current; (*argc)++; current = NULL; } else { + /* Even on empty input string return something not NULL. */ + if (vector == NULL) vector = zmalloc(sizeof(void*)); return vector; } } @@ -496,30 +729,55 @@ sds *sdssplitargs(char *line, int *argc) { err: while((*argc)--) sdsfree(vector[*argc]); - if (vector != NULL) free(vector); - if (current != NULL) sdsfree(current); + zfree(vector); + if (current) sdsfree(current); + *argc = 0; return NULL; } +/* Modify the string substituting all the occurrences of the set of + * characters specified in the 'from' string to the corresponding character + * in the 'to' array. + * + * For instance: sdsmapchars(mystring, "ho", "01", 2) + * will have the effect of turning the string "hello" into "0ell1". + * + * The function returns the sds string pointer, that is always the same + * as the input pointer since no resize is needed. */ +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen) { + size_t j, i, l = sdslen(s); + + for (j = 0; j < l; j++) { + for (i = 0; i < setlen; i++) { + if (s[j] == from[i]) { + s[j] = to[i]; + break; + } + } + } + return s; +} + +/* Join an array of C strings using the specified separator (also a C string). + * Returns the result as an sds string. */ +sds sdsjoin(char **argv, int argc, char *sep) { + sds join = sdsempty(); + int j; + + for (j = 0; j < argc; j++) { + join = sdscat(join, argv[j]); + if (j != argc-1) join = sdscat(join,sep); + } + return join; +} + #ifdef SDS_TEST_MAIN #include - -int __failed_tests = 0; -int __test_num = 0; -#define test_cond(descr,_c) do { \ - __test_num++; printf("%d - %s: ", __test_num, descr); \ - if(_c) printf("PASSED\n"); else {printf("FAILED\n"); __failed_tests++;} \ -} while(0); -#define test_report() do { \ - printf("%d tests, %d passed, %d failed\n", __test_num, \ - __test_num-__failed_tests, __failed_tests); \ - if (__failed_tests) { \ - printf("=== WARNING === We have failed tests here...\n"); \ - } \ -} while(0); +#include "testhelp.h" int main(void) { { + struct sdshdr *sh; sds x = sdsnew("foo"), y; test_cond("Create a string and obtain the length", @@ -599,7 +857,26 @@ int main(void) { x = sdsnew("aar"); y = sdsnew("bar"); test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + + { + int oldfree; + + sdsfree(x); + x = sdsnew("0"); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsnew() free/len buffers", sh->len == 1 && sh->free == 0); + x = sdsMakeRoomFor(x,1); + sh = (void*) (x-(sizeof(struct sdshdr))); + test_cond("sdsMakeRoomFor()", sh->len == 1 && sh->free > 0); + oldfree = sh->free; + x[1] = '1'; + sdsIncrLen(x,1); + test_cond("sdsIncrLen() -- content", x[0] == '0' && x[1] == '1'); + test_cond("sdsIncrLen() -- len", sh->len == 2); + test_cond("sdsIncrLen() -- free", sh->free == oldfree-1); + } } test_report() + return 0; } #endif diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 94f5871f54f..6f320113060 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -31,6 +31,8 @@ #ifndef __SDS_H #define __SDS_H +#define SDS_MAX_PREALLOC (1024*1024) + #include #include @@ -54,16 +56,17 @@ static inline size_t sdsavail(const sds s) { sds sdsnewlen(const void *init, size_t initlen); sds sdsnew(const char *init); -sds sdsempty(void); +sds sdsempty(); size_t sdslen(const sds s); sds sdsdup(const sds s); void sdsfree(sds s); -size_t sdsavail(sds s); +size_t sdsavail(const sds s); sds sdsgrowzero(sds s, size_t len); sds sdscatlen(sds s, const void *t, size_t len); sds sdscat(sds s, const char *t); -sds sdscpylen(sds s, char *t, size_t len); -sds sdscpy(sds s, char *t); +sds sdscatsds(sds s, const sds t); +sds sdscpylen(sds s, const char *t, size_t len); +sds sdscpy(sds s, const char *t); sds sdscatvprintf(sds s, const char *fmt, va_list ap); #ifdef __GNUC__ @@ -74,15 +77,24 @@ sds sdscatprintf(sds s, const char *fmt, ...); #endif sds sdstrim(sds s, const char *cset); -sds sdsrange(sds s, int start, int end); +void sdsrange(sds s, int start, int end); void sdsupdatelen(sds s); -int sdscmp(sds s1, sds s2); -sds *sdssplitlen(char *s, int len, char *sep, int seplen, int *count); +void sdsclear(sds s); +int sdscmp(const sds s1, const sds s2); +sds *sdssplitlen(const char *s, int len, const char *sep, int seplen, int *count); void sdsfreesplitres(sds *tokens, int count); void sdstolower(sds s); void sdstoupper(sds s); sds sdsfromlonglong(long long value); -sds sdscatrepr(sds s, char *p, size_t len); -sds *sdssplitargs(char *line, int *argc); +sds sdscatrepr(sds s, const char *p, size_t len); +sds *sdssplitargs(const char *line, int *argc); +sds sdsmapchars(sds s, const char *from, const char *to, size_t setlen); +sds sdsjoin(char **argv, int argc, char *sep); + +/* Low level functions exposed to the user API */ +sds sdsMakeRoomFor(sds s, size_t addlen); +void sdsIncrLen(sds s, int incr); +sds sdsRemoveFreeSpace(sds s); +size_t sdsAllocSize(sds s); #endif diff --git a/deps/hiredis/zmalloc.h b/deps/hiredis/zmalloc.h new file mode 100644 index 00000000000..99b87ace9ff --- /dev/null +++ b/deps/hiredis/zmalloc.h @@ -0,0 +1,13 @@ +/* Drop in replacement for zmalloc.h in order to just use libc malloc without + * any wrappering. */ + +#ifndef ZMALLOC_H +#define ZMALLOC_H + +#define zmalloc malloc +#define zrealloc realloc +#define zcalloc(x) calloc(x,1) +#define zfree free +#define zstrdup strdup + +#endif From 78644f5e3ec3b970d9a6e30c339af7098ea30e58 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 28 Jul 2013 11:00:09 +0200 Subject: [PATCH 0749/1752] Remove dead variable bothsds from object.c. Thanks to @run and @badboy for spotting this. Triva: clang was not able to provide me a warning about that when compiling. This closes #1024 and #1207, committing the change myself as the pull requests no longer apply cleanly after other changes to the same function. --- src/object.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/object.c b/src/object.c index 472d4a34bbb..5202c0c55f7 100644 --- a/src/object.c +++ b/src/object.c @@ -347,13 +347,11 @@ int compareStringObjectsWithFlags(robj *a, robj *b, int flags) { redisAssertWithInfo(NULL,a,a->type == REDIS_STRING && b->type == REDIS_STRING); char bufa[128], bufb[128], *astr, *bstr; size_t alen, blen, minlen; - int bothsds = 1; if (a == b) return 0; if (a->encoding != REDIS_ENCODING_RAW) { alen = ll2string(bufa,sizeof(bufa),(long) a->ptr); astr = bufa; - bothsds = 0; } else { astr = a->ptr; alen = sdslen(astr); @@ -361,7 +359,6 @@ int compareStringObjectsWithFlags(robj *a, robj *b, int flags) { if (b->encoding != REDIS_ENCODING_RAW) { blen = ll2string(bufb,sizeof(bufb),(long) b->ptr); bstr = bufb; - bothsds = 0; } else { bstr = b->ptr; blen = sdslen(bstr); From 13f7ade55176ec402010f81ef058cb78b42d5ada Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 28 Jul 2013 12:49:07 +0200 Subject: [PATCH 0750/1752] Fix replicationFeedSlaves() off-by-one bug. This fixes issue #1221. --- src/replication.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/replication.c b/src/replication.c index 20b03e00a86..900cb4ea606 100644 --- a/src/replication.c +++ b/src/replication.c @@ -188,7 +188,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { b[0] = '*'; len = ll2string(b+1,REDIS_LONGSTR_SIZE,argc); b += len+1; - buf_left -= len; + buf_left -= len+1; b[0] = '\r'; b[1] = '\n'; b += 2; @@ -218,7 +218,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { b[0] = '$'; len = ll2string(b+1,REDIS_LONGSTR_SIZE,objlen); b += len+1; - buf_left -= len; + buf_left -= len+1; b[0] = '\r'; b[1] = '\n'; b += 2; From f8e43aba783153092717349e77ec4bfde9353203 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 29 Jul 2013 17:39:19 +0200 Subject: [PATCH 0751/1752] Test: regression test for issue #1221. --- tests/integration/replication-4.tcl | 38 +++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/integration/replication-4.tcl b/tests/integration/replication-4.tcl index f84369f44da..6db9ffe2bce 100644 --- a/tests/integration/replication-4.tcl +++ b/tests/integration/replication-4.tcl @@ -96,3 +96,41 @@ start_server {tags {"repl"}} { } {NOREPLICAS*} } } + +start_server {tags {"repl"}} { + start_server {} { + set master [srv -1 client] + set master_host [srv -1 host] + set master_port [srv -1 port] + set slave [srv 0 client] + + test {First server should have role slave after SLAVEOF} { + $slave slaveof $master_host $master_port + wait_for_condition 50 100 { + [s 0 role] eq {slave} + } else { + fail "Replication not started." + } + } + + test {Replication: commands with many arguments (issue #1221)} { + # We now issue large MSET commands, that may trigger a specific + # class of bugs, see issue #1221. + for {set j 0} {$j < 100} {incr j} { + set cmd [list mset] + for {set x 0} {$x < 1000} {incr x} { + lappend cmd [randomKey] [randomValue] + } + $master {*}$cmd + } + + set retry 10 + while {$retry && ([$master debug digest] ne [$slave debug digest])}\ + { + after 1000 + incr retry -1 + } + assert {[$master dbsize] > 0} + } + } +} From cc946972840e071c7292cb28717df174807f2ea6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 30 Jul 2013 20:20:02 +0200 Subject: [PATCH 0752/1752] Redis 2.7.102 (2.8 Release Candidate 2). --- 00-RELEASENOTES | 11 +++++++++++ src/version.h | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index d543473fa88..c85366aadfe 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,17 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8 Release Candidate 2 (2.7.102) ] Release date: 30 Jul 2013 + +This is the second release candidate of Redis 2.8 (official version is 2.7.102). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH + +* [FIX] Fixed a critical replication bug, see issue #1221. +* [NEW] The new inline protocol now accepts quoted strings like, for example + you can now type in a telnet session: set 'foo bar' "hello world\n". + --[ Redis 2.8 Release Candidate 1 (2.7.101) ] Release date: 18 Jul 2013 This is the first release candidate of Redis 2.8 (official version is 2.7.101). diff --git a/src/version.h b/src/version.h index 5b8d18d05e6..7f5043d67f8 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.101" +#define REDIS_VERSION "2.7.102" From 500155b91bb9987ea73a38aa0900fbf86d0bb9f4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 5 Aug 2013 12:05:22 +0200 Subject: [PATCH 0753/1752] Draft #1 of a new expired keys collection algorithm. The main idea here is that when we are no longer to expire keys at the rate the are created, we can't block more in the normal expire cycle as this would result in too big latency spikes. For this reason the commit introduces a "fast" expire cycle that does not run for more than 1 millisecond but is called in the beforeSleep() hook of the event loop, so much more often, and with a frequency bound to the frequency of executed commnads. The fast expire cycle is only called when the standard expiration algorithm runs out of time, that is, consumed more than REDIS_EXPIRELOOKUPS_TIME_PERC of CPU in a given cycle without being able to take the number of already expired keys that are yet not collected to a number smaller than 25% of the number of keys. You can test this commit with different loads, but a simple way is to use the following: Extreme load with pipelining: redis-benchmark -r 100000000 -n 100000000 \ -P 32 set ele:rand:000000000000 foo ex 2 Remove the -P32 in order to avoid the pipelining for a more real-world load. In another terminal tab you can monitor the Redis behavior with: redis-cli -i 0.1 -r -1 info keyspace and redis-cli --latency-history Note: this commit will make Redis printing a lot of debug messages, it is not a good idea to use it in production. --- src/redis.c | 116 +++++++++++++++++++++++++++++++++++++++++++--------- src/redis.h | 5 ++- 2 files changed, 100 insertions(+), 21 deletions(-) diff --git a/src/redis.c b/src/redis.c index 3d0c6a5499a..e29cb241efa 100644 --- a/src/redis.c +++ b/src/redis.c @@ -628,14 +628,51 @@ void updateDictResizePolicy(void) { /* ======================= Cron: called every 100 ms ======================== */ +/* Helper function for the activeExpireCycle() function. + * This function will try to expire the key that is stored in the hash table + * entry 'de' of the 'expires' hash table of a Redis database. + * + * If the key is found to be expired, it is removed from the database and + * 1 is returned. Otherwise no operation is performed and 0 is returned. + * + * When a key is expired, server.stat_expiredkeys is incremented. + * + * The parameter 'now' is the current time in milliseconds as is passed + * to the function to avoid too many gettimeofday() syscalls. */ +int activeExpireCycleTryExpire(redisDb *db, struct dictEntry *de, long long now) { + long long t = dictGetSignedIntegerVal(de); + if (now > t) { + sds key = dictGetKey(de); + robj *keyobj = createStringObject(key,sdslen(key)); + + propagateExpire(db,keyobj); + dbDelete(db,keyobj); + notifyKeyspaceEvent(REDIS_NOTIFY_EXPIRED, + "expired",keyobj,db->id); + decrRefCount(keyobj); + server.stat_expiredkeys++; + return 1; + } else { + return 0; + } +} + /* Try to expire a few timed out keys. The algorithm used is adaptive and * will use few CPU cycles if there are few expiring keys, otherwise * it will get more aggressive to avoid that too much memory is used by * keys that can be removed from the keyspace. * * No more than REDIS_DBCRON_DBS_PER_CALL databases are tested at every - * iteration. */ -void activeExpireCycle(void) { + * iteration. + * + * If fast is non-zero the function will try to expire just one key ASAP + * from the current DB and return. This kind of call is used when Redis detects + * that timelimit_exit is true, so there is more work to do, and we do it + * more incrementally from the beforeSleep() function of the event loop. */ + +#define EXPIRED_HISTORY_LEN 10 + +void activeExpireCycle(int fast) { /* This function has some global state in order to continue the work * incrementally across calls. */ static unsigned int current_db = 0; /* Last DB tested. */ @@ -645,6 +682,31 @@ void activeExpireCycle(void) { unsigned int dbs_per_call = REDIS_DBCRON_DBS_PER_CALL; long long start = ustime(), timelimit; +#if 0 + static int expired_history[EXPIRED_HISTORY_LEN]; + static int expired_history_id = 0; + static int expired_perc_avg = 0; +#endif + + if (fast && !timelimit_exit) return; + +#if 0 + if (fast) { + if (!timelimit_exit) return; + + /* Let's try to expire a single key from the previous DB, the one that + * had enough keys expiring to reach the time limit. */ + redisDb *db = server.db+((current_db+server.dbnum-1) % server.dbnum); + dictEntry *de; + + for (j = 0; j < 100; j++) { + if ((de = dictGetRandomKey(db->expires)) == NULL) break; + activeExpireCycleTryExpire(db,de,server.mstime); + } + return; + } +#endif + /* We usually should test REDIS_DBCRON_DBS_PER_CALL per iteration, with * two exceptions: * @@ -663,6 +725,8 @@ void activeExpireCycle(void) { timelimit_exit = 0; if (timelimit <= 0) timelimit = 1; + if (fast) timelimit = 1000; /* 1 millisecond. */ + for (j = 0; j < dbs_per_call; j++) { int expired; redisDb *db = server.db+(current_db % server.dbnum); @@ -696,22 +760,9 @@ void activeExpireCycle(void) { num = REDIS_EXPIRELOOKUPS_PER_CRON; while (num--) { dictEntry *de; - long long t; if ((de = dictGetRandomKey(db->expires)) == NULL) break; - t = dictGetSignedIntegerVal(de); - if (now > t) { - sds key = dictGetKey(de); - robj *keyobj = createStringObject(key,sdslen(key)); - - propagateExpire(db,keyobj); - dbDelete(db,keyobj); - notifyKeyspaceEvent(REDIS_NOTIFY_EXPIRED, - "expired",keyobj,db->id); - decrRefCount(keyobj); - expired++; - server.stat_expiredkeys++; - } + if (activeExpireCycleTryExpire(db,de,now)) expired++; } /* We can't block forever here even if there are many keys to * expire. So after a given amount of milliseconds return to the @@ -721,8 +772,21 @@ void activeExpireCycle(void) { (ustime()-start) > timelimit) { timelimit_exit = 1; - return; } +#if 0 + expired_history_id = (expired_history_id+1) % EXPIRED_HISTORY_LEN; + expired_history[expired_history_id] = expired; + { + int i; + expired_perc_avg = 0; + for (i = 0; i < EXPIRED_HISTORY_LEN; i++) { + expired_perc_avg += expired_history[i]; + } + expired_perc_avg = (expired_perc_avg * 100) / (REDIS_EXPIRELOOKUPS_PER_CRON*EXPIRED_HISTORY_LEN); + // printf("Expired AVG: %d\n", expired_perc_avg); + } +#endif + if (timelimit_exit) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } } @@ -842,8 +906,12 @@ void clientsCron(void) { void databasesCron(void) { /* Expire keys by random sampling. Not required for slaves * as master will synthesize DELs for us. */ - if (server.active_expire_enabled && server.masterhost == NULL) - activeExpireCycle(); + if (server.active_expire_enabled && server.masterhost == NULL) { + long long totalex = server.stat_expiredkeys; + activeExpireCycle(0); + if (server.stat_expiredkeys - totalex) + printf("EXPIRED SLOW: %lld\n", server.stat_expiredkeys - totalex); + } /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad @@ -915,6 +983,7 @@ int serverCron(struct aeEventLoop *eventLoop, long long id, void *clientData) { * in objects at every object access, and accuracy is not needed. * To access a global var is faster than calling time(NULL) */ server.unixtime = time(NULL); + server.mstime = mstime(); run_with_period(100) trackOperationsPerSecond(); @@ -1074,6 +1143,14 @@ void beforeSleep(struct aeEventLoop *eventLoop) { listNode *ln; redisClient *c; + /* Run a fast expire cycle. */ + { + long long totalex = server.stat_expiredkeys; + activeExpireCycle(1); + if (server.stat_expiredkeys - totalex) + printf("EXPIRED FAST: %lld\n", server.stat_expiredkeys - totalex); + } + /* Try to process pending commands for clients that were just unblocked. */ while (listLength(server.unblocked_clients)) { ln = listFirst(server.unblocked_clients); @@ -1473,6 +1550,7 @@ void initServer() { server.ops_sec_last_sample_time = mstime(); server.ops_sec_last_sample_ops = 0; server.unixtime = time(NULL); + server.mstime = mstime(); server.lastbgsave_status = REDIS_OK; server.repl_good_slaves_count = 0; diff --git a/src/redis.h b/src/redis.h index 5fd7f0ec808..842c67bdb50 100644 --- a/src/redis.h +++ b/src/redis.h @@ -74,7 +74,7 @@ #define REDIS_MAXIDLETIME 0 /* default client timeout: infinite */ #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 -#define REDIS_EXPIRELOOKUPS_PER_CRON 10 /* lookup 10 expires per loop */ +#define REDIS_EXPIRELOOKUPS_PER_CRON 20 /* lookup 20 expires per loop */ #define REDIS_EXPIRELOOKUPS_TIME_PERC 25 /* CPU max % for keys collection */ #define REDIS_DBCRON_DBS_PER_CALL 16 #define REDIS_MAX_WRITE_PER_EVENT (1024*64) @@ -744,7 +744,8 @@ struct redisServer { size_t set_max_intset_entries; size_t zset_max_ziplist_entries; size_t zset_max_ziplist_value; - time_t unixtime; /* Unix time sampled every second. */ + time_t unixtime; /* Unix time sampled every cron cycle. */ + long long mstime; /* Like 'unixtime' but with milliseconds resolution. */ /* Pubsub */ dict *pubsub_channels; /* Map channels to list of subscribed clients */ list *pubsub_patterns; /* A list of pubsub_patterns */ From d54e373b8477cf4cb2c1a18fee98fc626f3f77ca Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 5 Aug 2013 16:11:56 +0200 Subject: [PATCH 0754/1752] Darft #2 for key collection algo: more improvements. This commit makes the fast collection cycle time configurable, at the same time it does not allow to run a new fast collection cycle for the same amount of time as the max duration of the fast collection cycle. --- src/redis.c | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index e29cb241efa..12cbb082d60 100644 --- a/src/redis.c +++ b/src/redis.c @@ -671,6 +671,7 @@ int activeExpireCycleTryExpire(redisDb *db, struct dictEntry *de, long long now) * more incrementally from the beforeSleep() function of the event loop. */ #define EXPIRED_HISTORY_LEN 10 +#define EXPIRE_FAST_CYCLE_DURATION 1000 void activeExpireCycle(int fast) { /* This function has some global state in order to continue the work @@ -681,6 +682,7 @@ void activeExpireCycle(int fast) { unsigned int j, iteration = 0; unsigned int dbs_per_call = REDIS_DBCRON_DBS_PER_CALL; long long start = ustime(), timelimit; + static long long last_fast_cycle = 0; #if 0 static int expired_history[EXPIRED_HISTORY_LEN]; @@ -688,7 +690,17 @@ void activeExpireCycle(int fast) { static int expired_perc_avg = 0; #endif - if (fast && !timelimit_exit) return; + if (fast) { + /* Don't start a fast cycle if the previous cycle did not exited + * for time limt. Also don't repeat a fast cycle for the same period + * as the fast cycle total duration itself. */ + if (!timelimit_exit) return; + if (start < last_fast_cycle + EXPIRE_FAST_CYCLE_DURATION) { + printf("CANT START A FAST CYCLE\n"); + return; + } + last_fast_cycle = start; + } #if 0 if (fast) { @@ -725,7 +737,7 @@ void activeExpireCycle(int fast) { timelimit_exit = 0; if (timelimit <= 0) timelimit = 1; - if (fast) timelimit = 1000; /* 1 millisecond. */ + if (fast) timelimit = EXPIRE_FAST_CYCLE_DURATION; /* in microseconds. */ for (j = 0; j < dbs_per_call; j++) { int expired; From 8686a70303d101210b82e93afc597f69f128ceb9 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 6 Aug 2013 12:36:13 +0200 Subject: [PATCH 0755/1752] Remove dead code and fix comments for new expire code. --- src/redis.c | 66 ++++++++--------------------------------------------- 1 file changed, 10 insertions(+), 56 deletions(-) diff --git a/src/redis.c b/src/redis.c index 12cbb082d60..bec3aac39d8 100644 --- a/src/redis.c +++ b/src/redis.c @@ -665,12 +665,14 @@ int activeExpireCycleTryExpire(redisDb *db, struct dictEntry *de, long long now) * No more than REDIS_DBCRON_DBS_PER_CALL databases are tested at every * iteration. * - * If fast is non-zero the function will try to expire just one key ASAP - * from the current DB and return. This kind of call is used when Redis detects - * that timelimit_exit is true, so there is more work to do, and we do it - * more incrementally from the beforeSleep() function of the event loop. */ + * If fast is non-zero the function will try to run a "fast" expire cycle that + * takes no longer than EXPIRE_FAST_CYCLE_DURATION microseconds, and is not + * repeated again before the same amount of time. + * + * This kind of call is used when Redis detects that timelimit_exit is + * true, so there is more work to do, and we do it more incrementally from + * the beforeSleep() function of the event loop. */ -#define EXPIRED_HISTORY_LEN 10 #define EXPIRE_FAST_CYCLE_DURATION 1000 void activeExpireCycle(int fast) { @@ -684,41 +686,15 @@ void activeExpireCycle(int fast) { long long start = ustime(), timelimit; static long long last_fast_cycle = 0; -#if 0 - static int expired_history[EXPIRED_HISTORY_LEN]; - static int expired_history_id = 0; - static int expired_perc_avg = 0; -#endif - if (fast) { /* Don't start a fast cycle if the previous cycle did not exited * for time limt. Also don't repeat a fast cycle for the same period * as the fast cycle total duration itself. */ if (!timelimit_exit) return; - if (start < last_fast_cycle + EXPIRE_FAST_CYCLE_DURATION) { - printf("CANT START A FAST CYCLE\n"); - return; - } + if (start < last_fast_cycle + EXPIRE_FAST_CYCLE_DURATION) return; last_fast_cycle = start; } -#if 0 - if (fast) { - if (!timelimit_exit) return; - - /* Let's try to expire a single key from the previous DB, the one that - * had enough keys expiring to reach the time limit. */ - redisDb *db = server.db+((current_db+server.dbnum-1) % server.dbnum); - dictEntry *de; - - for (j = 0; j < 100; j++) { - if ((de = dictGetRandomKey(db->expires)) == NULL) break; - activeExpireCycleTryExpire(db,de,server.mstime); - } - return; - } -#endif - /* We usually should test REDIS_DBCRON_DBS_PER_CALL per iteration, with * two exceptions: * @@ -785,19 +761,6 @@ void activeExpireCycle(int fast) { { timelimit_exit = 1; } -#if 0 - expired_history_id = (expired_history_id+1) % EXPIRED_HISTORY_LEN; - expired_history[expired_history_id] = expired; - { - int i; - expired_perc_avg = 0; - for (i = 0; i < EXPIRED_HISTORY_LEN; i++) { - expired_perc_avg += expired_history[i]; - } - expired_perc_avg = (expired_perc_avg * 100) / (REDIS_EXPIRELOOKUPS_PER_CRON*EXPIRED_HISTORY_LEN); - // printf("Expired AVG: %d\n", expired_perc_avg); - } -#endif if (timelimit_exit) return; } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); } @@ -918,12 +881,8 @@ void clientsCron(void) { void databasesCron(void) { /* Expire keys by random sampling. Not required for slaves * as master will synthesize DELs for us. */ - if (server.active_expire_enabled && server.masterhost == NULL) { - long long totalex = server.stat_expiredkeys; + if (server.active_expire_enabled && server.masterhost == NULL) activeExpireCycle(0); - if (server.stat_expiredkeys - totalex) - printf("EXPIRED SLOW: %lld\n", server.stat_expiredkeys - totalex); - } /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad @@ -1156,12 +1115,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { redisClient *c; /* Run a fast expire cycle. */ - { - long long totalex = server.stat_expiredkeys; - activeExpireCycle(1); - if (server.stat_expiredkeys - totalex) - printf("EXPIRED FAST: %lld\n", server.stat_expiredkeys - totalex); - } + activeExpireCycle(1); /* Try to process pending commands for clients that were just unblocked. */ while (listLength(server.unblocked_clients)) { From 00c8cfef74a5a2954f3f44078f8f08f458c9ce76 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 6 Aug 2013 12:55:49 +0200 Subject: [PATCH 0756/1752] Some activeExpireCycle() refactoring. --- src/redis.c | 45 ++++++++++++++++++++++++++------------------- src/redis.h | 8 ++++++-- 2 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/redis.c b/src/redis.c index bec3aac39d8..85b5a44ff6c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -665,33 +665,37 @@ int activeExpireCycleTryExpire(redisDb *db, struct dictEntry *de, long long now) * No more than REDIS_DBCRON_DBS_PER_CALL databases are tested at every * iteration. * - * If fast is non-zero the function will try to run a "fast" expire cycle that - * takes no longer than EXPIRE_FAST_CYCLE_DURATION microseconds, and is not - * repeated again before the same amount of time. - * * This kind of call is used when Redis detects that timelimit_exit is * true, so there is more work to do, and we do it more incrementally from - * the beforeSleep() function of the event loop. */ - -#define EXPIRE_FAST_CYCLE_DURATION 1000 + * the beforeSleep() function of the event loop. + * + * Expire cycle type: + * + * If type is ACTIVE_EXPIRE_CYCLE_FAST the function will try to run a + * "fast" expire cycle that takes no longer than EXPIRE_FAST_CYCLE_DURATION + * microseconds, and is not repeated again before the same amount of time. + * + * If type is ACTIVE_EXPIRE_CYCLE_SLOW, that normal expire cycle is + * executed, where the time limit is a percentage of the REDIS_HZ period + * as specified by the REDIS_EXPIRELOOKUPS_TIME_PERC define. */ -void activeExpireCycle(int fast) { +void activeExpireCycle(int type) { /* This function has some global state in order to continue the work * incrementally across calls. */ static unsigned int current_db = 0; /* Last DB tested. */ static int timelimit_exit = 0; /* Time limit hit in previous call? */ + static long long last_fast_cycle = 0; /* When last fast cycle ran. */ unsigned int j, iteration = 0; unsigned int dbs_per_call = REDIS_DBCRON_DBS_PER_CALL; long long start = ustime(), timelimit; - static long long last_fast_cycle = 0; - if (fast) { + if (type == ACTIVE_EXPIRE_CYCLE_FAST) { /* Don't start a fast cycle if the previous cycle did not exited * for time limt. Also don't repeat a fast cycle for the same period * as the fast cycle total duration itself. */ if (!timelimit_exit) return; - if (start < last_fast_cycle + EXPIRE_FAST_CYCLE_DURATION) return; + if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION) return; last_fast_cycle = start; } @@ -705,15 +709,16 @@ void activeExpireCycle(int fast) { if (dbs_per_call > server.dbnum || timelimit_exit) dbs_per_call = server.dbnum; - /* We can use at max REDIS_EXPIRELOOKUPS_TIME_PERC percentage of CPU time + /* We can use at max ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC percentage of CPU time * per iteration. Since this function gets called with a frequency of * server.hz times per second, the following is the max amount of * microseconds we can spend in this function. */ - timelimit = 1000000*REDIS_EXPIRELOOKUPS_TIME_PERC/server.hz/100; + timelimit = 1000000*ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC/server.hz/100; timelimit_exit = 0; if (timelimit <= 0) timelimit = 1; - if (fast) timelimit = EXPIRE_FAST_CYCLE_DURATION; /* in microseconds. */ + if (type == ACTIVE_EXPIRE_CYCLE_FAST) + timelimit = ACTIVE_EXPIRE_CYCLE_FAST_DURATION; /* in microseconds. */ for (j = 0; j < dbs_per_call; j++) { int expired; @@ -744,8 +749,8 @@ void activeExpireCycle(int fast) { /* The main collection cycle. Sample random keys among keys * with an expire set, checking for expired ones. */ expired = 0; - if (num > REDIS_EXPIRELOOKUPS_PER_CRON) - num = REDIS_EXPIRELOOKUPS_PER_CRON; + if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP) + num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP; while (num--) { dictEntry *de; @@ -762,7 +767,9 @@ void activeExpireCycle(int fast) { timelimit_exit = 1; } if (timelimit_exit) return; - } while (expired > REDIS_EXPIRELOOKUPS_PER_CRON/4); + /* We don't repeat the cycle if there are less than 25% of keys + * found expired in the current DB. */ + } while (expired > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP/4); } } @@ -882,7 +889,7 @@ void databasesCron(void) { /* Expire keys by random sampling. Not required for slaves * as master will synthesize DELs for us. */ if (server.active_expire_enabled && server.masterhost == NULL) - activeExpireCycle(0); + activeExpireCycle(ACTIVE_EXPIRE_CYCLE_SLOW); /* Perform hash tables rehashing if needed, but only if there are no * other processes saving the DB on disk. Otherwise rehashing is bad @@ -1115,7 +1122,7 @@ void beforeSleep(struct aeEventLoop *eventLoop) { redisClient *c; /* Run a fast expire cycle. */ - activeExpireCycle(1); + activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST); /* Try to process pending commands for clients that were just unblocked. */ while (listLength(server.unblocked_clients)) { diff --git a/src/redis.h b/src/redis.h index 842c67bdb50..984dadc2346 100644 --- a/src/redis.h +++ b/src/redis.h @@ -74,8 +74,6 @@ #define REDIS_MAXIDLETIME 0 /* default client timeout: infinite */ #define REDIS_DEFAULT_DBNUM 16 #define REDIS_CONFIGLINE_MAX 1024 -#define REDIS_EXPIRELOOKUPS_PER_CRON 20 /* lookup 20 expires per loop */ -#define REDIS_EXPIRELOOKUPS_TIME_PERC 25 /* CPU max % for keys collection */ #define REDIS_DBCRON_DBS_PER_CALL 16 #define REDIS_MAX_WRITE_PER_EVENT (1024*64) #define REDIS_SHARED_SELECT_CMDS 10 @@ -124,6 +122,12 @@ #define REDIS_PEER_ID_LEN (REDIS_IP_STR_LEN+32) /* Must be enough for ip:port */ #define REDIS_BINDADDR_MAX 16 +#define ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP 20 /* Loopkups per loop. */ +#define ACTIVE_EXPIRE_CYCLE_FAST_DURATION 1000 /* Microseconds */ +#define ACTIVE_EXPIRE_CYCLE_SLOW_TIME_PERC 25 /* CPU max % for keys collection */ +#define ACTIVE_EXPIRE_CYCLE_SLOW 0 +#define ACTIVE_EXPIRE_CYCLE_FAST 1 + /* Protocol and I/O related defines */ #define REDIS_MAX_QUERYBUF_LEN (1024*1024*1024) /* 1GB max query buffer. */ #define REDIS_IOBUF_LEN (1024*16) /* Generic I/O buffer size */ From 31d0f341000fc18b81717607ba82496ca285fb80 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 6 Aug 2013 12:59:04 +0200 Subject: [PATCH 0757/1752] activeExpireCycle(): fix about fast cycle early start. We don't want to repeat a fast cycle too soon, the previous code was broken, we need to wait two times the period *since* the start of the previous cycle in order to avoid there is an even space between cycles: .-> start .-> second start | | +-------------+-------------+--------------+ | first cycle | pause | second cycle | +-------------+-------------+--------------+ The second and first start must be PERIOD*2 useconds apart hence the *2 in the new code. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 85b5a44ff6c..1f1f43ae5c5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -695,7 +695,7 @@ void activeExpireCycle(int type) { * for time limt. Also don't repeat a fast cycle for the same period * as the fast cycle total duration itself. */ if (!timelimit_exit) return; - if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION) return; + if (start < last_fast_cycle + ACTIVE_EXPIRE_CYCLE_FAST_DURATION*2) return; last_fast_cycle = start; } From fa48b1fa3247ee9f3a52536488b3e591347feed4 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 6 Aug 2013 15:00:43 +0200 Subject: [PATCH 0758/1752] Add per-db average TTL information in INFO output. Example: db0:keys=221913,expires=221913,avg_ttl=655 The algorithm uses a running average with only two samples (current and previous). Keys found to be expired are considered at TTL zero even if the actual TTL can be negative. The TTL is reported in milliseconds. --- src/redis.c | 33 +++++++++++++++++++++++++++++---- src/redis.h | 1 + 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index 1f1f43ae5c5..0696c8ee3fa 100644 --- a/src/redis.c +++ b/src/redis.c @@ -733,10 +733,14 @@ void activeExpireCycle(int type) { * of the keys were expired. */ do { unsigned long num, slots; - long long now; + long long now, ttl_sum; + int ttl_samples; /* If there is nothing to expire try next DB ASAP. */ - if ((num = dictSize(db->expires)) == 0) break; + if ((num = dictSize(db->expires)) == 0) { + db->avg_ttl = 0; + break; + } slots = dictSlots(db->expires); now = mstime(); @@ -749,14 +753,33 @@ void activeExpireCycle(int type) { /* The main collection cycle. Sample random keys among keys * with an expire set, checking for expired ones. */ expired = 0; + ttl_sum = 0; + ttl_samples = 0; + if (num > ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP) num = ACTIVE_EXPIRE_CYCLE_LOOKUPS_PER_LOOP; + while (num--) { dictEntry *de; + long long ttl; if ((de = dictGetRandomKey(db->expires)) == NULL) break; + ttl = dictGetSignedIntegerVal(de)-now; if (activeExpireCycleTryExpire(db,de,now)) expired++; + if (ttl < 0) ttl = 0; + ttl_sum += ttl; + ttl_samples++; } + + /* Update the average TTL stats for this database. */ + if (ttl_samples) { + long long avg_ttl = ttl_sum/ttl_samples; + + if (db->avg_ttl == 0) db->avg_ttl = avg_ttl; + /* Smooth the value averaging with the previous one. */ + db->avg_ttl = (db->avg_ttl+avg_ttl)/2; + } + /* We can't block forever here even if there are many keys to * expire. So after a given amount of milliseconds return to the * caller waiting for the other active expire cycle. */ @@ -1490,6 +1513,7 @@ void initServer() { server.db[j].ready_keys = dictCreate(&setDictType,NULL); server.db[j].watched_keys = dictCreate(&keylistDictType,NULL); server.db[j].id = j; + server.db[j].avg_ttl = 0; } server.pubsub_channels = dictCreate(&keylistDictType,NULL); server.pubsub_patterns = listCreate(); @@ -2496,8 +2520,9 @@ sds genRedisInfoString(char *section) { keys = dictSize(server.db[j].dict); vkeys = dictSize(server.db[j].expires); if (keys || vkeys) { - info = sdscatprintf(info, "db%d:keys=%lld,expires=%lld\r\n", - j, keys, vkeys); + info = sdscatprintf(info, + "db%d:keys=%lld,expires=%lld,avg_ttl=%lld\r\n", + j, keys, vkeys, server.db[j].avg_ttl); } } } diff --git a/src/redis.h b/src/redis.h index 984dadc2346..a3028689b42 100644 --- a/src/redis.h +++ b/src/redis.h @@ -398,6 +398,7 @@ typedef struct redisDb { dict *ready_keys; /* Blocked keys that received a PUSH */ dict *watched_keys; /* WATCHED keys for MULTI/EXEC CAS */ int id; + long long avg_ttl; /* Average TTL, just for stats */ } redisDb; /* Client MULTI/EXEC state */ From 151ad68540dbf799e1f964ca61fda17a4888ff69 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 6 Aug 2013 18:50:54 +0200 Subject: [PATCH 0759/1752] redis-benchmark: ability to SELECT a specifid db number. --- src/redis-benchmark.c | 46 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 69c740242dd..a1c7b78b67c 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -73,6 +73,8 @@ static struct config { int csv; int loop; int idlemode; + int dbnum; + sds dbnumstr; char *tests; } config; @@ -85,6 +87,9 @@ typedef struct _client { long long start; /* start time of a request */ long long latency; /* request latency */ int pending; /* Number of pending requests (sent but no reply received) */ + int selectlen; /* If non-zero, a SELECT of 'selectlen' bytes is currently + used as a prefix of the pipline of commands. This gets + discarded the first time it's sent. */ } *client; /* Prototypes */ @@ -199,6 +204,15 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) { freeReplyObject(reply); + if (c->selectlen) { + /* This is the OK from SELECT. Just discard the SELECT + * from the buffer. */ + c->pending--; + sdsrange(c->obuf,c->selectlen,-1); + c->selectlen = 0; + continue; + } + if (config.requests_finished < config.requests) config.latency[config.requests_finished++] = c->latency; c->pending--; @@ -269,13 +283,26 @@ static client createClient(char *cmd, size_t len) { } /* Suppress hiredis cleanup of unused buffers for max speed. */ c->context->reader->maxbuf = 0; + /* Queue N requests accordingly to the pipeline size. */ c->obuf = sdsempty(); + if (config.dbnum != 0) { + /* If a DB number different than zero is selected, prefix our request + * buffer with the SELECT command, that will be discarded the first + * time the replies are received, so if the client is reused the + * SELECT command will not be used again. */ + c->obuf = sdscatprintf(c->obuf,"*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + (int)sdslen(config.dbnumstr),config.dbnumstr); + c->selectlen = sdslen(c->obuf); + } else { + c->selectlen = 0; + } for (j = 0; j < config.pipeline; j++) c->obuf = sdscatlen(c->obuf,cmd,len); c->randlen = 0; c->written = 0; c->pending = config.pipeline; + if (c->selectlen) c->pending++; /* Find substrings in the output buffer that need to be randomized. */ if (config.randomkeys) { @@ -286,8 +313,6 @@ static client createClient(char *cmd, size_t len) { p += 6; } } - -/* redisSetReplyObjectFunctions(c->context,NULL); */ aeCreateFileEvent(config.el,c->context->fd,AE_WRITABLE,writeHandler,c); listAddNodeTail(config.clients,c); config.liveclients++; @@ -296,9 +321,18 @@ static client createClient(char *cmd, size_t len) { static void createMissingClients(client c) { int n = 0; + char *buf = c->obuf; + size_t buflen = sdslen(c->obuf); + + /* If we are cloning from a client with a SELECT prefix, skip it since the + * client will be created with the prefixed SELECT if needed. */ + if (c->selectlen) { + buf += c->selectlen; + buflen -= c->selectlen; + } while(config.liveclients < config.numclients) { - createClient(c->obuf,sdslen(c->obuf)/config.pipeline); + createClient(buf,buflen/config.pipeline); /* Listen backlog is quite limited on most systems */ if (++n > 64) { @@ -421,6 +455,10 @@ int parseOptions(int argc, const char **argv) { config.tests = sdscat(config.tests,(char*)argv[++i]); config.tests = sdscat(config.tests,","); sdstolower(config.tests); + } else if (!strcmp(argv[i],"--dbnum")) { + if (lastarg) goto invalid; + config.dbnum = atoi(argv[++i]); + config.dbnumstr = sdsfromlonglong(config.dbnum); } else if (!strcmp(argv[i],"--help")) { exit_status = 0; goto usage; @@ -447,6 +485,7 @@ int parseOptions(int argc, const char **argv) { " -c Number of parallel connections (default 50)\n" " -n Total number of requests (default 10000)\n" " -d Data size of SET/GET value in bytes (default 2)\n" +" -dbnum SELECT the specified db number (default 0)\n" " -k 1=keep alive 0=reconnect (default 1)\n" " -r Use random keys for SET/GET/INCR, random values for SADD\n" " Using this option the benchmark will get/set keys\n" @@ -535,6 +574,7 @@ int main(int argc, const char **argv) { config.hostport = 6379; config.hostsocket = NULL; config.tests = NULL; + config.dbnum = 0; i = parseOptions(argc,argv); argc -= i; From 46c1bafeab81412472bf8eeb390f2d71fdc21e99 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 6 Aug 2013 19:01:54 +0200 Subject: [PATCH 0760/1752] redis-benchmark: fix db selection when :rand: feature is used. --- src/redis-benchmark.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index a1c7b78b67c..f53b333b1d0 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -205,10 +205,16 @@ static void readHandler(aeEventLoop *el, int fd, void *privdata, int mask) { freeReplyObject(reply); if (c->selectlen) { + int j; + /* This is the OK from SELECT. Just discard the SELECT * from the buffer. */ c->pending--; sdsrange(c->obuf,c->selectlen,-1); + /* We also need to fix the pointers to the strings + * we need to randomize. */ + for (j = 0; j < c->randlen; j++) + c->randptr[j] -= c->selectlen; c->selectlen = 0; continue; } From bcc796593182c5cd842f5333505012c2219c3dba Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 7 Aug 2013 15:58:51 +0200 Subject: [PATCH 0761/1752] redis-benchmark: max pipeline length hardcoded limit removed. --- src/redis-benchmark.c | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index f53b333b1d0..c275833888b 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -47,6 +47,7 @@ #include "zmalloc.h" #define REDIS_NOTUSED(V) ((void) V) +#define RANDPTR_INITIAL_SIZE 8 static struct config { aeEventLoop *el; @@ -81,12 +82,13 @@ static struct config { typedef struct _client { redisContext *context; sds obuf; - char *randptr[32]; /* needed for MSET against 10 keys */ - size_t randlen; - unsigned int written; /* bytes of 'obuf' already written */ - long long start; /* start time of a request */ - long long latency; /* request latency */ - int pending; /* Number of pending requests (sent but no reply received) */ + char **randptr; /* Pointers to :rand: strings inside the command buf */ + size_t randlen; /* Number of pointers in client->randptr */ + size_t randfree; /* Number of unused pointers in client->randptr */ + unsigned int written; /* Bytes of 'obuf' already written */ + long long start; /* Start time of a request */ + long long latency; /* Request latency */ + int pending; /* Number of pending requests (replies to consume) */ int selectlen; /* If non-zero, a SELECT of 'selectlen' bytes is currently used as a prefix of the pipline of commands. This gets discarded the first time it's sent. */ @@ -306,6 +308,8 @@ static client createClient(char *cmd, size_t len) { for (j = 0; j < config.pipeline; j++) c->obuf = sdscatlen(c->obuf,cmd,len); c->randlen = 0; + c->randfree = RANDPTR_INITIAL_SIZE; + c->randptr = zmalloc(sizeof(char*)*c->randfree); c->written = 0; c->pending = config.pipeline; if (c->selectlen) c->pending++; @@ -314,8 +318,12 @@ static client createClient(char *cmd, size_t len) { if (config.randomkeys) { char *p = c->obuf; while ((p = strstr(p,":rand:")) != NULL) { - assert(c->randlen < (signed)(sizeof(c->randptr)/sizeof(char*))); + if (c->randfree == 0) { + c->randptr = zrealloc(c->randptr,sizeof(char*)*c->randlen*2); + c->randfree += c->randlen; + } c->randptr[c->randlen++] = p+6; + c->randfree--; p += 6; } } From a79862de2afd3daff94696387efcbbdae829463b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 7 Aug 2013 15:59:59 +0200 Subject: [PATCH 0762/1752] redis-benchmark: fix memory leak introduced by 346256f --- src/redis-benchmark.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index c275833888b..dcd77b58eab 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -125,6 +125,7 @@ static void freeClient(client c) { aeDeleteFileEvent(config.el,c->context->fd,AE_READABLE); redisFree(c->context); sdsfree(c->obuf); + zfree(c->randptr); zfree(c); config.liveclients--; ln = listSearchKey(config.clients,c); From a256b83448ca94961aa25b00abf0ad04197dc9d5 Mon Sep 17 00:00:00 2001 From: Jan-Erik Rediger Date: Wed, 7 Aug 2013 16:05:09 +0200 Subject: [PATCH 0763/1752] Little typo --- tests/test_helper.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index cc1a801df57..0f7a93434df 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -465,7 +465,7 @@ proc close_replication_stream {s} { # With the parallel test running multiple Redis instances at the same time # we need a fast enough computer, otherwise a lot of tests may generate # false positives. -# If the computer is too slow we revert the sequetial test without any +# If the computer is too slow we revert the sequential test without any # parallelism, that is, clients == 1. proc is_a_slow_computer {} { set start [clock milliseconds] From 4fe67cc103d27f7aa0ff73cde5ca05eae8d06b41 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 8 Aug 2013 14:31:54 +0200 Subject: [PATCH 0764/1752] redis-benchmark: replace snprintf()+memcpy with faster code. This change was profiler-driven, but the actual effect is hard to measure in real-world redis benchmark runs. --- src/redis-benchmark.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index dcd77b58eab..ba885d9780c 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -152,13 +152,18 @@ static void resetClient(client c) { } static void randomizeClientKey(client c) { - char buf[32]; - size_t i, r; + size_t i; for (i = 0; i < c->randlen; i++) { - r = random() % config.randomkeys_keyspacelen; - snprintf(buf,sizeof(buf),"%012zu",r); - memcpy(c->randptr[i],buf,12); + char *p = c->randptr[i]+11; + size_t r = random() % config.randomkeys_keyspacelen; + size_t j; + + for (j = 0; j < 12; j++) { + *p = '0'+r%10; + r/=10; + p--; + } } } From 689829f2e1e0539c55a85d68410649677f932a23 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 8 Aug 2013 16:42:08 +0200 Subject: [PATCH 0765/1752] redis-benchmark: changes to random arguments substitution. Before this commit redis-benchmark supported random argumetns in the form of :rand:000000000000. In every string of that form, the zeros were replaced with a random number of 12 digits at every command invocation. However this was far from perfect as did not allowed to generate simply random numbers as arguments, there was always the :rand: prefix. Now instead every argument in the form __rand_int__ is replaced with a 12 digits number. Note that "__rand_int__" is 12 characters itself. In order to implement the new semantic, it was needed to change a few thigns in the internals of redis-benchmark, as new clients are created cloning old clients, so without a stable prefix such as ":rand:" the old way of cloning the client was no longer able to understand, from the old command line, what was the position of the random strings to substitute. Now instead a client structure is passed as a reference for cloning, so that we can directly clone the offsets inside the command line. --- src/redis-benchmark.c | 108 +++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 28 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index ba885d9780c..59b906ed76d 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -278,7 +278,29 @@ static void writeHandler(aeEventLoop *el, int fd, void *privdata, int mask) { } } -static client createClient(char *cmd, size_t len) { +/* Create a benchmark client, configured to send the command passed as 'cmd' of + * 'len' bytes. + * + * The command is copied N times in the client output buffer (that is reused + * again and again to send the request to the server) accordingly to the configured + * pipeline size. + * + * Also an initial SELECT command is prepended in order to make sure the right + * database is selected, if needed. The initial SELECT will be discarded as soon + * as the first reply is received. + * + * To create a client from scratch, the 'from' pointer is set to NULL. If instead + * we want to create a client using another client as reference, the 'from' pointer + * points to the client to use as reference. In such a case the following + * information is take from the 'from' client: + * + * 1) The command line to use. + * 2) The offsets of the __rand_int__ elements inside the command line, used + * for arguments randomization. + * + * Even when cloning another client, the SELECT command is automatically prefixed + * if needed. */ +static client createClient(char *cmd, size_t len, client from) { int j; client c = zmalloc(sizeof(struct _client)); @@ -298,39 +320,65 @@ static client createClient(char *cmd, size_t len) { /* Suppress hiredis cleanup of unused buffers for max speed. */ c->context->reader->maxbuf = 0; - /* Queue N requests accordingly to the pipeline size. */ + /* Build the request buffer: + * Queue N requests accordingly to the pipeline size, or simply clone + * the example client buffer. */ c->obuf = sdsempty(); + + /* If a DB number different than zero is selected, prefix our request + * buffer with the SELECT command, that will be discarded the first + * time the replies are received, so if the client is reused the + * SELECT command will not be used again. */ if (config.dbnum != 0) { - /* If a DB number different than zero is selected, prefix our request - * buffer with the SELECT command, that will be discarded the first - * time the replies are received, so if the client is reused the - * SELECT command will not be used again. */ c->obuf = sdscatprintf(c->obuf,"*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", (int)sdslen(config.dbnumstr),config.dbnumstr); c->selectlen = sdslen(c->obuf); } else { c->selectlen = 0; } - for (j = 0; j < config.pipeline; j++) - c->obuf = sdscatlen(c->obuf,cmd,len); - c->randlen = 0; - c->randfree = RANDPTR_INITIAL_SIZE; - c->randptr = zmalloc(sizeof(char*)*c->randfree); + + /* Append the request itself. */ + if (from) { + c->obuf = sdscatlen(c->obuf, + from->obuf+from->selectlen, + sdslen(from->obuf)-from->selectlen); + } else { + for (j = 0; j < config.pipeline; j++) + c->obuf = sdscatlen(c->obuf,cmd,len); + } c->written = 0; c->pending = config.pipeline; + c->randptr = NULL; + c->randlen = 0; if (c->selectlen) c->pending++; /* Find substrings in the output buffer that need to be randomized. */ if (config.randomkeys) { - char *p = c->obuf; - while ((p = strstr(p,":rand:")) != NULL) { - if (c->randfree == 0) { - c->randptr = zrealloc(c->randptr,sizeof(char*)*c->randlen*2); - c->randfree += c->randlen; + if (from) { + c->randlen = from->randlen; + c->randfree = 0; + c->randptr = zmalloc(sizeof(char*)*c->randlen); + /* copy the offsets. */ + for (j = 0; j < c->randlen; j++) { + c->randptr[j] = c->obuf + (from->randptr[j]-from->obuf); + /* Adjust for the different select prefix length. */ + c->randptr[j] += c->selectlen - from->selectlen; + } + } else { + char *p = c->obuf; + + c->randlen = 0; + c->randfree = RANDPTR_INITIAL_SIZE; + c->randptr = zmalloc(sizeof(char*)*c->randfree); + while ((p = strstr(p,"__rand_int__")) != NULL) { + if (c->randfree == 0) { + c->randptr = zrealloc(c->randptr,sizeof(char*)*c->randlen*2); + c->randfree += c->randlen; + } + c->randptr[c->randlen++] = p; + c->randfree--; + p += 12; /* 12 is strlen("__rand_int__). */ } - c->randptr[c->randlen++] = p+6; - c->randfree--; - p += 6; } } aeCreateFileEvent(config.el,c->context->fd,AE_WRITABLE,writeHandler,c); @@ -352,7 +400,7 @@ static void createMissingClients(client c) { } while(config.liveclients < config.numclients) { - createClient(buf,buflen/config.pipeline); + createClient(NULL,0,c); /* Listen backlog is quite limited on most systems */ if (++n > 64) { @@ -403,7 +451,7 @@ static void benchmark(char *title, char *cmd, int len) { config.requests_issued = 0; config.requests_finished = 0; - c = createClient(cmd,len); + c = createClient(cmd,len,NULL); createMissingClients(c); config.start = mstime(); @@ -530,8 +578,12 @@ int parseOptions(int argc, const char **argv) { " $ redis-benchmark -t set -n 1000000 -r 100000000\n\n" " Benchmark 127.0.0.1:6379 for a few commands producing CSV output:\n" " $ redis-benchmark -t ping,set,get -n 100000 --csv\n\n" +" Benchmark a specific command line:\n" +" $ redis-benchmark -r 10000 -n 10000 eval 'return redis.call(\"ping\")' 0\n\n" " Fill a list with 10000 random elements:\n" -" $ redis-benchmark -r 10000 -n 10000 lpush mylist ele:rand:000000000000\n\n" +" $ redis-benchmark -r 10000 -n 10000 lpush mylist __rand_int__\n\n" +" On user specified command lines __rand_int__ is replaced with a random integer\n" +" with a range of values selected by the -r option.\n" ); exit(exit_status); } @@ -608,7 +660,7 @@ int main(int argc, const char **argv) { if (config.idlemode) { printf("Creating %d idle connections and waiting forever (Ctrl+C when done)\n", config.numclients); - c = createClient("",0); /* will never receive a reply */ + c = createClient("",0,NULL); /* will never receive a reply */ createMissingClients(c); aeMain(config.el); /* and will wait for every */ @@ -647,19 +699,19 @@ int main(int argc, const char **argv) { } if (test_is_selected("set")) { - len = redisFormatCommand(&cmd,"SET foo:rand:000000000000 %s",data); + len = redisFormatCommand(&cmd,"SET key:__rand_int__ %s",data); benchmark("SET",cmd,len); free(cmd); } if (test_is_selected("get")) { - len = redisFormatCommand(&cmd,"GET foo:rand:000000000000"); + len = redisFormatCommand(&cmd,"GET key:__rand_int__"); benchmark("GET",cmd,len); free(cmd); } if (test_is_selected("incr")) { - len = redisFormatCommand(&cmd,"INCR counter:rand:000000000000"); + len = redisFormatCommand(&cmd,"INCR counter:__rand_int__"); benchmark("INCR",cmd,len); free(cmd); } @@ -678,7 +730,7 @@ int main(int argc, const char **argv) { if (test_is_selected("sadd")) { len = redisFormatCommand(&cmd, - "SADD myset counter:rand:000000000000"); + "SADD myset element:__rand_int__"); benchmark("SADD",cmd,len); free(cmd); } @@ -728,7 +780,7 @@ int main(int argc, const char **argv) { const char *argv[21]; argv[0] = "MSET"; for (i = 1; i < 21; i += 2) { - argv[i] = "foo:rand:000000000000"; + argv[i] = "key:__rand_int__"; argv[i+1] = data; } len = redisFormatCommandArgv(&cmd,21,argv,NULL); From 0e72cc601f2152156df0544552c455a9fc66445a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 12 Aug 2013 11:38:21 +0200 Subject: [PATCH 0766/1752] Fix sdsempty() prototype in sds.h. --- deps/hiredis/sds.h | 2 +- src/sds.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deps/hiredis/sds.h b/deps/hiredis/sds.h index 6f320113060..615c751cdcd 100644 --- a/deps/hiredis/sds.h +++ b/deps/hiredis/sds.h @@ -56,7 +56,7 @@ static inline size_t sdsavail(const sds s) { sds sdsnewlen(const void *init, size_t initlen); sds sdsnew(const char *init); -sds sdsempty(); +sds sdsempty(void); size_t sdslen(const sds s); sds sdsdup(const sds s); void sdsfree(sds s); diff --git a/src/sds.h b/src/sds.h index 6f320113060..615c751cdcd 100644 --- a/src/sds.h +++ b/src/sds.h @@ -56,7 +56,7 @@ static inline size_t sdsavail(const sds s) { sds sdsnewlen(const void *init, size_t initlen); sds sdsnew(const char *init); -sds sdsempty(); +sds sdsempty(void); size_t sdslen(const sds s); sds sdsdup(const sds s); void sdsfree(sds s); From 3eab283b654c5abd001ef07dfcc037a817b6c98d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 12 Aug 2013 11:50:54 +0200 Subject: [PATCH 0767/1752] Fix a PSYNC bug caused by a variable name typo. --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 900cb4ea606..c76c1293c39 100644 --- a/src/replication.c +++ b/src/replication.c @@ -254,7 +254,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { aux[len+1] = '\r'; aux[len+2] = '\n'; feedReplicationBacklog(aux,len+3); - feedReplicationBacklogWithObject(argv[j]); + feedReplicationBacklogWithObject(argv[i]); feedReplicationBacklogWithObject(shared.crlf); } } From 45d4e06e9301b3c6e45e1d0a2f50dc04b079c196 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 12 Aug 2013 12:10:38 +0200 Subject: [PATCH 0768/1752] replicationFeedSlave() reworked for correctness and speed. The previous code using a static buffer as an optimization was lame: 1) Premature optimization, actually it was *slower* than naive code because resulted into the creation / destruction of the object encapsulating the output buffer. 2) The code was very hard to test, since it was needed to have specific tests for command lines exceeding the size of the static buffer. 3) As a result of "2" the code was bugged as the current tests were not able to stress specific corner cases. It was replaced with easy to understand code that is safer and faster. --- src/replication.c | 139 ++++++++++++++-------------------------------- 1 file changed, 42 insertions(+), 97 deletions(-) diff --git a/src/replication.c b/src/replication.c index c76c1293c39..2073cd904db 100644 --- a/src/replication.c +++ b/src/replication.c @@ -138,15 +138,11 @@ void feedReplicationBacklogWithObject(robj *o) { feedReplicationBacklog(p,len); } -#define FEEDSLAVE_BUF_SIZE (1024*64) void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { listNode *ln; listIter li; - int j, i, len; - char buf[FEEDSLAVE_BUF_SIZE], *b = buf; + int j, len; char llstr[REDIS_LONGSTR_SIZE]; - int buf_left = FEEDSLAVE_BUF_SIZE; - robj *o; /* If there aren't slaves, and there is no backlog buffer to populate, * we can return ASAP. */ @@ -155,116 +151,66 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* We can't have slaves attached and no backlog. */ redisAssert(!(listLength(slaves) != 0 && server.repl_backlog == NULL)); - /* What we do here is to try to write as much data as possible in a static - * buffer "buf" that is used to create an object that is later sent to all - * the slaves. This way we do the decoding only one time for most commands - * not containing big payloads. */ - - /* Create the SELECT command into the static buffer if needed. */ + /* Send SELECT command to every slave if needed. */ if (server.slaveseldb != dictid) { - char *selectcmd; - size_t sclen; + robj *selectcmd; + /* For a few DBs we have pre-computed SELECT command. */ if (dictid >= 0 && dictid < REDIS_SHARED_SELECT_CMDS) { - selectcmd = shared.select[dictid]->ptr; - sclen = sdslen(selectcmd); - memcpy(b,selectcmd,sclen); - b += sclen; - buf_left -= sclen; + selectcmd = shared.select[dictid]; } else { int dictid_len; dictid_len = ll2string(llstr,sizeof(llstr),dictid); - sclen = snprintf(b,buf_left,"*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", - dictid_len, llstr); - b += sclen; - buf_left -= sclen; + selectcmd = createObject(REDIS_STRING, + sdscatprintf(sdsempty(), + "*2\r\n$6\r\nSELECT\r\n$%d\r\n%s\r\n", + dictid_len, llstr)); } - } - server.slaveseldb = dictid; - /* Add the multi bulk reply size to the static buffer, that is, the number - * of arguments of the command to send to every slave. */ - b[0] = '*'; - len = ll2string(b+1,REDIS_LONGSTR_SIZE,argc); - b += len+1; - buf_left -= len+1; - b[0] = '\r'; - b[1] = '\n'; - b += 2; - buf_left -= 2; - - /* Try to use the static buffer for as much arguments is possible. */ - for (j = 0; j < argc; j++) { - int objlen; - char *objptr; + /* Add the SELECT command into the backlog. */ + if (server.repl_backlog) feedReplicationBacklogWithObject(selectcmd); - if (argv[j]->encoding != REDIS_ENCODING_RAW && - argv[j]->encoding != REDIS_ENCODING_INT) { - redisPanic("Unexpected encoding"); - } - if (argv[j]->encoding == REDIS_ENCODING_RAW) { - objlen = sdslen(argv[j]->ptr); - objptr = argv[j]->ptr; - } else { - objlen = ll2string(llstr,REDIS_LONGSTR_SIZE,(long)argv[j]->ptr); - objptr = llstr; + /* Send it to slaves. */ + listRewind(slaves,&li); + while((ln = listNext(&li))) { + redisClient *slave = ln->value; + addReply(slave,selectcmd); } - /* We need enough space for bulk reply encoding, newlines, and - * the data itself. */ - if (buf_left < objlen+REDIS_LONGSTR_SIZE+32) break; - - /* Write $...CRLF */ - b[0] = '$'; - len = ll2string(b+1,REDIS_LONGSTR_SIZE,objlen); - b += len+1; - buf_left -= len+1; - b[0] = '\r'; - b[1] = '\n'; - b += 2; - buf_left -= 2; - - /* And data plus CRLF */ - memcpy(b,objptr,objlen); - b += objlen; - buf_left -= objlen; - b[0] = '\r'; - b[1] = '\n'; - b += 2; - buf_left -= 2; - } - - /* Create an object with the static buffer content. */ - redisAssert(buf_left < FEEDSLAVE_BUF_SIZE); - o = createStringObject(buf,b-buf); - - /* If we have a backlog, populate it with data and increment - * the global replication offset. */ + + if (dictid < 0 || dictid >= REDIS_SHARED_SELECT_CMDS) + decrRefCount(selectcmd); + } + server.slaveseldb = dictid; + + /* Write the command to the replication backlog if any. */ if (server.repl_backlog) { - feedReplicationBacklogWithObject(o); - for (i = j; i < argc; i++) { - char aux[REDIS_LONGSTR_SIZE+3]; - long objlen = stringObjectLen(argv[i]); + char aux[REDIS_LONGSTR_SIZE+3]; + + /* Add the multi bulk reply length. */ + aux[0] = '*'; + len = ll2string(aux+1,sizeof(aux-1),argc); + aux[len+1] = '\r'; + aux[len+2] = '\n'; + feedReplicationBacklog(aux,len+3); + + for (j = 0; j < argc; j++) { + long objlen = stringObjectLen(argv[j]); /* We need to feed the buffer with the object as a bulk reply * not just as a plain string, so create the $..CRLF payload len * ad add the final CRLF */ aux[0] = '$'; - len = ll2string(aux+1,objlen,sizeof(aux)-1); + len = ll2string(aux+1,sizeof(aux)-1,objlen); aux[len+1] = '\r'; aux[len+2] = '\n'; feedReplicationBacklog(aux,len+3); - feedReplicationBacklogWithObject(argv[i]); - feedReplicationBacklogWithObject(shared.crlf); + feedReplicationBacklogWithObject(argv[j]); + feedReplicationBacklogWithObject(aux+len+1,2); } } - /* Write data to slaves. Here we do two things: - * 1) We write the "o" object that was created using the accumulated - * static buffer. - * 2) We write any additional argument of the command to replicate that - * was not written inside the static buffer for lack of space. - */ + /* Write the command to every slave. */ listRewind(slaves,&li); while((ln = listNext(&li))) { redisClient *slave = ln->value; @@ -276,15 +222,14 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { * are queued in the output buffer until the initial SYNC completes), * or are already in sync with the master. */ - /* First, trasmit the object created from the static buffer. */ - addReply(slave,o); + /* Add the multi bulk length. */ + addReplyMultiBulkLen(slave,argc); /* Finally any additional argument that was not stored inside the * static buffer if any (from j to argc). */ - for (i = j; i < argc; i++) - addReplyBulk(slave,argv[i]); + for (j = 0; j < argc; j++) + addReplyBulk(slave,argv[j]); } - decrRefCount(o); } void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj **argv, int argc) { From 004f00bfa461c7ffd3d17862bb9c50659b8c6405 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 12 Aug 2013 12:38:52 +0200 Subject: [PATCH 0769/1752] replicationFeedSlaves() func name typo: feedReplicationBacklogWithObject -> feedReplicationBacklog. --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 2073cd904db..cb455042b14 100644 --- a/src/replication.c +++ b/src/replication.c @@ -206,7 +206,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { aux[len+2] = '\n'; feedReplicationBacklog(aux,len+3); feedReplicationBacklogWithObject(argv[j]); - feedReplicationBacklogWithObject(aux+len+1,2); + feedReplicationBacklog(aux+len+1,2); } } From 14a1cba3e3d208e64400193d42641ba10daa2719 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 12 Aug 2013 12:43:26 +0200 Subject: [PATCH 0770/1752] Use precomptued objects for bulk and mbulk prefixes. --- src/networking.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 22b7dd8e742..8a335bc9d44 100644 --- a/src/networking.c +++ b/src/networking.c @@ -461,7 +461,10 @@ void addReplyLongLong(redisClient *c, long long ll) { } void addReplyMultiBulkLen(redisClient *c, long length) { - addReplyLongLongWithPrefix(c,length,'*'); + if (length < REDIS_SHARED_BULKHDR_LEN) + addReply(c,shared.mbulkhdr[length]); + else + addReplyLongLongWithPrefix(c,length,'*'); } /* Create the length prefix of a bulk reply, example: $2234 */ @@ -483,7 +486,11 @@ void addReplyBulkLen(redisClient *c, robj *obj) { len++; } } - addReplyLongLongWithPrefix(c,len,'$'); + + if (len < REDIS_SHARED_BULKHDR_LEN) + addReply(c,shared.bulkhdr[len]); + else + addReplyLongLongWithPrefix(c,len,'$'); } /* Add a Redis Object as a bulk reply */ From 706056ca166d6d8ca80d4a05edd17fb354b67cb7 Mon Sep 17 00:00:00 2001 From: yihuang Date: Tue, 13 Aug 2013 17:47:42 +0800 Subject: [PATCH 0771/1752] fix lua_cmsgpack pack map as array --- deps/lua/src/lua_cmsgpack.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/lua/src/lua_cmsgpack.c b/deps/lua/src/lua_cmsgpack.c index 90612136169..53dc1cf61f0 100644 --- a/deps/lua/src/lua_cmsgpack.c +++ b/deps/lua/src/lua_cmsgpack.c @@ -374,7 +374,7 @@ static int table_is_an_array(lua_State *L) { while(lua_next(L,-2)) { /* Stack: ... key value */ lua_pop(L,1); /* Stack: ... key */ - if (!lua_isnumber(L,-1)) goto not_array; + if (lua_type(L,-1) != LUA_TNUMBER) goto not_array; n = lua_tonumber(L,-1); idx = n; if (idx != n || idx < 1) goto not_array; From 05379f49ee6763ecea361d70e369a86ed82f231e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 16 Aug 2013 14:08:04 +0200 Subject: [PATCH 0772/1752] dict.c iterator API misuse protection. dict.c allows the user to create unsafe iterators, that are iterators that will not touch the dictionary data structure in any way, preventing copy on write, but at the same time are limited in their usage. The limitation is that when itearting with an unsafe iterator, no call to other dictionary functions must be done inside the iteration loop, otherwise the dictionary may be incrementally rehashed resulting into missing elements in the set of the elements returned by the iterator. However after introducing this kind of iterators a number of bugs were found due to misuses of the API, and we are still finding bugs about this issue. The bugs are not trivial to track because the effect is just missing elements during the iteartion. This commit introduces auto-detection of the API misuse. The idea is that an unsafe iterator has a contract: from initialization to the release of the iterator the dictionary should not change. So we take a fingerprint of the dictionary state, xoring a few important dict properties when the unsafe iteartor is initialized. We later check when the iterator is released if the fingerprint is still the same. If it is not, we found a misuse of the iterator, as not allowed API calls changed the internal state of the dictionary. This code was checked against a real bug, issue #1240. This is what Redis prints (aborting) when a misuse is detected: Assertion failed: (iter->fingerprint == dictFingerprint(iter->d)), function dictReleaseIterator, file dict.c, line 587. --- src/dict.c | 34 ++++++++++++++++++++++++++++++---- src/dict.h | 1 + 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/dict.c b/src/dict.c index c6a1b88cfbd..a967be46c9b 100644 --- a/src/dict.c +++ b/src/dict.c @@ -505,6 +505,24 @@ void *dictFetchValue(dict *d, const void *key) { return he ? dictGetVal(he) : NULL; } +/* A fingerprint is a 64 bit number that represents the state of the dictionary + * at a given time, it's just a few dict properties xored together. + * When an unsafe iterator is initialized, we get the dict fingerprint, and check + * the fingerprint again when the iterator is released. + * If the two fingerprints are different it means that the user of the iterator + * performed forbidden operations against the dictionary while iterating. */ +long long dictFingerprint(dict *d) { + long long fingerprint = 0; + + fingerprint ^= (long long) d->ht[0].table; + fingerprint ^= (long long) d->ht[0].size; + fingerprint ^= (long long) d->ht[0].used; + fingerprint ^= (long long) d->ht[1].table; + fingerprint ^= (long long) d->ht[1].size; + fingerprint ^= (long long) d->ht[1].used; + return fingerprint; +} + dictIterator *dictGetIterator(dict *d) { dictIterator *iter = zmalloc(sizeof(*iter)); @@ -530,8 +548,12 @@ dictEntry *dictNext(dictIterator *iter) while (1) { if (iter->entry == NULL) { dictht *ht = &iter->d->ht[iter->table]; - if (iter->safe && iter->index == -1 && iter->table == 0) - iter->d->iterators++; + if (iter->index == -1 && iter->table == 0) { + if (iter->safe) + iter->d->iterators++; + else + iter->fingerprint = dictFingerprint(iter->d); + } iter->index++; if (iter->index >= (signed) ht->size) { if (dictIsRehashing(iter->d) && iter->table == 0) { @@ -558,8 +580,12 @@ dictEntry *dictNext(dictIterator *iter) void dictReleaseIterator(dictIterator *iter) { - if (iter->safe && !(iter->index == -1 && iter->table == 0)) - iter->d->iterators--; + if (!(iter->index == -1 && iter->table == 0)) { + if (iter->safe) + iter->d->iterators--; + else + assert(iter->fingerprint == dictFingerprint(iter->d)); + } zfree(iter); } diff --git a/src/dict.h b/src/dict.h index 3a311f17111..4d750ae8591 100644 --- a/src/dict.h +++ b/src/dict.h @@ -88,6 +88,7 @@ typedef struct dictIterator { dict *d; int table, index, safe; dictEntry *entry, *nextEntry; + long long fingerprint; /* unsafe iterator fingerprint for misuse detection */ } dictIterator; /* This is the initial size of every hash table */ From 3ad87c652b601f8a5fa1ee4cbf6802ddc02c6ed7 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 16 Aug 2013 15:26:44 +0200 Subject: [PATCH 0773/1752] Properly init/release iterators in zunionInterGenericCommand(). This commit does mainly two things: 1) It fixes zunionInterGenericCommand() by removing mass-initialization of all the iterators used, so that we don't violate the unsafe iterator API of dictionaries. This fixes issue #1240. 2) Since the zui* APIs required the allocator to be initialized in the zsetopsrc structure in order to use non-iterator related APIs, this commit fixes this strict requirement by accessing objects directly via the op->subject->ptr pointer we have to the object. --- src/t_zset.c | 37 ++++++++++++++++++------------------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index 8ef9c5376b7..ed97a078fa8 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1270,20 +1270,20 @@ int zuiLength(zsetopsrc *op) { return 0; if (op->type == REDIS_SET) { - iterset *it = &op->iter.set; if (op->encoding == REDIS_ENCODING_INTSET) { - return intsetLen(it->is.is); + return intsetLen(op->subject->ptr); } else if (op->encoding == REDIS_ENCODING_HT) { - return dictSize(it->ht.dict); + dict *ht = op->subject->ptr; + return dictSize(ht); } else { redisPanic("Unknown set encoding"); } } else if (op->type == REDIS_ZSET) { - iterzset *it = &op->iter.zset; if (op->encoding == REDIS_ENCODING_ZIPLIST) { - return zzlLength(it->zl.zl); + return zzlLength(op->subject->ptr); } else if (op->encoding == REDIS_ENCODING_SKIPLIST) { - return it->sl.zs->zsl->length; + zset *zs = op->subject->ptr; + return zs->zsl->length; } else { redisPanic("Unknown sorted set encoding"); } @@ -1419,18 +1419,19 @@ int zuiFind(zsetopsrc *op, zsetopval *val, double *score) { return 0; if (op->type == REDIS_SET) { - iterset *it = &op->iter.set; - if (op->encoding == REDIS_ENCODING_INTSET) { - if (zuiLongLongFromValue(val) && intsetFind(it->is.is,val->ell)) { + if (zuiLongLongFromValue(val) && + intsetFind(op->subject->ptr,val->ell)) + { *score = 1.0; return 1; } else { return 0; } } else if (op->encoding == REDIS_ENCODING_HT) { + dict *ht = op->subject->ptr; zuiObjectFromValue(val); - if (dictFind(it->ht.dict,val->ele) != NULL) { + if (dictFind(ht,val->ele) != NULL) { *score = 1.0; return 1; } else { @@ -1440,19 +1441,19 @@ int zuiFind(zsetopsrc *op, zsetopval *val, double *score) { redisPanic("Unknown set encoding"); } } else if (op->type == REDIS_ZSET) { - iterzset *it = &op->iter.zset; zuiObjectFromValue(val); if (op->encoding == REDIS_ENCODING_ZIPLIST) { - if (zzlFind(it->zl.zl,val->ele,score) != NULL) { + if (zzlFind(op->subject->ptr,val->ele,score) != NULL) { /* Score is already set by zzlFind. */ return 1; } else { return 0; } } else if (op->encoding == REDIS_ENCODING_SKIPLIST) { + zset *zs = op->subject->ptr; dictEntry *de; - if ((de = dictFind(it->sl.zs->dict,val->ele)) != NULL) { + if ((de = dictFind(zs->dict,val->ele)) != NULL) { *score = *(double*)dictGetVal(de); return 1; } else { @@ -1580,9 +1581,6 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { } } - for (i = 0; i < setnum; i++) - zuiInitIterator(&src[i]); - /* sort sets from the smallest to largest, this will improve our * algorithm's performance */ qsort(src,setnum,sizeof(zsetopsrc),zuiCompareByCardinality); @@ -1596,6 +1594,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { if (zuiLength(&src[0]) > 0) { /* Precondition: as src[0] is non-empty and the inputs are ordered * by size, all src[i > 0] are non-empty too. */ + zuiInitIterator(&src[0]); while (zuiNext(&src[0],&zval)) { double score, value; @@ -1629,12 +1628,14 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { maxelelen = sdslen(tmp->ptr); } } + zuiClearIterator(&src[0]); } } else if (op == REDIS_OP_UNION) { for (i = 0; i < setnum; i++) { if (zuiLength(&src[i]) == 0) continue; + zuiInitIterator(&src[i]); while (zuiNext(&src[i],&zval)) { double score, value; @@ -1670,14 +1671,12 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { if (sdslen(tmp->ptr) > maxelelen) maxelelen = sdslen(tmp->ptr); } + zuiClearIterator(&src[i]); } } else { redisPanic("Unknown operator"); } - for (i = 0; i < setnum; i++) - zuiClearIterator(&src[i]); - if (dbDelete(c->db,dstkey)) { signalModifiedKey(c->db,dstkey); touched = 1; From 836855073310a84648d125179e1506f12b8c59ca Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 16 Aug 2013 15:31:25 +0200 Subject: [PATCH 0774/1752] Fix comments for correctness in zunionInterGenericCommand(). Related to issue #1240. --- src/t_zset.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/t_zset.c b/src/t_zset.c index ed97a078fa8..b4cb9d00c5b 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -1639,7 +1639,7 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { while (zuiNext(&src[i],&zval)) { double score, value; - /* Skip key when already processed */ + /* Skip an element that when already processed */ if (dictFind(dstzset->dict,zuiObjectFromValue(&zval)) != NULL) continue; @@ -1647,8 +1647,10 @@ void zunionInterGenericCommand(redisClient *c, robj *dstkey, int op) { score = src[i].weight * zval.score; if (isnan(score)) score = 0; - /* Because the inputs are sorted by size, it's only possible - * for sets at larger indices to hold this element. */ + /* We need to check only next sets to see if this element + * exists, since we process every element just one time so + * it can't exist in a previous set (otherwise it would be + * already processed). */ for (j = (i+1); j < setnum; j++) { /* It is not safe to access the zset we are * iterating, so explicitly check for equal object. */ From c7da5fc6c150750c59ad8dde8324d45e725f5a56 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Aug 2013 11:29:18 +0200 Subject: [PATCH 0775/1752] dictFingerprint() fingerprinting made more robust. The previous hashing used the trivial algorithm of xoring the integers together. This is not optimal as it is very likely that different hash table setups will hash the same, for instance an hash table at the start of the rehashing process, and at the end, will have the same fingerprint. Now we hash N integers in a smarter way, by summing every integer to the previous hash, and taking the integer hashing again (see the code for further details). This way it is a lot less likely that we get a collision. Moreover this way of hashing explicitly protects from the same set of integers in a different order to hash to the same number. This commit is related to issue #1240. --- src/dict.c | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/dict.c b/src/dict.c index a967be46c9b..3c1061f347d 100644 --- a/src/dict.c +++ b/src/dict.c @@ -512,15 +512,35 @@ void *dictFetchValue(dict *d, const void *key) { * If the two fingerprints are different it means that the user of the iterator * performed forbidden operations against the dictionary while iterating. */ long long dictFingerprint(dict *d) { - long long fingerprint = 0; - - fingerprint ^= (long long) d->ht[0].table; - fingerprint ^= (long long) d->ht[0].size; - fingerprint ^= (long long) d->ht[0].used; - fingerprint ^= (long long) d->ht[1].table; - fingerprint ^= (long long) d->ht[1].size; - fingerprint ^= (long long) d->ht[1].used; - return fingerprint; + long long integers[6], hash = 0; + int j; + + integers[0] = (long long) d->ht[0].table; + integers[1] = d->ht[0].size; + integers[2] = d->ht[0].used; + integers[3] = (long long) d->ht[1].table; + integers[4] = d->ht[1].size; + integers[5] = d->ht[1].used; + + /* We hash N integers by summing every successive integer with the integer + * hashing of the previous sum. Basically: + * + * Result = hash(hash(hash(int1)+int2)+int3) ... + * + * This way the same set of integers in a different order will (likely) hash + * to a different number. */ + for (j = 0; j < 6; j++) { + hash += integers[j]; + /* For the hashing step we use Tomas Wang's 64 bit integer hash. */ + hash = (~hash) + (hash << 21); // hash = (hash << 21) - hash - 1; + hash = hash ^ (hash >> 24); + hash = (hash + (hash << 3)) + (hash << 8); // hash * 265 + hash = hash ^ (hash >> 14); + hash = (hash + (hash << 2)) + (hash << 4); // hash * 21 + hash = hash ^ (hash >> 28); + hash = hash + (hash << 31); + } + return hash; } dictIterator *dictGetIterator(dict *d) From bd353d9a3c8178e6a3bed1b0522e214edb048d83 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Aug 2013 14:53:38 +0200 Subject: [PATCH 0776/1752] Added redisassert.h as drop in replacement for assert.h. By using redisassert.h version of assert() you get stack traces in the log instead of a process disappearing on assertions. --- src/redisassert.h | 45 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 src/redisassert.h diff --git a/src/redisassert.h b/src/redisassert.h new file mode 100644 index 00000000000..e2cf47cb521 --- /dev/null +++ b/src/redisassert.h @@ -0,0 +1,45 @@ +/* redisassert.h -- Drop in replacemnet assert.h that prints the stack trace + * in the Redis logs. + * + * This file should be included instead of "assert.h" inside libraries used by + * Redis that are using assertions, so instead of Redis disappearing with + * SIGABORT, we get the details and stack trace inside the log file. + * + * ---------------------------------------------------------------------------- + * + * Copyright (c) 2006-2012, Salvatore Sanfilippo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of Redis nor the names of its contributors may be used + * to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef __REDIS_ASSERT_H__ +#define __REDIS_ASSERT_H__ + +#define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1))) + +void _redisAssert(char *estr, char *file, int line); + +#endif From 837817b2b711c2d75143853be164d477481a9730 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Aug 2013 14:54:31 +0200 Subject: [PATCH 0777/1752] assert.h replaced with redisassert.h when appropriate. Also a warning was suppressed by including unistd.h in redisassert.h (needed for _exit()). --- src/dict.c | 2 +- src/pqsort.c | 1 - src/redisassert.h | 4 +++- src/ziplist.c | 2 +- src/zipmap.c | 1 - 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dict.c b/src/dict.c index 3c1061f347d..b041febcfa6 100644 --- a/src/dict.c +++ b/src/dict.c @@ -39,13 +39,13 @@ #include #include #include -#include #include #include #include #include "dict.h" #include "zmalloc.h" +#include "redisassert.h" /* Using dictEnableResize() / dictDisableResize() we make possible to * enable/disable resizing of the hash table as needed. This is very important diff --git a/src/pqsort.c b/src/pqsort.c index 57c217f9446..7325a88a440 100644 --- a/src/pqsort.c +++ b/src/pqsort.c @@ -39,7 +39,6 @@ #include -#include #include #include diff --git a/src/redisassert.h b/src/redisassert.h index e2cf47cb521..e5825c0f595 100644 --- a/src/redisassert.h +++ b/src/redisassert.h @@ -38,7 +38,9 @@ #ifndef __REDIS_ASSERT_H__ #define __REDIS_ASSERT_H__ -#define redisAssert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1))) +#include /* for _exit() */ + +#define assert(_e) ((_e)?(void)0 : (_redisAssert(#_e,__FILE__,__LINE__),_exit(1))) void _redisAssert(char *estr, char *file, int line); diff --git a/src/ziplist.c b/src/ziplist.c index fdfc2e9ce09..d78f8f5da80 100644 --- a/src/ziplist.c +++ b/src/ziplist.c @@ -105,12 +105,12 @@ #include #include #include -#include #include #include "zmalloc.h" #include "util.h" #include "ziplist.h" #include "endianconv.h" +#include "redisassert.h" #define ZIP_END 255 #define ZIP_BIGLEN 254 diff --git a/src/zipmap.c b/src/zipmap.c index d9b7c8b31e0..3cd56e5ffb0 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -78,7 +78,6 @@ #include #include -#include #include "zmalloc.h" #include "endianconv.h" From 009515f9140fc1330c14af994040a8885056986c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Aug 2013 15:10:33 +0200 Subject: [PATCH 0778/1752] Fixed type in dict.c comment: 265 -> 256. --- src/dict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.c b/src/dict.c index b041febcfa6..91c3932061f 100644 --- a/src/dict.c +++ b/src/dict.c @@ -534,7 +534,7 @@ long long dictFingerprint(dict *d) { /* For the hashing step we use Tomas Wang's 64 bit integer hash. */ hash = (~hash) + (hash << 21); // hash = (hash << 21) - hash - 1; hash = hash ^ (hash >> 24); - hash = (hash + (hash << 3)) + (hash << 8); // hash * 265 + hash = (hash + (hash << 3)) + (hash << 8); // hash * 256 hash = hash ^ (hash >> 14); hash = (hash + (hash << 2)) + (hash << 4); // hash * 21 hash = hash ^ (hash >> 28); From 56212725754fa9616dc9e0a2d1e7916ce867a39a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Aug 2013 17:25:57 +0200 Subject: [PATCH 0779/1752] Revert "Fixed type in dict.c comment: 265 -> 256." This reverts commit 009515f9140fc1330c14af994040a8885056986c. --- src/dict.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dict.c b/src/dict.c index 91c3932061f..b041febcfa6 100644 --- a/src/dict.c +++ b/src/dict.c @@ -534,7 +534,7 @@ long long dictFingerprint(dict *d) { /* For the hashing step we use Tomas Wang's 64 bit integer hash. */ hash = (~hash) + (hash << 21); // hash = (hash << 21) - hash - 1; hash = hash ^ (hash >> 24); - hash = (hash + (hash << 3)) + (hash << 8); // hash * 256 + hash = (hash + (hash << 3)) + (hash << 8); // hash * 265 hash = hash ^ (hash >> 14); hash = (hash + (hash << 2)) + (hash << 4); // hash * 21 hash = hash ^ (hash >> 28); From 6a88a84ec79c5bd85e9357101ff699621bab1bae Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 19 Aug 2013 18:04:32 +0200 Subject: [PATCH 0780/1752] Redis 2.7.103 (2.8 Release Candidate 3). --- 00-RELEASENOTES | 16 ++++++++++++++++ src/version.h | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index c85366aadfe..648f9b2d4ae 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,22 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8 Release Candidate 3 (2.7.103) ] Release date: 19 Aug 2013 + +This is the third release candidate of Redis 2.8 (official version is 2.7.103). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH + +* [FIX] Improved expired keys collection algorithm. Even under heavy load keys + to be expired can't accumulate because of lack of CPU time. +* [FIX] Replication speed regression fixed (issue #1238). +* [FIX] Fixed an hard to trigger PSYNC bug. +* [FIX] Fixed Issue #1240, ZUNIONSTORE could lead to wrong result. +* [NEW] Add per-db average TTL information in INFO output. +* [NEW] redis-benchmark improvements. +* [NEW] dict.c API wrong usage detection. + --[ Redis 2.8 Release Candidate 2 (2.7.102) ] Release date: 30 Jul 2013 This is the second release candidate of Redis 2.8 (official version is 2.7.102). diff --git a/src/version.h b/src/version.h index 7f5043d67f8..97cdc352aed 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.102" +#define REDIS_VERSION "2.7.103" From 2c018b14fb8d5419e9443284edb02caf0b7e2fa6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 20 Aug 2013 11:49:55 +0200 Subject: [PATCH 0781/1752] dictFingerprint(): cast pointers to integer of same size. --- src/dict.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dict.c b/src/dict.c index b041febcfa6..ee560c822cb 100644 --- a/src/dict.c +++ b/src/dict.c @@ -515,10 +515,10 @@ long long dictFingerprint(dict *d) { long long integers[6], hash = 0; int j; - integers[0] = (long long) d->ht[0].table; + integers[0] = (long) d->ht[0].table; integers[1] = d->ht[0].size; integers[2] = d->ht[0].used; - integers[3] = (long long) d->ht[1].table; + integers[3] = (long) d->ht[1].table; integers[4] = d->ht[1].size; integers[5] = d->ht[1].used; From 7c861340b157a19649ccda55137fa2dfa628fa87 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 20 Aug 2013 12:04:57 +0200 Subject: [PATCH 0782/1752] Use printf %zu specifier to print private_dirty. --- src/aof.c | 2 +- src/rdb.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/aof.c b/src/aof.c index 89f17abab8d..6fb969fa823 100644 --- a/src/aof.c +++ b/src/aof.c @@ -970,7 +970,7 @@ int rewriteAppendOnlyFileBackground(void) { if (private_dirty) { redisLog(REDIS_NOTICE, - "AOF rewrite: %lu MB of memory used by copy-on-write", + "AOF rewrite: %zu MB of memory used by copy-on-write", private_dirty/(1024*1024)); } exitFromChild(0); diff --git a/src/rdb.c b/src/rdb.c index 2581c007496..512dc3cb0b8 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -737,7 +737,7 @@ int rdbSaveBackground(char *filename) { if (private_dirty) { redisLog(REDIS_NOTICE, - "RDB: %lu MB of memory used by copy-on-write", + "RDB: %zu MB of memory used by copy-on-write", private_dirty/(1024*1024)); } } From 0a4656d63f90fa6bd2f3bbb488eaaaf752ed1d83 Mon Sep 17 00:00:00 2001 From: Allan Date: Wed, 24 Jul 2013 21:34:55 +0800 Subject: [PATCH 0783/1752] fixed bug issue of #1213 --- src/redis.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 0696c8ee3fa..bdfe607a31b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1468,9 +1468,12 @@ void initServer() { for (j = 0; j < server.bindaddr_count || j == 0; j++) { if (server.bindaddr[j] == NULL) { /* Bind * for both IPv6 and IPv4. */ - server.ipfd[0] = anetTcp6Server(server.neterr,server.port,NULL); - if (server.ipfd[0] != ANET_ERR) server.ipfd_count++; - server.ipfd[1] = anetTcpServer(server.neterr,server.port,NULL); + server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,NULL); + if (server.ipfd[server.ipfd_count] != ANET_ERR) server.ipfd_count++; + + server.ipfd[server.ipfd_count] = anetTcpServer(server.neterr,server.port,NULL); + + } else if (strchr(server.bindaddr[j],':')) { /* Bind IPv6 address. */ server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,server.bindaddr[j]); From 6c2b34a4656f432c8199c2bba454b64ec1438fd4 Mon Sep 17 00:00:00 2001 From: Allan Date: Thu, 25 Jul 2013 15:28:33 +0800 Subject: [PATCH 0784/1752] fixed initServer failed if no IPV4 or no IPV6 --- src/redis.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/redis.c b/src/redis.c index bdfe607a31b..d591468c1b1 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1436,6 +1436,7 @@ void adjustOpenFilesLimit(void) { void initServer() { int j; + int ip_count; signal(SIGHUP, SIG_IGN); signal(SIGPIPE, SIG_IGN); @@ -1467,12 +1468,18 @@ void initServer() { if (server.bindaddr_count == 0) server.bindaddr[0] = NULL; for (j = 0; j < server.bindaddr_count || j == 0; j++) { if (server.bindaddr[j] == NULL) { - /* Bind * for both IPv6 and IPv4. */ - server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,NULL); - if (server.ipfd[server.ipfd_count] != ANET_ERR) server.ipfd_count++; + /* Bind * for both IPv6 and IPv4. + * Should consider that someone only has IPV6 and someone only get IPV4 */ + ip_count = 0; + server.ipfd[ip_count] = anetTcp6Server(server.neterr,server.port,NULL); + if (server.ipfd[ip_count] != ANET_ERR) ip_count++; - server.ipfd[server.ipfd_count] = anetTcpServer(server.neterr,server.port,NULL); + server.ipfd[ip_count] = anetTcpServer(server.neterr,server.port,NULL); + if(server.ipfd[ip_count] != ANET_ERR ) ip_count++; + /* It should be ip_count plus one + * because out of this branch, the server.ipfd_count would increase */ + server.ipfd_count = ip_count - 1; } else if (strchr(server.bindaddr[j],':')) { /* Bind IPv6 address. */ From 778f753deb2f856ff57dbedf6df7dfd2a2db157d Mon Sep 17 00:00:00 2001 From: Allan Date: Thu, 25 Jul 2013 15:36:00 +0800 Subject: [PATCH 0785/1752] fixed initServer fail while having no IPv6 nor IPv4 --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index d591468c1b1..54562ff3752 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1479,7 +1479,7 @@ void initServer() { /* It should be ip_count plus one * because out of this branch, the server.ipfd_count would increase */ - server.ipfd_count = ip_count - 1; + server.ipfd_count += (ip_count - 1); } else if (strchr(server.bindaddr[j],':')) { /* Bind IPv6 address. */ From 2c94d80f58f2794ae3028a48ebdcc92e25b83e03 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 21 Aug 2013 11:36:09 +0200 Subject: [PATCH 0786/1752] Fix for issue #1214 simplified. --- src/redis.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/src/redis.c b/src/redis.c index 54562ff3752..2a2ce83c089 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1436,7 +1436,6 @@ void adjustOpenFilesLimit(void) { void initServer() { int j; - int ip_count; signal(SIGHUP, SIG_IGN); signal(SIGPIPE, SIG_IGN); @@ -1468,19 +1467,18 @@ void initServer() { if (server.bindaddr_count == 0) server.bindaddr[0] = NULL; for (j = 0; j < server.bindaddr_count || j == 0; j++) { if (server.bindaddr[j] == NULL) { - /* Bind * for both IPv6 and IPv4. - * Should consider that someone only has IPV6 and someone only get IPV4 */ - ip_count = 0; - server.ipfd[ip_count] = anetTcp6Server(server.neterr,server.port,NULL); - if (server.ipfd[ip_count] != ANET_ERR) ip_count++; - - server.ipfd[ip_count] = anetTcpServer(server.neterr,server.port,NULL); - if(server.ipfd[ip_count] != ANET_ERR ) ip_count++; - - /* It should be ip_count plus one - * because out of this branch, the server.ipfd_count would increase */ - server.ipfd_count += (ip_count - 1); - + /* Bind * for both IPv6 and IPv4, we enter here only if + * server.bindaddr_count == 0, so we try to bind and then + * break to exit the loop ASAP. */ + server.ipfd[server.ipfd_count] = + anetTcp6Server(server.neterr,server.port,NULL); + if (server.ipfd[server.ipfd_count] != ANET_ERR) + server.ipfd_count++; + server.ipfd[server.ipfd_count] = + anetTcpServer(server.neterr,server.port,NULL); + if(server.ipfd[server.ipfd_count] != ANET_ERR) + server.ipfd_count++; + break; } else if (strchr(server.bindaddr[j],':')) { /* Bind IPv6 address. */ server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,server.bindaddr[j]); From bd6327855b3a68b9a1de8203fce9f7a8b427d914 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 22 Aug 2013 13:02:59 +0200 Subject: [PATCH 0787/1752] Print error message when can't bind * on any address. --- src/redis.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/redis.c b/src/redis.c index 2a2ce83c089..b87116d2197 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1468,8 +1468,7 @@ void initServer() { for (j = 0; j < server.bindaddr_count || j == 0; j++) { if (server.bindaddr[j] == NULL) { /* Bind * for both IPv6 and IPv4, we enter here only if - * server.bindaddr_count == 0, so we try to bind and then - * break to exit the loop ASAP. */ + * server.bindaddr_count == 0. */ server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,NULL); if (server.ipfd[server.ipfd_count] != ANET_ERR) @@ -1478,7 +1477,10 @@ void initServer() { anetTcpServer(server.neterr,server.port,NULL); if(server.ipfd[server.ipfd_count] != ANET_ERR) server.ipfd_count++; - break; + /* Exit the loop if we were able to bind * on IPv4 or IPv6, + * otherwise server.ipfd[server.ipfd_count] will be ANET_ERR + * and we'll print an error and exit. */ + if (server.ipfd_count) break; } else if (strchr(server.bindaddr[j],':')) { /* Bind IPv6 address. */ server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,server.bindaddr[j]); From f5903846bcb3583302bc96b6dc8d6f639ecb0933 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 22 Aug 2013 14:01:16 +0200 Subject: [PATCH 0788/1752] Opening TCP listening ports refactored into a function. --- src/redis.c | 96 ++++++++++++++++++++++++++++++++--------------------- 1 file changed, 58 insertions(+), 38 deletions(-) diff --git a/src/redis.c b/src/redis.c index b87116d2197..d6fb5bde09e 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1434,6 +1434,61 @@ void adjustOpenFilesLimit(void) { } } +/* Initialize a set of file descriptors to listen to the specified 'port' + * binding the addresses specified in the Redis server configuration. + * + * The listening file descriptors are stored in the integer array 'fds' + * and their number is set in '*count'. + * + * The addresses to bind are specified in the global server.bindaddr array + * and their number is server.bindaddr_count. If the server configuration + * contains no specific addresses to bind, this function will try to + * bind * (all addresses) for both the IPv4 and IPv6 protocols. + * + * On success the function returns REDIS_OK. + * + * On error the function returns REDIS_ERR. For the function to be on + * error, at least one of the server.bindaddr addresses was + * impossible to bind, or no bind addresses were specified in the server + * configuration but the function is not able to bind * for at least + * one of the IPv4 or IPv6 protocols. */ +int listenToPort(int port, int *fds, int *count) { + int j; + + /* Force binding of 0.0.0.0 if no bind address is specified, always + * entering the loop if j == 0. */ + if (server.bindaddr_count == 0) server.bindaddr[0] = NULL; + for (j = 0; j < server.bindaddr_count || j == 0; j++) { + if (server.bindaddr[j] == NULL) { + /* Bind * for both IPv6 and IPv4, we enter here only if + * server.bindaddr_count == 0. */ + fds[*count] = anetTcp6Server(server.neterr,port,NULL); + if (fds[*count] != ANET_ERR) (*count)++; + fds[*count] = anetTcpServer(server.neterr,port,NULL); + if (fds[*count] != ANET_ERR) (*count)++; + /* Exit the loop if we were able to bind * on IPv4 or IPv6, + * otherwise fds[*count] will be ANET_ERR and we'll print an + * error and return to the caller with an error. */ + if (*count) break; + } else if (strchr(server.bindaddr[j],':')) { + /* Bind IPv6 address. */ + fds[*count] = anetTcp6Server(server.neterr,port,server.bindaddr[j]); + } else { + /* Bind IPv4 address. */ + fds[*count] = anetTcpServer(server.neterr,port,server.bindaddr[j]); + } + if (fds[*count] == ANET_ERR) { + redisLog(REDIS_WARNING, + "Creating Server TCP listening socket %s:%d: %s", + server.bindaddr[j] ? server.bindaddr[j] : "*", + server.port, server.neterr); + return REDIS_ERR; + } + (*count)++; + } + return REDIS_OK; +} + void initServer() { int j; @@ -1460,44 +1515,9 @@ void initServer() { server.el = aeCreateEventLoop(server.maxclients+REDIS_EVENTLOOP_FDSET_INCR); server.db = zmalloc(sizeof(redisDb)*server.dbnum); - /* Open the TCP listening sockets. */ - if (server.port != 0) { - /* Force binding of 0.0.0.0 if no bind address is specified, always - * entering the loop if j == 0. */ - if (server.bindaddr_count == 0) server.bindaddr[0] = NULL; - for (j = 0; j < server.bindaddr_count || j == 0; j++) { - if (server.bindaddr[j] == NULL) { - /* Bind * for both IPv6 and IPv4, we enter here only if - * server.bindaddr_count == 0. */ - server.ipfd[server.ipfd_count] = - anetTcp6Server(server.neterr,server.port,NULL); - if (server.ipfd[server.ipfd_count] != ANET_ERR) - server.ipfd_count++; - server.ipfd[server.ipfd_count] = - anetTcpServer(server.neterr,server.port,NULL); - if(server.ipfd[server.ipfd_count] != ANET_ERR) - server.ipfd_count++; - /* Exit the loop if we were able to bind * on IPv4 or IPv6, - * otherwise server.ipfd[server.ipfd_count] will be ANET_ERR - * and we'll print an error and exit. */ - if (server.ipfd_count) break; - } else if (strchr(server.bindaddr[j],':')) { - /* Bind IPv6 address. */ - server.ipfd[server.ipfd_count] = anetTcp6Server(server.neterr,server.port,server.bindaddr[j]); - } else { - /* Bind IPv4 address. */ - server.ipfd[server.ipfd_count] = anetTcpServer(server.neterr,server.port,server.bindaddr[j]); - } - if (server.ipfd[server.ipfd_count] == ANET_ERR) { - redisLog(REDIS_WARNING, - "Creating Server TCP listening socket %s:%d: %s", - server.bindaddr[j] ? server.bindaddr[j] : "*", - server.port, server.neterr); - exit(1); - } - server.ipfd_count++; - } - } + /* Open the TCP listening socket for the user commands. */ + if (listenToPort(server.port,server.ipfd,&server.ipfd_count) == REDIS_ERR) + exit(1); /* Open the listening Unix domain socket. */ if (server.unixsocket != NULL) { From 9df63fcabf5018b20e1624fb2c5851fc9dd8a8bd Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 09:31:43 +0200 Subject: [PATCH 0789/1752] Only run the fast active expire cycle if master & enabled. --- src/redis.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index d6fb5bde09e..54749be0161 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1144,8 +1144,10 @@ void beforeSleep(struct aeEventLoop *eventLoop) { listNode *ln; redisClient *c; - /* Run a fast expire cycle. */ - activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST); + /* Run a fast expire cycle (the called function will return + * ASAP if a fast cycle is not needed). */ + if (server.active_expire_enabled && server.masterhost == NULL) + activeExpireCycle(ACTIVE_EXPIRE_CYCLE_FAST); /* Try to process pending commands for clients that were just unblocked. */ while (listLength(server.unblocked_clients)) { From 9c3e6c3aa098f5b83a350b8817fc26c3a157814d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 10:16:01 +0200 Subject: [PATCH 0790/1752] Update server.lastbgsave_status when fork() fails. --- src/rdb.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rdb.c b/src/rdb.c index 512dc3cb0b8..de21c75fa3e 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -746,6 +746,7 @@ int rdbSaveBackground(char *filename) { /* Parent */ server.stat_fork_time = ustime()-start; if (childpid == -1) { + server.lastbgsave_status = REDIS_ERR; redisLog(REDIS_WARNING,"Can't save in background: fork: %s", strerror(errno)); return REDIS_ERR; From f05b1f6889c1f41caad42e128778ed44c212e4bf Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 11:52:12 +0200 Subject: [PATCH 0791/1752] DEBUG SDSLEN added. This command is only useful for low-level debugging of memory issues due to sds wasting memory as empty buffer at the end of the string. --- src/debug.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/debug.c b/src/debug.c index 0a947e56475..b36bfddafd8 100644 --- a/src/debug.c +++ b/src/debug.c @@ -296,6 +296,29 @@ void debugCommand(redisClient *c) { (void*)val, val->refcount, strenc, (long long) rdbSavedObjectLen(val), val->lru, estimateObjectIdleTime(val)); + } else if (!strcasecmp(c->argv[1]->ptr,"sdslen") && c->argc == 3) { + dictEntry *de; + robj *val; + sds key; + + if ((de = dictFind(c->db->dict,c->argv[2]->ptr)) == NULL) { + addReply(c,shared.nokeyerr); + return; + } + val = dictGetVal(de); + key = dictGetKey(de); + + if (val->type != REDIS_STRING || !sdsEncodedObject(val)) { + addReplyError(c,"Not an sds encoded string."); + } else { + addReplyStatusFormat(c, + "key_sds_len:%lld, key_sds_avail:%lld, " + "val_sds_len:%lld, val_sds_avail:%lld", + (long long) sdslen(key), + (long long) sdsavail(key), + (long long) sdslen(val->ptr), + (long long) sdsavail(val->ptr)); + } } else if (!strcasecmp(c->argv[1]->ptr,"populate") && c->argc == 3) { long keys, j; robj *key, *val; From df21771c698310d4bc6e47c3aaee8dd8abbbede8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 11:54:38 +0200 Subject: [PATCH 0792/1752] Don't over-allocate the sds string for large bulk requests. The call to sdsMakeRoomFor() did not accounted for the amount of data already present in the query buffer, resulting into over-allocation. --- src/networking.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 8a335bc9d44..0de8705389e 100644 --- a/src/networking.c +++ b/src/networking.c @@ -969,13 +969,13 @@ int processMultibulkBuffer(redisClient *c) { if (ll >= REDIS_MBULK_BIG_ARG) { /* If we are going to read a large object from network * try to make it likely that it will start at c->querybuf - * boundary so that we can optimized object creation + * boundary so that we can optimize object creation * avoiding a large copy of data. */ sdsrange(c->querybuf,pos,-1); pos = 0; /* Hint the sds library about the amount of bytes this string is * going to contain. */ - c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2); + c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-sdslen(c->querybuf)); } c->bulklen = ll; } From 0f32c37fc92b5a69cd9940c0dd1df9da6921a644 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 12:26:02 +0200 Subject: [PATCH 0793/1752] Fix DEBUG SDSLEN after 2.8 back port. --- src/debug.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/debug.c b/src/debug.c index b36bfddafd8..7c1a277f769 100644 --- a/src/debug.c +++ b/src/debug.c @@ -308,7 +308,7 @@ void debugCommand(redisClient *c) { val = dictGetVal(de); key = dictGetKey(de); - if (val->type != REDIS_STRING || !sdsEncodedObject(val)) { + if (val->type != REDIS_STRING || val->encoding != REDIS_ENCODING_RAW) { addReplyError(c,"Not an sds encoded string."); } else { addReplyStatusFormat(c, From 79a1d335c85be3b66a89962f71e830df7eea8c35 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 11:56:47 +0200 Subject: [PATCH 0794/1752] tryObjectEncoding(): don't call stringl2() for too big strings. We are sure that a string that is longer than 21 chars cannot be represented by a 64 bit signed integer, as -(2^64) is 21 chars: strlen(-18446744073709551616) => 21 --- src/object.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index 5202c0c55f7..1c9c37e8014 100644 --- a/src/object.c +++ b/src/object.c @@ -278,6 +278,7 @@ int isObjectRepresentableAsLongLong(robj *o, long long *llval) { robj *tryObjectEncoding(robj *o) { long value; sds s = o->ptr; + size_t len; if (o->encoding != REDIS_ENCODING_RAW) return o; /* Already encoded */ @@ -291,7 +292,8 @@ robj *tryObjectEncoding(robj *o) { redisAssertWithInfo(NULL,o,o->type == REDIS_STRING); /* Check if we can represent this string as a long integer */ - if (!string2l(s,sdslen(s),&value)) return o; + len = sdslen(s); + if (len > 21 || !string2l(s,len,&value)) return o; /* Ok, this object can be encoded... * From facc5c0661fa13ff395d9fd250d76161d090072f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 11:59:34 +0200 Subject: [PATCH 0795/1752] tryObjectEncoding(): optimize sds strings if possible. When no encoding is possible, at least try to reallocate the sds string with one that does not waste memory (with free space at the end of the buffer) when the string is large enough. --- src/object.c | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/object.c b/src/object.c index 1c9c37e8014..6095a11540b 100644 --- a/src/object.c +++ b/src/object.c @@ -293,7 +293,26 @@ robj *tryObjectEncoding(robj *o) { /* Check if we can represent this string as a long integer */ len = sdslen(s); - if (len > 21 || !string2l(s,len,&value)) return o; + if (len > 21 || !string2l(s,len,&value)) { + /* We can't encode the object... + * + * Do the last try, and at least optimize the SDS string inside + * the string object to require little space, in case there + * is more than 10% of free space at the end of the SDS string. + * + * We do that for larger strings, using the arbitrary value + * of 32 bytes. This code was backported from the unstable branch + * where this is performed when the object is too large to be + * encoded as EMBSTR. */ + if (len > 32 && + o->encoding == REDIS_ENCODING_RAW && + sdsavail(s) > len/10) + { + o->ptr = sdsRemoveFreeSpace(o->ptr); + } + /* Return the original object. */ + return o; + } /* Ok, this object can be encoded... * From a57294b339c89b87ef8f74ae2b95794dc3cd9b1c Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 27 Aug 2013 13:00:06 +0200 Subject: [PATCH 0796/1752] Fix an hypothetical issue in processMultibulkBuffer(). --- src/networking.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 0de8705389e..ff3d951ffd1 100644 --- a/src/networking.c +++ b/src/networking.c @@ -967,15 +967,19 @@ int processMultibulkBuffer(redisClient *c) { pos += newline-(c->querybuf+pos)+2; if (ll >= REDIS_MBULK_BIG_ARG) { + size_t qblen; + /* If we are going to read a large object from network * try to make it likely that it will start at c->querybuf * boundary so that we can optimize object creation * avoiding a large copy of data. */ sdsrange(c->querybuf,pos,-1); pos = 0; + qblen = sdslen(c->querybuf); /* Hint the sds library about the amount of bytes this string is * going to contain. */ - c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-sdslen(c->querybuf)); + if (qblen < ll+2) + c->querybuf = sdsMakeRoomFor(c->querybuf,ll+2-qblen); } c->bulklen = ll; } From 752b1fca904ddf428d8e1561d9746d004b35f7ea Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Aug 2013 11:49:23 +0200 Subject: [PATCH 0797/1752] Fixed critical memory leak from EVAL. Multiple missing calls to lua_pop prevented the error handler function pushed on the stack for lua_pcall() to be popped before returning, causing a memory leak in almost all the code paths of EVAL (both successful calls and calls returning errors). This caused two issues: Lua leaking memory (and this was very visible from INFO memory output, as the 'used_memory_lua' field reported an always increasing amount of memory used), and as a result slower and slower GC cycles resulting in all the CPU being used. Thanks to Tanguy Le Barzic for noticing something was wrong with his 2.8 slave, and for creating a testing EC2 environment where I was able to investigate the issue. --- src/scripting.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/scripting.c b/src/scripting.c index baf58527923..ac1a913f035 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -876,7 +876,12 @@ void evalGenericCommand(redisClient *c, int evalsha) { addReply(c, shared.noscripterr); return; } - if (luaCreateFunction(c,lua,funcname,c->argv[1]) == REDIS_ERR) return; + if (luaCreateFunction(c,lua,funcname,c->argv[1]) == REDIS_ERR) { + lua_pop(lua,1); /* remove the error handler from the stack. */ + /* The error is sent to the client by luaCreateFunction() + * itself when it returns REDIS_ERR. */ + return; + } /* Now the following is guaranteed to return non nil */ lua_getglobal(lua, funcname); redisAssert(!lua_isnil(lua,-1)); @@ -905,7 +910,6 @@ void evalGenericCommand(redisClient *c, int evalsha) { /* At this point whether this script was never seen before or if it was * already defined, we can call it. We have zero arguments and expect * a single return value. */ - err = lua_pcall(lua,0,1,-2); /* Perform some cleanup that we need to do both on error and success. */ @@ -924,11 +928,12 @@ void evalGenericCommand(redisClient *c, int evalsha) { if (err) { addReplyErrorFormat(c,"Error running script (call to %s): %s\n", funcname, lua_tostring(lua,-1)); - lua_pop(lua,1); /* Consume the Lua reply. */ + lua_pop(lua,2); /* Consume the Lua reply and remove error handler. */ } else { /* On success convert the Lua return value into Redis protocol, and * send it to * the client. */ - luaReplyToRedisReply(c,lua); + luaReplyToRedisReply(c,lua); /* Convert and consume the reply. */ + lua_pop(lua,1); /* Remove the error handler. */ } /* EVALSHA should be propagated to Slave and AOF file as full EVAL, unless From 0ea9a20d476c887276c0d5d57bfcb3b9d5fe2c9a Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 29 Aug 2013 16:23:57 +0200 Subject: [PATCH 0798/1752] Test: added a memory efficiency test. --- tests/test_helper.tcl | 1 + tests/unit/memefficiency.tcl | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 tests/unit/memefficiency.tcl diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 0f7a93434df..1f95129533b 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -45,6 +45,7 @@ set ::all_tests { unit/obuf-limits unit/dump unit/bitops + unit/memefficiency } # Index to the next test to run in the ::all_tests list. set ::next_test 0 diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl new file mode 100644 index 00000000000..3612f06e56f --- /dev/null +++ b/tests/unit/memefficiency.tcl @@ -0,0 +1,32 @@ +proc test_memory_efficiency {range} { + r flushall + set base_mem [s used_memory] + set written 0 + for {set j 0} {$j < 10000} {incr j} { + set key key:$j + set val [string repeat A [expr {int(rand()*$range)}]] + r set $key $val + incr written [string length $key] + incr written [string length $val] + incr written 2 ;# A separator is the minimum to store key-value data. + } + set current_mem [s used_memory] + set used [expr {$current_mem-$base_mem}] + set efficiency [expr {double($written)/$used}] + return $efficiency +} + +start_server {tags {"memefficiency"}} { + foreach {size_range expected_min_efficiency} { + 32 0.15 + 64 0.25 + 128 0.35 + 1024 0.75 + 16384 0.90 + } { + test "Memory efficiency with values in range $size_range" { + set efficiency [test_memory_efficiency $size_range] + assert {$efficiency >= $expected_min_efficiency} + } + } +} From 23acf93f9776dde3903a319592803e33f062d784 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Aug 2013 08:59:11 +0200 Subject: [PATCH 0799/1752] Test: Lua stack leak regression test added. --- tests/unit/scripting.tcl | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/scripting.tcl b/tests/unit/scripting.tcl index 3e08f630c62..ec5230bfe3e 100644 --- a/tests/unit/scripting.tcl +++ b/tests/unit/scripting.tcl @@ -282,6 +282,21 @@ start_server {tags {"scripting"}} { assert {$rand2 ne $rand3} } + test {EVAL does not leak in the Lua stack} { + r set x 0 + # Use a non blocking client to speedup the loop. + set rd [redis_deferring_client] + for {set j 0} {$j < 10000} {incr j} { + $rd eval {return redis.call("incr",KEYS[1])} 1 x + } + for {set j 0} {$j < 10000} {incr j} { + $rd read + } + assert {[s used_memory_lua] < 1024*100} + $rd close + r get x + } {10000} + test {EVAL processes writes from AOF in read-only slaves} { r flushall r config set appendonly yes From 56ce06866448ccf6d08a3b0eb8802aa3749ce5a9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 30 Aug 2013 09:52:06 +0200 Subject: [PATCH 0800/1752] Redis 2.7.104 (2.8 Release Candidate 4). --- 00-RELEASENOTES | 13 +++++++++++++ src/version.h | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 648f9b2d4ae..0ab37a33725 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,19 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8 Release Candidate 4 (2.7.104) ] Release date: 30 Aug 2013 + +This is the fourth release candidate of Redis 2.8 (official version is 2.7.104). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH because of the EVAL memory leak. + +* [FIX] Fixed a serious EVAL memory leak in the Lua stack. +* [FIX] Fixed server startup when no IPv6 address exists in any interface. +* [FIX] Send MISCONFIG error when BGSAVE fails because can't fork. +* [FIX] Memory efficiency with large (> a few kbytes) values improved considerably. +* [NEW] DEBUG SDSLEN for sds memory debugging. + --[ Redis 2.8 Release Candidate 3 (2.7.103) ] Release date: 19 Aug 2013 This is the third release candidate of Redis 2.8 (official version is 2.7.103). diff --git a/src/version.h b/src/version.h index 97cdc352aed..530c13eb3a3 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.103" +#define REDIS_VERSION "2.7.104" From c713b1ebdd8253d13974f5d536896698c72f4f70 Mon Sep 17 00:00:00 2001 From: Maxim Zakharov Date: Wed, 28 Aug 2013 15:44:40 +1000 Subject: [PATCH 0801/1752] A mistype fixed --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index cb455042b14..3abfdf160a0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -189,7 +189,7 @@ void replicationFeedSlaves(list *slaves, int dictid, robj **argv, int argc) { /* Add the multi bulk reply length. */ aux[0] = '*'; - len = ll2string(aux+1,sizeof(aux-1),argc); + len = ll2string(aux+1,sizeof(aux)-1,argc); aux[len+1] = '\r'; aux[len+2] = '\n'; feedReplicationBacklog(aux,len+3); From 9a0f27a5d0a465f01bb133676f0d6f154f73a069 Mon Sep 17 00:00:00 2001 From: Maxim Zakharov Date: Wed, 28 Aug 2013 15:34:52 +1000 Subject: [PATCH 0802/1752] mistype fixed --- src/zipmap.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zipmap.h b/src/zipmap.h index acb25d67a7a..9cf1b248435 100644 --- a/src/zipmap.h +++ b/src/zipmap.h @@ -32,7 +32,7 @@ * POSSIBILITY OF SUCH DAMAGE. */ -#ifndef _ZIMMAP_H +#ifndef _ZIPMAP_H #define _ZIPMAP_H unsigned char *zipmapNew(void); From 2d55f1ca7feb99111cb2e7b3d2845440043a769b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 4 Sep 2013 10:35:53 +0200 Subject: [PATCH 0803/1752] redis-cli: fix big keys search when the key no longer exist. The code freed a reply object that was never created, resulting in a segfault every time randomkey returned a key that was deleted before we queried it for size. --- src/redis-cli.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/redis-cli.c b/src/redis-cli.c index f14b4788239..64173256cdf 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -1323,7 +1323,6 @@ static void findBigKeys(void) { } else if (!strcmp(reply2->str,"none")) { freeReplyObject(reply1); freeReplyObject(reply2); - freeReplyObject(reply3); continue; } else { fprintf(stderr, "Unknown key type '%s' for key '%s'\n", From af967ff9cb8fa230754d2d2ac1cc8988c7fd0f52 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 17 Sep 2013 09:46:01 +0200 Subject: [PATCH 0804/1752] Allow AUTH / PING when disconnected from slave and serve-stale-data is no. --- src/redis.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 54749be0161..aa8dbb16b7f 100644 --- a/src/redis.c +++ b/src/redis.c @@ -208,8 +208,8 @@ struct redisCommand redisCommandTable[] = { {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, - {"auth",authCommand,2,"rsl",0,NULL,0,0,0,0,0}, - {"ping",pingCommand,1,"r",0,NULL,0,0,0,0,0}, + {"auth",authCommand,2,"rslt",0,NULL,0,0,0,0,0}, + {"ping",pingCommand,1,"rt",0,NULL,0,0,0,0,0}, {"echo",echoCommand,2,"r",0,NULL,0,0,0,0,0}, {"save",saveCommand,1,"ars",0,NULL,0,0,0,0,0}, {"bgsave",bgsaveCommand,1,"ar",0,NULL,0,0,0,0,0}, From 20c506f43211924a1b33826f41b79503911242dc Mon Sep 17 00:00:00 2001 From: Michel Martens Date: Sat, 21 Sep 2013 21:36:32 +0200 Subject: [PATCH 0805/1752] Document the redis-cli --csv option. --- src/redis-cli.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/redis-cli.c b/src/redis-cli.c index 64173256cdf..265cea0c17f 100644 --- a/src/redis-cli.c +++ b/src/redis-cli.c @@ -783,6 +783,7 @@ static void usage() { " -c Enable cluster mode (follow -ASK and -MOVED redirections)\n" " --raw Use raw formatting for replies (default when STDOUT is\n" " not a tty)\n" +" --csv Output in CSV format\n" " --latency Enter a special mode continuously sampling latency\n" " --latency-history Like --latency but tracking latency changes over time.\n" " Default time interval is 15 sec. Change it using -i.\n" From 81f614ef6f1eaf5c1c5699cf00b87c1c02da216e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 30 Sep 2013 11:53:18 +0200 Subject: [PATCH 0806/1752] Add REWRITE to CONFIG subcommands help message. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index f657776306d..8c588a06772 100644 --- a/src/config.c +++ b/src/config.c @@ -1732,7 +1732,7 @@ void configCommand(redisClient *c) { } } else { addReplyError(c, - "CONFIG subcommand must be one of GET, SET, RESETSTAT"); + "CONFIG subcommand must be one of GET, SET, RESETSTAT, REWRITE"); } return; From c8c1006cf4dd01ae8df844ae2c8fe36ca1bff5bc Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2013 12:25:09 +0200 Subject: [PATCH 0807/1752] PSYNC: safer handling of PSYNC requests. There was a bug that over-esteemed the amount of backlog available, however this could only happen when a slave was asking for an offset that was in the "future" compared to the master replication backlog. Now this case is handled well and logged as an incident in the master log file. --- src/replication.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 3abfdf160a0..2652b293324 100644 --- a/src/replication.c +++ b/src/replication.c @@ -356,10 +356,14 @@ int masterTryPartialResynchronization(redisClient *c) { REDIS_OK) goto need_full_resync; if (!server.repl_backlog || psync_offset < server.repl_backlog_off || - psync_offset >= (server.repl_backlog_off + server.repl_backlog_size)) + psync_offset > (server.repl_backlog_off + server.repl_backlog_histlen)) { redisLog(REDIS_NOTICE, "Unable to partial resync with the slave for lack of backlog (Slave request was: %lld).", psync_offset); + if (psync_offset > server.master_repl_offset) { + redisLog(REDIS_WARNING, + "Warning: slave tried to PSYNC with an offset that is greater than the master replication offset."); + } goto need_full_resync; } From d7fa6d9aba0bb765d299171c796aeff754261e08 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2013 12:59:24 +0200 Subject: [PATCH 0808/1752] Replication: fix master timeout. Since we started sending REPLCONF ACK from slaves to masters, the lastinteraction field of the client structure is always refreshed as soon as there is room in the socket output buffer, so masters in timeout are detected with too much delay (the socket buffer takes a lot of time to be filled by small REPLCONF ACK entries). This commit only counts data received as interactions with a master, solving the issue. --- src/networking.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index ff3d951ffd1..4233dd306be 100644 --- a/src/networking.c +++ b/src/networking.c @@ -813,7 +813,13 @@ void sendReplyToClient(aeEventLoop *el, int fd, void *privdata, int mask) { return; } } - if (totwritten > 0) c->lastinteraction = server.unixtime; + if (totwritten > 0) { + /* For clients representing masters we don't count sending data + * as an interaction, since we always send REPLCONF ACK commands + * that take some time to just fill the socket output buffer. + * We just rely on data / pings received for timeout detection. */ + if (!(c->flags & REDIS_MASTER)) c->lastinteraction = server.unixtime; + } if (c->bufpos == 0 && listLength(c->reply) == 0) { c->sentlen = 0; aeDeleteFileEvent(server.el,c->fd,AE_WRITABLE); From df0c96002d76e52093045416636415ba2e39a346 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 4 Oct 2013 16:12:25 +0200 Subject: [PATCH 0809/1752] Replication: install the write handler when reusing a cached master. Sometimes when we resurrect a cached master after a successful partial resynchronization attempt, there is pending data in the output buffers of the client structure representing the master (likely REPLCONF ACK commands). If we don't reinstall the write handler, it will never be installed again by addReply*() family functions as they'll assume that if there is already data pending, the write handler is already installed. This bug caused some slaves after a successful partial sync to never send REPLCONF ACK, and continuously being detected as timing out by the master, with a disconnection / reconnection loop. --- src/replication.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/replication.c b/src/replication.c index 2652b293324..c0018cec84b 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1347,6 +1347,16 @@ void replicationResurrectCachedMaster(int newfd) { redisLog(REDIS_WARNING,"Error resurrecting the cached master, impossible to add the readable handler: %s", strerror(errno)); freeClientAsync(server.master); /* Close ASAP. */ } + + /* We may also need to install the write handler as well if there is + * pending data in the write buffers. */ + if (server.master->bufpos || listLength(server.master->reply)) { + if (aeCreateFileEvent(server.el, newfd, AE_WRITABLE, + sendReplyToClient, server.master)) { + redisLog(REDIS_WARNING,"Error resurrecting the cached master, impossible to add the writable handler: %s", strerror(errno)); + freeClientAsync(server.master); /* Close ASAP. */ + } + } } /* ------------------------- MIN-SLAVES-TO-WRITE --------------------------- */ From 508d3c1bb70e86fc02bbec197c09c6947c01f1a9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 9 Oct 2013 13:08:42 +0200 Subject: [PATCH 0810/1752] Redis 2.7.105 (2.8 Release Candidate 5). --- 00-RELEASENOTES | 13 +++++++++++++ src/version.h | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 0ab37a33725..c060974549e 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,19 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8 Release Candidate 5 (2.7.105) ] Release date: 9 Oct 2013 + +This is the 5th release candidate of Redis 2.8 (official version is 2.7.105). +Important bugs fixed inside. + +# UPGRADE URGENCY: HIGH because of many non critical replication bugs fixed. + +* [FIX] redis-cli: don't crash with --bigkeys when the key no longer exist. +* [FIX] Allow AUTH / PING when disconnected from slave and serve-stale-data is no. +* [FIX] PSYNC: safer handling of PSYNC requests with offsets in the future. +* [FIX] Replication: Fix master timeout detection. +* [FIX] Replication: Correctly install the write handler after successful PSYNC. + --[ Redis 2.8 Release Candidate 4 (2.7.104) ] Release date: 30 Aug 2013 This is the fourth release candidate of Redis 2.8 (official version is 2.7.104). diff --git a/src/version.h b/src/version.h index 530c13eb3a3..a1ef1977fc4 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.104" +#define REDIS_VERSION "2.7.105" From 9546f7846b2ca40d966ffa8bb0744850140b7eb3 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 18:13:39 +0100 Subject: [PATCH 0811/1752] redis-benchmark: update help for new __rand_int__ form. --- src/redis-benchmark.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/redis-benchmark.c b/src/redis-benchmark.c index 59b906ed76d..bf18d2ab398 100644 --- a/src/redis-benchmark.c +++ b/src/redis-benchmark.c @@ -556,12 +556,11 @@ int parseOptions(int argc, const char **argv) { " -dbnum SELECT the specified db number (default 0)\n" " -k 1=keep alive 0=reconnect (default 1)\n" " -r Use random keys for SET/GET/INCR, random values for SADD\n" -" Using this option the benchmark will get/set keys\n" -" in the form mykey_rand:000000012456 instead of constant\n" -" keys, the argument determines the max\n" -" number of values for the random number. For instance\n" -" if set to 10 only rand:000000000000 - rand:000000000009\n" -" range will be allowed.\n" +" Using this option the benchmark will expand the string __rand_int__\n" +" inside an argument with a 12 digits number in the specified range\n" +" from 0 to keyspacelen-1. The substitution changes every time a command\n" +" is executed. Default tests use this to hit random keys in the\n" +" specified range.\n" " -P Pipeline requests. Default 1 (no pipeline).\n" " -q Quiet. Just show query/sec values\n" " --csv Output in CSV format\n" From 0dd95e23ab9d61a93ff706856322d8edfda33336 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 9 Jul 2012 01:00:26 -0700 Subject: [PATCH 0812/1752] Add SCAN command --- src/db.c | 100 +++++++++++++++++++++++++++++++++++++++++++ src/dict.c | 92 +++++++++++++++++++++++++++++++++++++++ src/dict.h | 3 ++ src/redis.c | 1 + src/redis.h | 1 + tests/unit/basic.tcl | 61 ++++++++++++++++++++++++++ 6 files changed, 258 insertions(+) diff --git a/src/db.c b/src/db.c index 34e9d3476fd..c544307044b 100644 --- a/src/db.c +++ b/src/db.c @@ -309,6 +309,106 @@ void keysCommand(redisClient *c) { setDeferredMultiBulkLength(c,replylen,numkeys); } +void scanCallback(void *privdata, const dictEntry *de) { + list *keys = (list *)privdata; + sds key = dictGetKey(de); + robj *kobj = createStringObject(key, sdslen(key)); + listAddNodeTail(keys, kobj); +} + +void scanCommand(redisClient *c) { + int rv; + int i, j; + char buf[32]; + list *keys = listCreate(); + listNode *ln, *ln_; + unsigned long cursor = 0; + long count = 1; + sds pat; + int patlen, patnoop = 1; + + /* Use sscanf because we need an *unsigned* long */ + rv = sscanf(c->argv[1]->ptr, "%lu", &cursor); + if (rv != 1) { + addReplyError(c, "invalid cursor"); + goto cleanup; + } + + i = 2; + while (i < c->argc) { + j = c->argc - i; + if (!strcasecmp(c->argv[i]->ptr, "count") && j >= 2) { + if (getLongFromObjectOrReply(c, c->argv[i+1], &count, NULL) != REDIS_OK) { + goto cleanup; + } + + if (count < 1) { + addReply(c,shared.syntaxerr); + goto cleanup; + } + + i += 2; + } else if (!strcasecmp(c->argv[i]->ptr, "pattern") && j >= 2) { + pat = c->argv[i+1]->ptr; + patlen = sdslen(pat); + + /* The pattern is a no-op iff == "*" */ + patnoop = (pat[0] == '*' && patlen == 1); + + i += 2; + } else { + addReply(c,shared.syntaxerr); + goto cleanup; + } + } + + do { + cursor = dictScan(c->db->dict, cursor, scanCallback, keys); + } while (cursor && listLength(keys) < count); + + /* Filter keys */ + ln = listFirst(keys); + while (ln) { + robj *kobj = listNodeValue(ln); + ln_ = listNextNode(ln); + + /* Keep key iff pattern matches and it hasn't expired */ + if ((patnoop || stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) && + (expireIfNeeded(c->db, kobj) == 0)) + { + /* Keep */ + } else { + decrRefCount(kobj); + listDelNode(keys, ln); + } + + ln = ln_; + } + + addReplyMultiBulkLen(c, 2); + + rv = snprintf(buf, sizeof(buf), "%lu", cursor); + redisAssert(rv < sizeof(buf)); + addReplyBulkCBuffer(c, buf, rv); + + addReplyMultiBulkLen(c, listLength(keys)); + while ((ln = listFirst(keys)) != NULL) { + robj *kobj = listNodeValue(ln); + addReplyBulk(c, kobj); + decrRefCount(kobj); + listDelNode(keys, ln); + } + +cleanup: + while ((ln = listFirst(keys)) != NULL) { + robj *kobj = listNodeValue(ln); + decrRefCount(kobj); + listDelNode(keys, ln); + } + + listRelease(keys); +} + void dbsizeCommand(redisClient *c) { addReplyLongLong(c,dictSize(c->db->dict)); } diff --git a/src/dict.c b/src/dict.c index ee560c822cb..27ee5eaf4d9 100644 --- a/src/dict.c +++ b/src/dict.c @@ -648,6 +648,98 @@ dictEntry *dictGetRandomKey(dict *d) return he; } +/* Function to reverse bits. Algorithm from: + * http://graphics.stanford.edu/~seander/bithacks.html#ReverseParallel */ +static unsigned long rev(unsigned long v) { + unsigned long s = 8 * sizeof(v); // bit size; must be power of 2 + unsigned long mask = ~0; + while ((s >>= 1) > 0) { + mask ^= (mask << s); + v = ((v >> s) & mask) | ((v << s) & ~mask); + } + return v; +} + +unsigned long dictScan(dict *d, + unsigned long v, + dictScanFunction *fn, + void *privdata) +{ + dictht *t0, *t1; + const dictEntry *de; + unsigned long s0, s1; + unsigned long m0, m1; + + if (!dictIsRehashing(d)) { + t0 = &(d->ht[0]); + m0 = t0->sizemask; + + /* Emit entries at cursor */ + de = t0->table[v & m0]; + while (de) { + fn(privdata, de); + de = de->next; + } + + } else { + t0 = &d->ht[0]; + t1 = &d->ht[1]; + + /* Make sure t0 is the smaller and t1 is the bigger table */ + if (t0->size > t1->size) { + t0 = &d->ht[1]; + t1 = &d->ht[0]; + } + + s0 = t0->size; + s1 = t1->size; + m0 = t0->sizemask; + m1 = t1->sizemask; + + /* Emit entries at cursor */ + de = t0->table[v & m0]; + while (de) { + fn(privdata, de); + de = de->next; + } + + /* Iterate over indices in larger table that are the expansion + * of the index pointed to by the cursor in the smaller table */ + do { + /* Emit entries at cursor */ + de = t1->table[v & m1]; + while (de) { + fn(privdata, de); + de = de->next; + } + + /* Increment bits not covered by the smaller mask */ + v = (((v | m0) + 1) & ~m0) | (v & m0); + + /* Continue while bits covered by mask difference is non-zero */ + } while (v & (m0 ^ m1)); + } + + /* Set unmasked bits so incrementing the reversed cursor + * operates on the masked bits of the smaller table */ + v |= ~m0; + + /* Increment the reverse cursor */ + v = rev(v); + v++; + v = rev(v); + + /* Only preprare cursor for the next iteration when it is non-zero, + * so that 0 can be used as end-of-scan sentinel. */ + if (v) { + /* Set unmasked bits so the cursor will keep its position + * regardless of the mask in the next iterations */ + v |= ~m0; + } + + return v; +} + /* ------------------------- private functions ------------------------------ */ /* Expand the hash table if needed */ diff --git a/src/dict.h b/src/dict.h index 4d750ae8591..11e1b97eedf 100644 --- a/src/dict.h +++ b/src/dict.h @@ -91,6 +91,8 @@ typedef struct dictIterator { long long fingerprint; /* unsafe iterator fingerprint for misuse detection */ } dictIterator; +typedef void (dictScanFunction)(void *privdata, const dictEntry *de); + /* This is the initial size of every hash table */ #define DICT_HT_INITIAL_SIZE 4 @@ -165,6 +167,7 @@ int dictRehash(dict *d, int n); int dictRehashMilliseconds(dict *d, int ms); void dictSetHashFunctionSeed(unsigned int initval); unsigned int dictGetHashFunctionSeed(void); +unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, void *privdata); /* Hash table types */ extern dictType dictTypeHeapStringCopyKey; diff --git a/src/redis.c b/src/redis.c index aa8dbb16b7f..e06c6d10fc3 100644 --- a/src/redis.c +++ b/src/redis.c @@ -207,6 +207,7 @@ struct redisCommand redisCommandTable[] = { {"pexpire",pexpireCommand,3,"w",0,NULL,1,1,1,0,0}, {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0}, + {"scan",scanCommand,-1,"RS",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, {"auth",authCommand,2,"rslt",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"rt",0,NULL,0,0,0,0,0}, diff --git a/src/redis.h b/src/redis.h index a3028689b42..5de7819ba8c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1214,6 +1214,7 @@ void incrbyfloatCommand(redisClient *c); void selectCommand(redisClient *c); void randomkeyCommand(redisClient *c); void keysCommand(redisClient *c); +void scanCommand(redisClient *c); void dbsizeCommand(redisClient *c); void lastsaveCommand(redisClient *c); void saveCommand(redisClient *c); diff --git a/tests/unit/basic.tcl b/tests/unit/basic.tcl index c766b3de998..a4a0e791a58 100644 --- a/tests/unit/basic.tcl +++ b/tests/unit/basic.tcl @@ -754,4 +754,65 @@ start_server {tags {"basic"}} { set ttl [r ttl foo] assert {$ttl <= 10 && $ttl > 5} } + + test {KEYS * two times with long key, Github issue #1208} { + r flushdb + r set dlskeriewrioeuwqoirueioqwrueoqwrueqw test + r keys * + r keys * + } {dlskeriewrioeuwqoirueioqwrueoqwrueqw} + + test "SCAN basic" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys $k + if {$cur == 0} break + } + + set keys [lsort -unique [concat {*}$keys]] + assert_equal 1000 [llength $keys] + } + + test "SCAN COUNT" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur count 5] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys $k + if {$cur == 0} break + } + + set keys [lsort -unique [concat {*}$keys]] + assert_equal 1000 [llength $keys] + } + + test "SCAN PATTERN" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur pattern "key:1??"] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys $k + if {$cur == 0} break + } + + set keys [lsort -unique [concat {*}$keys]] + assert_equal 100 [llength $keys] + } } From 8c4ff679fe8e9e47df0c7ab895a7691b85add35f Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 10 Jul 2012 15:51:43 -0700 Subject: [PATCH 0813/1752] SCAN requires at least 1 argument --- src/db.c | 2 ++ src/redis.c | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index c544307044b..8342c7388f5 100644 --- a/src/db.c +++ b/src/db.c @@ -327,6 +327,8 @@ void scanCommand(redisClient *c) { sds pat; int patlen, patnoop = 1; + redisAssert(c->argc >= 2); + /* Use sscanf because we need an *unsigned* long */ rv = sscanf(c->argv[1]->ptr, "%lu", &cursor); if (rv != 1) { diff --git a/src/redis.c b/src/redis.c index e06c6d10fc3..c04507e0e33 100644 --- a/src/redis.c +++ b/src/redis.c @@ -207,7 +207,7 @@ struct redisCommand redisCommandTable[] = { {"pexpire",pexpireCommand,3,"w",0,NULL,1,1,1,0,0}, {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0}, - {"scan",scanCommand,-1,"RS",0,NULL,0,0,0,0,0}, + {"scan",scanCommand,-2,"RS",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, {"auth",authCommand,2,"rslt",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"rt",0,NULL,0,0,0,0,0}, From fd22106a9806a397656d3dcbd7400e2245e7823e Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 23 Apr 2013 10:02:36 -0700 Subject: [PATCH 0814/1752] Fix error in scan algorithm The irrelevant bits shouldn't be masked to 1. This can result in slots being skipped when the hash table is resized between calls to the iterator. --- src/dict.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/dict.c b/src/dict.c index 27ee5eaf4d9..5c145d6d0ee 100644 --- a/src/dict.c +++ b/src/dict.c @@ -729,14 +729,6 @@ unsigned long dictScan(dict *d, v++; v = rev(v); - /* Only preprare cursor for the next iteration when it is non-zero, - * so that 0 can be used as end-of-scan sentinel. */ - if (v) { - /* Set unmasked bits so the cursor will keep its position - * regardless of the mask in the next iterations */ - v |= ~m0; - } - return v; } From 33409b4c6ca126a9cecdf096b39de7e372ddf431 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:45:32 +0200 Subject: [PATCH 0815/1752] SCAN option name changed: pattern -> match. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 8342c7388f5..16a043ffc9b 100644 --- a/src/db.c +++ b/src/db.c @@ -350,7 +350,7 @@ void scanCommand(redisClient *c) { } i += 2; - } else if (!strcasecmp(c->argv[i]->ptr, "pattern") && j >= 2) { + } else if (!strcasecmp(c->argv[i]->ptr, "match") && j >= 2) { pat = c->argv[i+1]->ptr; patlen = sdslen(pat); From 8cdaf604ced48ebe295aa7a394c7265e7c4d8386 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:45:56 +0200 Subject: [PATCH 0816/1752] Fixed typo in SCAN comment. iff -> if. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 16a043ffc9b..bb6ff833666 100644 --- a/src/db.c +++ b/src/db.c @@ -354,7 +354,7 @@ void scanCommand(redisClient *c) { pat = c->argv[i+1]->ptr; patlen = sdslen(pat); - /* The pattern is a no-op iff == "*" */ + /* The pattern is a no-op if == "*" */ patnoop = (pat[0] == '*' && patlen == 1); i += 2; From d1c1f62ed548f12aa245606b793d859f93bf189f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:48:18 +0200 Subject: [PATCH 0817/1752] SCAN: use define REDIS_LONGSTR_SIZE instead of fixed len. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index bb6ff833666..4d89a9234d1 100644 --- a/src/db.c +++ b/src/db.c @@ -319,7 +319,7 @@ void scanCallback(void *privdata, const dictEntry *de) { void scanCommand(redisClient *c) { int rv; int i, j; - char buf[32]; + char buf[REDIS_LONGSTR_SIZE]; list *keys = listCreate(); listNode *ln, *ln_; unsigned long cursor = 0; From b44f1589a48244e428b1af27539a0da839452da8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:49:08 +0200 Subject: [PATCH 0818/1752] SCAN: remove useless assertion, already enforced by command table. --- src/db.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/db.c b/src/db.c index 4d89a9234d1..06af89724b7 100644 --- a/src/db.c +++ b/src/db.c @@ -327,8 +327,6 @@ void scanCommand(redisClient *c) { sds pat; int patlen, patnoop = 1; - redisAssert(c->argc >= 2); - /* Use sscanf because we need an *unsigned* long */ rv = sscanf(c->argv[1]->ptr, "%lu", &cursor); if (rv != 1) { From 907e06cd61dd09679d6fa91327dbb354b9ed0c1e Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:51:08 +0200 Subject: [PATCH 0819/1752] SCAN: remove additional newlines to conform to Redis code base. --- src/db.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/db.c b/src/db.c index 06af89724b7..9c692478c39 100644 --- a/src/db.c +++ b/src/db.c @@ -381,12 +381,10 @@ void scanCommand(redisClient *c) { decrRefCount(kobj); listDelNode(keys, ln); } - ln = ln_; } addReplyMultiBulkLen(c, 2); - rv = snprintf(buf, sizeof(buf), "%lu", cursor); redisAssert(rv < sizeof(buf)); addReplyBulkCBuffer(c, buf, rv); @@ -405,7 +403,6 @@ void scanCommand(redisClient *c) { decrRefCount(kobj); listDelNode(keys, ln); } - listRelease(keys); } From 21da9b06a446a60a9ed3ebb8062b26d11a8a2a7f Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:54:37 +0200 Subject: [PATCH 0820/1752] SCAN: improve variable names for readability. --- src/db.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/db.c b/src/db.c index 9c692478c39..34c0bc194d1 100644 --- a/src/db.c +++ b/src/db.c @@ -321,7 +321,7 @@ void scanCommand(redisClient *c) { int i, j; char buf[REDIS_LONGSTR_SIZE]; list *keys = listCreate(); - listNode *ln, *ln_; + listNode *node, *nextnode; unsigned long cursor = 0; long count = 1; sds pat; @@ -367,10 +367,10 @@ void scanCommand(redisClient *c) { } while (cursor && listLength(keys) < count); /* Filter keys */ - ln = listFirst(keys); - while (ln) { - robj *kobj = listNodeValue(ln); - ln_ = listNextNode(ln); + node = listFirst(keys); + while (node) { + robj *kobj = listNodeValue(node); + nextnode = listNextNode(node); /* Keep key iff pattern matches and it hasn't expired */ if ((patnoop || stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) && @@ -379,9 +379,9 @@ void scanCommand(redisClient *c) { /* Keep */ } else { decrRefCount(kobj); - listDelNode(keys, ln); + listDelNode(keys, node); } - ln = ln_; + node = nextnode; } addReplyMultiBulkLen(c, 2); @@ -390,18 +390,18 @@ void scanCommand(redisClient *c) { addReplyBulkCBuffer(c, buf, rv); addReplyMultiBulkLen(c, listLength(keys)); - while ((ln = listFirst(keys)) != NULL) { - robj *kobj = listNodeValue(ln); + while ((node = listFirst(keys)) != NULL) { + robj *kobj = listNodeValue(node); addReplyBulk(c, kobj); decrRefCount(kobj); - listDelNode(keys, ln); + listDelNode(keys, node); } cleanup: - while ((ln = listFirst(keys)) != NULL) { - robj *kobj = listNodeValue(ln); + while ((node = listFirst(keys)) != NULL) { + robj *kobj = listNodeValue(node); decrRefCount(kobj); - listDelNode(keys, ln); + listDelNode(keys, node); } listRelease(keys); } From 108c6e6734de62c90c2caeebd24da1c045bcb3c1 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:55:28 +0200 Subject: [PATCH 0821/1752] SCAN: Fix test after option renamed from PATTERN to MATCH. --- tests/unit/basic.tcl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/basic.tcl b/tests/unit/basic.tcl index a4a0e791a58..67651666e4e 100644 --- a/tests/unit/basic.tcl +++ b/tests/unit/basic.tcl @@ -798,14 +798,14 @@ start_server {tags {"basic"}} { assert_equal 1000 [llength $keys] } - test "SCAN PATTERN" { + test "SCAN MATCH" { r flushdb r debug populate 1000 set cur 0 set keys {} while 1 { - set res [r scan $cur pattern "key:1??"] + set res [r scan $cur match "key:1??"] set cur [lindex $res 0] set k [lindex $res 1] lappend keys $k From 5fa5153bf5e7892ce127c1d7fecea23e697788b5 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 11:58:03 +0200 Subject: [PATCH 0822/1752] SCAN: simplify keys list cleanup using listSetFreeMethod(). --- src/db.c | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/db.c b/src/db.c index 34c0bc194d1..dee09d9adeb 100644 --- a/src/db.c +++ b/src/db.c @@ -398,11 +398,7 @@ void scanCommand(redisClient *c) { } cleanup: - while ((node = listFirst(keys)) != NULL) { - robj *kobj = listNodeValue(node); - decrRefCount(kobj); - listDelNode(keys, node); - } + listSetFreeMethod(keys,decrRefCountVoid); listRelease(keys); } From 854b8be616dfd4f51adb925ec4bb9dc6a54aefe3 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 12:00:13 +0200 Subject: [PATCH 0823/1752] Revert "Fixed typo in SCAN comment. iff -> if." Probably here Pieter means "if and only if". This reverts commit 43fdf3b404413f6e3b2c39ac491238c70ecee81c. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index dee09d9adeb..b3a6f493b80 100644 --- a/src/db.c +++ b/src/db.c @@ -352,7 +352,7 @@ void scanCommand(redisClient *c) { pat = c->argv[i+1]->ptr; patlen = sdslen(pat); - /* The pattern is a no-op if == "*" */ + /* The pattern is a no-op iff == "*" */ patnoop = (pat[0] == '*' && patlen == 1); i += 2; From 1d659784f4f5394ecc23f18181d1cf75c681fb89 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 12:01:49 +0200 Subject: [PATCH 0824/1752] SCAN: stay inside 80 cols. --- src/db.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/db.c b/src/db.c index b3a6f493b80..660a91cc7f2 100644 --- a/src/db.c +++ b/src/db.c @@ -338,7 +338,9 @@ void scanCommand(redisClient *c) { while (i < c->argc) { j = c->argc - i; if (!strcasecmp(c->argv[i]->ptr, "count") && j >= 2) { - if (getLongFromObjectOrReply(c, c->argv[i+1], &count, NULL) != REDIS_OK) { + if (getLongFromObjectOrReply(c, c->argv[i+1], &count, NULL) + != REDIS_OK) + { goto cleanup; } @@ -373,7 +375,8 @@ void scanCommand(redisClient *c) { nextnode = listNextNode(node); /* Keep key iff pattern matches and it hasn't expired */ - if ((patnoop || stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) && + if ((patnoop || + stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) && (expireIfNeeded(c->db, kobj) == 0)) { /* Keep */ From 0880bec4e2fe1387bf31a6453d7077ce79f799d6 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 17:01:30 +0200 Subject: [PATCH 0825/1752] dictScan() algorithm documented. --- src/dict.c | 84 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/src/dict.c b/src/dict.c index 5c145d6d0ee..1ac29f1deb1 100644 --- a/src/dict.c +++ b/src/dict.c @@ -660,6 +660,90 @@ static unsigned long rev(unsigned long v) { return v; } +/* dictScan() is used to iterate over the elements of a dictionary. + * + * Iterating works in the following way: + * + * 1) Initially you call the function using a cursor (v) value of 0. + * 2) The function performs one step of the iteration, and returns the + * new cursor value that you must use in the next call. + * 3) When the returned cursor is 0, the iteration is complete. + * + * The function guarantees that all the elements that are present in the + * dictionary from the start to the end of the iteration are returned. + * However it is possible that some element is returned multiple time. + * + * For every element returned, the callback 'fn' passed as argument is + * called, with 'privdata' as first argument and the dictionar entry + * 'de' as second argument. + * + * HOW IT WORKS. + * + * The algorithm used in the iteration was designed by Pieter Noordhuis. + * The main idea is to increment a cursor starting from the higher order + * bits, that is, instead of incrementing the cursor normally, the bits + * of the cursor are reversed, then the cursor is incremented, and finally + * the bits are reversed again. + * + * This strategy is needed because the hash table may be resized from one + * call to the other call of the same iteration. + * + * dict.c hash tables are always power of two in size, and they + * use chaining, so the position of an element in a given table is given + * always by computing the bitwise AND between Hash(key) and SIZE-1 + * (where SIZE-1 is always the mask that is equivalent to taking the rest + * of the division between the Hash of the key and SIZE). + * + * For example if the current hash table size is 64, the mask is + * (in binary) 1111. The position of a key in the hash table will be always + * the last four bits of the hash output, and so forth. + * + * WHAT HAPPENS DURING REHASHING, WHEN YOU HAVE TWO TABLES? + * + * If the hash table grows, elements can go anyway in one multiple of + * the old bucket: for example let's say that we already iterated with + * a 4 bit cursor 1100, since the mask is 1111 (hash table size = 16). + * + * If the hash table will be resized to 64 elements, and the new mask will + * be 111111, the new buckets that you obtain substituting in ??1100 + * either 0 or 1, can be targeted only by keys that we already visited + * when scanning the bucket 1100 in the smaller hash table. + * + * By iterating the higher bits first, because of the inverted counter, the + * cursor does not need to restart if the table size gets bigger, and will + * just continue iterating with cursors that don't have '1100' at the end, + * nor any other combination of final 4 bits already explored. + * + * Similarly when the table size shrinks over time, for example going from + * 16 to 8, If a combination of the lower three bits (the mask for size 8 + * is 111) was already completely explored, it will not be visited again + * as we are sure that, we tried for example, both 0111 and 1111 (all the + * variations of the higher bit) so we don't need to test it again. + * + * But wait... You have *two* tables in a given moment during rehashing! + * + * Yes, this is true, but we always iterate the smaller one of the tables, + * testing also all the expansions of the current cursor into the larger + * table. So for example if the current cursor is 101 and we also have a + * larger table of size 16, we also test (0)101 and (1)101 inside the larger + * table. This reduces the problem back to having only one table, where + * the larger one, if exists, is just an expansion of the smaller one. + * + * LIMITATIONS + * + * This iterator is completely stateless, and this is a huge advantage, + * including no additional memory used. + * + * The disadvantages resulting from this design are: + * + * 1) It is possible that we return duplicated elements. However this is usually + * easy to deal with in the application level. + * 2) The iterator must return multiple elements per call, as it needs to always + * return all the keys chained in a given bucket, and all the expansions, so + * we are sure we don't miss keys moving. + * 3) The reverse cursor is somewhat hard to understand at first, but this + * comment is supposed to help. + */ unsigned long dictScan(dict *d, unsigned long v, dictScanFunction *fn, From 1a06bbe430ff57753c2f722040c7f2ff3edde010 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 25 Oct 2013 17:05:55 +0200 Subject: [PATCH 0826/1752] Fixed typos in dictScan() comment. --- src/dict.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dict.c b/src/dict.c index 1ac29f1deb1..3e010921415 100644 --- a/src/dict.c +++ b/src/dict.c @@ -698,7 +698,7 @@ static unsigned long rev(unsigned long v) { * (in binary) 1111. The position of a key in the hash table will be always * the last four bits of the hash output, and so forth. * - * WHAT HAPPENS DURING REHASHING, WHEN YOU HAVE TWO TABLES? + * WHAT HAPPENS IF THE TABLE CHANGES IN SIZE? * * If the hash table grows, elements can go anyway in one multiple of * the old bucket: for example let's say that we already iterated with @@ -720,7 +720,7 @@ static unsigned long rev(unsigned long v) { * as we are sure that, we tried for example, both 0111 and 1111 (all the * variations of the higher bit) so we don't need to test it again. * - * But wait... You have *two* tables in a given moment during rehashing! + * WAIT... YOU HAVE *TWO* TABLES DURING REHASHING! * * Yes, this is true, but we always iterate the smaller one of the tables, * testing also all the expansions of the current cursor into the larger From 9e62d9e79c5ea516b264387bf1306bbafae867ce Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 11:11:34 +0100 Subject: [PATCH 0827/1752] SCAN: refactored into scanGenericCommand. The new implementation is capable of iterating the keyspace but also sets, hashes, and sorted sets, and can be used to implement SSCAN, ZSCAN and HSCAN. --- src/db.c | 140 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 126 insertions(+), 14 deletions(-) diff --git a/src/db.c b/src/db.c index 660a91cc7f2..402ad378ac7 100644 --- a/src/db.c +++ b/src/db.c @@ -309,32 +309,76 @@ void keysCommand(redisClient *c) { setDeferredMultiBulkLength(c,replylen,numkeys); } +/* This callback is used by scanGenericCommand in order to collect elements + * returned by the dictionary iterator into a list. */ void scanCallback(void *privdata, const dictEntry *de) { - list *keys = (list *)privdata; - sds key = dictGetKey(de); - robj *kobj = createStringObject(key, sdslen(key)); - listAddNodeTail(keys, kobj); + void **pd = (void**) privdata; + list *keys = pd[0]; + robj *o = pd[1]; + robj *key, *val = NULL; + + if (o == NULL) { + sds sdskey = dictGetKey(de); + key = createStringObject(sdskey, sdslen(sdskey)); + } else if (o->type == REDIS_SET) { + key = dictGetKey(de); + incrRefCount(key); + } else if (o->type == REDIS_HASH) { + key = dictGetKey(de); + incrRefCount(key); + val = dictGetVal(de); + incrRefCount(val); + } else if (o->type == REDIS_ZSET) { + key = dictGetKey(de); + incrRefCount(key); + val = createStringObjectFromLongDouble(*(double*)dictGetVal(de)); + } else { + redisPanic("Type not handled in SCAN callback."); + } + + listAddNodeTail(keys, key); + if (val) listAddNodeTail(keys, val); } -void scanCommand(redisClient *c) { +/* This command implements SCAN, HSCAN and SSCAN commands. + * If object 'o' is passed, then it must be an Hash or Set object, otherwise + * if 'o' is NULL the command will operate on the dictionary associated with + * the current database. + * + * When 'o' is not NULL the function assumes that the first argument in + * the client arguments vector is a key so it skips it before iterating + * in order to parse options. + * + * In the case of an Hash object the function returns both the field and value + * of every element on the Hash. */ +void scanGenericCommand(redisClient *c, robj *o) { int rv; int i, j; char buf[REDIS_LONGSTR_SIZE]; list *keys = listCreate(); listNode *node, *nextnode; unsigned long cursor = 0; - long count = 1; + long count = 10; sds pat; int patlen, patnoop = 1; + dict *ht; + + /* Object must be NULL (to iterate keys names), or the type of the object + * must be Set, Sorted Set, or Hash. */ + redisAssert(o == NULL || o->type == REDIS_SET || o->type == REDIS_HASH || + o->type == REDIS_ZSET); + + /* Set i to the first option argument. The previous one is the cursor. */ + i = (o == NULL) ? 2 : 3; /* Skip the key argument if needed. */ /* Use sscanf because we need an *unsigned* long */ - rv = sscanf(c->argv[1]->ptr, "%lu", &cursor); + rv = sscanf(c->argv[i-1]->ptr, "%lu", &cursor); if (rv != 1) { addReplyError(c, "invalid cursor"); goto cleanup; } - i = 2; + /* Step 1: Parse options. */ while (i < c->argc) { j = c->argc - i; if (!strcasecmp(c->argv[i]->ptr, "count") && j >= 2) { @@ -364,29 +408,92 @@ void scanCommand(redisClient *c) { } } - do { - cursor = dictScan(c->db->dict, cursor, scanCallback, keys); - } while (cursor && listLength(keys) < count); + /* Step 2: Iterate the collection. + * + * Note that if the object is encoded with a ziplist, intset, or any other + * representation that is not an hash table, we are sure that it is also + * composed of a small number of elements. So to avoid taking state we + * just return everything inside the object in a single call, setting the + * cursor to zero to signal the end of the iteration. */ + + /* Handle the case of an hash table. */ + ht = NULL; + if (o == NULL) { + ht = c->db->dict; + } else if (o->type == REDIS_SET && o->encoding == REDIS_ENCODING_HT) { + ht = o->ptr; + } else if (o->type == REDIS_HASH && o->encoding == REDIS_ENCODING_HT) { + ht = o->ptr; + count *= 2; /* We return key / value for this type. */ + } else if (o->type == REDIS_ZSET && o->encoding == REDIS_ENCODING_SKIPLIST) { + zset *zs = o->ptr; + ht = zs->dict; + count *= 2; /* We return key / value for this type. */ + } + + if (ht) { + void *privdata[2]; + + /* We pass two pointers to the callback: the list to which it will + * add new elements, and the object containing the dictionary so that + * it is possible to fetch more data in a type-dependent way. */ + privdata[0] = keys; + privdata[1] = o; + do { + cursor = dictScan(ht, cursor, scanCallback, privdata); + } while (cursor && listLength(keys) < count); + } else if (o->type == REDIS_SET) { + int pos = 0; + long long ll; + + while(intsetGet(o->ptr,pos++,&ll)) + listAddNodeTail(keys,createStringObjectFromLongLong(ll)); + } else if (o->type == REDIS_HASH || o->type == REDIS_ZSET) { + unsigned char *p = ziplistIndex(o->ptr,0); + unsigned char *vstr; + unsigned int vlen; + long long vll; + + while(p) { + ziplistGet(p,&vstr,&vlen,&vll); + listAddNodeTail(keys, + (vstr != NULL) ? createStringObject((char*)vstr,vlen) : + createStringObjectFromLongLong(vll)); + ziplistNext(o->ptr,p); + } + } else { + redisPanic("Not handled encoding in SCAN."); + } - /* Filter keys */ + /* Step 3: Filter elements. */ node = listFirst(keys); while (node) { robj *kobj = listNodeValue(node); nextnode = listNextNode(node); - /* Keep key iff pattern matches and it hasn't expired */ + /* Keep key iff pattern matches and, if we are iterating the key + * space, check that the key hasn't expired. */ if ((patnoop || stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) && - (expireIfNeeded(c->db, kobj) == 0)) + (o != NULL || expireIfNeeded(c->db, kobj) == 0)) { /* Keep */ } else { decrRefCount(kobj); listDelNode(keys, node); + /* Also remove the value for hashes and sorted sets. */ + if (o && (o->type == REDIS_ZSET || o->type == REDIS_HASH)) { + node = nextnode; + kobj = listNodeValue(node); + nextnode = listNextNode(node); + decrRefCount(kobj); + listDelNode(keys, node); + } } node = nextnode; } + /* Step 4: Reply to the client. */ addReplyMultiBulkLen(c, 2); rv = snprintf(buf, sizeof(buf), "%lu", cursor); redisAssert(rv < sizeof(buf)); @@ -405,6 +512,11 @@ void scanCommand(redisClient *c) { listRelease(keys); } +/* The SCAN command completely relies on scanGenericCommand. */ +void scanCommand(redisClient *c) { + scanGenericCommand(c,NULL); +} + void dbsizeCommand(redisClient *c) { addReplyLongLong(c,dictSize(c->db->dict)); } From 7f063b1c8c82997ba29d33ab22f86d88b9372568 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 11:13:43 +0100 Subject: [PATCH 0828/1752] SCAN is a random command and does not require output sorting. Sorting the output helps when we want to turn a non-deterministic into a deterministic command, in that case this is not possible. --- src/redis.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index c04507e0e33..c57f200ca6d 100644 --- a/src/redis.c +++ b/src/redis.c @@ -207,7 +207,7 @@ struct redisCommand redisCommandTable[] = { {"pexpire",pexpireCommand,3,"w",0,NULL,1,1,1,0,0}, {"pexpireat",pexpireatCommand,3,"w",0,NULL,1,1,1,0,0}, {"keys",keysCommand,2,"rS",0,NULL,0,0,0,0,0}, - {"scan",scanCommand,-2,"RS",0,NULL,0,0,0,0,0}, + {"scan",scanCommand,-2,"rR",0,NULL,0,0,0,0,0}, {"dbsize",dbsizeCommand,1,"r",0,NULL,0,0,0,0,0}, {"auth",authCommand,2,"rslt",0,NULL,0,0,0,0,0}, {"ping",pingCommand,1,"rt",0,NULL,0,0,0,0,0}, From 17785f40b52416392fb21a2e03707897fa9720bd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 11:17:18 +0100 Subject: [PATCH 0829/1752] dictScan(): empty hash table requires special handling. --- src/dict.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/dict.c b/src/dict.c index 3e010921415..d3b83d52d27 100644 --- a/src/dict.c +++ b/src/dict.c @@ -754,6 +754,8 @@ unsigned long dictScan(dict *d, unsigned long s0, s1; unsigned long m0, m1; + if (dictSize(d) == 0) return 0; + if (!dictIsRehashing(d)) { t0 = &(d->ht[0]); m0 = t0->sizemask; From 24ecabc9953a76e4f76f35b90da76940d84ca591 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 11:17:32 +0100 Subject: [PATCH 0830/1752] SSCAN implemented. --- src/redis.c | 2 ++ src/redis.h | 4 +++- src/t_set.c | 8 ++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index c57f200ca6d..a425be08245 100644 --- a/src/redis.c +++ b/src/redis.c @@ -162,6 +162,7 @@ struct redisCommand redisCommandTable[] = { {"sdiff",sdiffCommand,-2,"rS",0,NULL,1,-1,1,0,0}, {"sdiffstore",sdiffstoreCommand,-3,"wm",0,NULL,1,-1,1,0,0}, {"smembers",sinterCommand,2,"rS",0,NULL,1,1,1,0,0}, + {"sscan",sscanCommand,-3,"rR",0,NULL,1,1,1,0,0}, {"zadd",zaddCommand,-4,"wm",0,NULL,1,1,1,0,0}, {"zincrby",zincrbyCommand,4,"wm",0,NULL,1,1,1,0,0}, {"zrem",zremCommand,-3,"w",0,NULL,1,1,1,0,0}, @@ -1187,6 +1188,7 @@ void createSharedObjects(void) { shared.emptymultibulk = createObject(REDIS_STRING,sdsnew("*0\r\n")); shared.pong = createObject(REDIS_STRING,sdsnew("+PONG\r\n")); shared.queued = createObject(REDIS_STRING,sdsnew("+QUEUED\r\n")); + shared.emptyscan = createObject(REDIS_STRING,sdsnew("*2\r\n$1\r\n0\r\n*0\r\n")); shared.wrongtypeerr = createObject(REDIS_STRING,sdsnew( "-WRONGTYPE Operation against a key holding the wrong kind of value\r\n")); shared.nokeyerr = createObject(REDIS_STRING,sdsnew( diff --git a/src/redis.h b/src/redis.h index 5de7819ba8c..7ba3660936e 100644 --- a/src/redis.h +++ b/src/redis.h @@ -499,7 +499,7 @@ struct sharedObjectsStruct { *masterdownerr, *roslaveerr, *execaborterr, *noautherr, *noreplicaserr, *oomerr, *plus, *messagebulk, *pmessagebulk, *subscribebulk, *unsubscribebulk, *psubscribebulk, *punsubscribebulk, *del, *rpop, *lpop, - *lpush, + *lpush, *emptyscan, *select[REDIS_SHARED_SELECT_CMDS], *integers[REDIS_SHARED_INTEGERS], *mbulkhdr[REDIS_SHARED_BULKHDR_LEN], /* "*\r\n" */ @@ -1167,6 +1167,7 @@ int selectDb(redisClient *c, int id); void signalModifiedKey(redisDb *db, robj *key); void signalFlushedDb(int dbid); unsigned int GetKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count); +void scanGenericCommand(redisClient *c, robj *o); /* API to get key arguments from commands */ #define REDIS_GETKEYS_ALL 0 @@ -1250,6 +1251,7 @@ void sunionCommand(redisClient *c); void sunionstoreCommand(redisClient *c); void sdiffCommand(redisClient *c); void sdiffstoreCommand(redisClient *c); +void sscanCommand(redisClient *c); void syncCommand(redisClient *c); void flushdbCommand(redisClient *c); void flushallCommand(redisClient *c); diff --git a/src/t_set.c b/src/t_set.c index a522cd88ae8..ad17d46108f 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -911,3 +911,11 @@ void sdiffCommand(redisClient *c) { void sdiffstoreCommand(redisClient *c) { sunionDiffGenericCommand(c,c->argv+2,c->argc-2,c->argv[1],REDIS_OP_DIFF); } + +void sscanCommand(redisClient *c) { + robj *set; + + if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || + checkType(c,set,REDIS_SET)) return; + scanGenericCommand(c,set); +} From 1cc7e646af8d01145df3aa1012903553194556df Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 11:32:34 +0100 Subject: [PATCH 0831/1752] HSCAN implemented. --- src/db.c | 2 +- src/redis.c | 1 + src/redis.h | 1 + src/t_hash.c | 8 ++++++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 402ad378ac7..1c91c3c12b7 100644 --- a/src/db.c +++ b/src/db.c @@ -459,7 +459,7 @@ void scanGenericCommand(redisClient *c, robj *o) { listAddNodeTail(keys, (vstr != NULL) ? createStringObject((char*)vstr,vlen) : createStringObjectFromLongLong(vll)); - ziplistNext(o->ptr,p); + p = ziplistNext(o->ptr,p); } } else { redisPanic("Not handled encoding in SCAN."); diff --git a/src/redis.c b/src/redis.c index a425be08245..649c77662dd 100644 --- a/src/redis.c +++ b/src/redis.c @@ -192,6 +192,7 @@ struct redisCommand redisCommandTable[] = { {"hvals",hvalsCommand,2,"rS",0,NULL,1,1,1,0,0}, {"hgetall",hgetallCommand,2,"r",0,NULL,1,1,1,0,0}, {"hexists",hexistsCommand,3,"r",0,NULL,1,1,1,0,0}, + {"hscan",hscanCommand,-3,"rR",0,NULL,1,1,1,0,0}, {"incrby",incrbyCommand,3,"wm",0,NULL,1,1,1,0,0}, {"decrby",decrbyCommand,3,"wm",0,NULL,1,1,1,0,0}, {"incrbyfloat",incrbyfloatCommand,3,"wm",0,NULL,1,1,1,0,0}, diff --git a/src/redis.h b/src/redis.h index 7ba3660936e..16ec5ae894c 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1308,6 +1308,7 @@ void hkeysCommand(redisClient *c); void hvalsCommand(redisClient *c); void hgetallCommand(redisClient *c); void hexistsCommand(redisClient *c); +void hscanCommand(redisClient *c); void configCommand(redisClient *c); void hincrbyCommand(redisClient *c); void hincrbyfloatCommand(redisClient *c); diff --git a/src/t_hash.c b/src/t_hash.c index 959c9ca81d3..0c5e7af7723 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -762,3 +762,11 @@ void hexistsCommand(redisClient *c) { addReply(c, hashTypeExists(o,c->argv[2]) ? shared.cone : shared.czero); } + +void hscanCommand(redisClient *c) { + robj *o; + + if ((o= lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || + checkType(c,o,REDIS_HASH)) return; + scanGenericCommand(c,o); +} From 35816a2e4dece1d2e03c630e20247d8fed3e7945 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 11:36:42 +0100 Subject: [PATCH 0832/1752] ZSCAN implemented. --- src/redis.c | 1 + src/redis.h | 1 + src/t_zset.c | 8 ++++++++ 3 files changed, 10 insertions(+) diff --git a/src/redis.c b/src/redis.c index 649c77662dd..d4ff261ff50 100644 --- a/src/redis.c +++ b/src/redis.c @@ -179,6 +179,7 @@ struct redisCommand redisCommandTable[] = { {"zscore",zscoreCommand,3,"r",0,NULL,1,1,1,0,0}, {"zrank",zrankCommand,3,"r",0,NULL,1,1,1,0,0}, {"zrevrank",zrevrankCommand,3,"r",0,NULL,1,1,1,0,0}, + {"zscan",zscanCommand,-3,"rR",0,NULL,1,1,1,0,0}, {"hset",hsetCommand,4,"wm",0,NULL,1,1,1,0,0}, {"hsetnx",hsetnxCommand,4,"wm",0,NULL,1,1,1,0,0}, {"hget",hgetCommand,3,"r",0,NULL,1,1,1,0,0}, diff --git a/src/redis.h b/src/redis.h index 16ec5ae894c..aecbcb86af8 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1304,6 +1304,7 @@ void hlenCommand(redisClient *c); void zremrangebyrankCommand(redisClient *c); void zunionstoreCommand(redisClient *c); void zinterstoreCommand(redisClient *c); +void zscanCommand(redisClient *c); void hkeysCommand(redisClient *c); void hvalsCommand(redisClient *c); void hgetallCommand(redisClient *c); diff --git a/src/t_zset.c b/src/t_zset.c index b4cb9d00c5b..9178bf550ba 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2204,3 +2204,11 @@ void zrankCommand(redisClient *c) { void zrevrankCommand(redisClient *c) { zrankGenericCommand(c, 1); } + +void zscanCommand(redisClient *c) { + robj *o; + + if ((o= lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || + checkType(c,o,REDIS_ZSET)) return; + scanGenericCommand(c,o); +} From bb2df795e020fb16bdcc7fc5690aea8e0c8e74f1 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Oct 2013 13:20:03 +0100 Subject: [PATCH 0833/1752] Aesthetic fix (missing space) into HSCAN and ZSCAN implementations. Thanks to @badboy for reporting. --- src/t_hash.c | 2 +- src/t_zset.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/t_hash.c b/src/t_hash.c index 0c5e7af7723..23fe487b65d 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -766,7 +766,7 @@ void hexistsCommand(redisClient *c) { void hscanCommand(redisClient *c) { robj *o; - if ((o= lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || checkType(c,o,REDIS_HASH)) return; scanGenericCommand(c,o); } diff --git a/src/t_zset.c b/src/t_zset.c index 9178bf550ba..809fae5944c 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2208,7 +2208,7 @@ void zrevrankCommand(redisClient *c) { void zscanCommand(redisClient *c) { robj *o; - if ((o= lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || + if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || checkType(c,o,REDIS_ZSET)) return; scanGenericCommand(c,o); } From 02617b6e922fee68c22ae8a50dfc09c4a9453f6b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2013 11:34:01 +0100 Subject: [PATCH 0834/1752] SCAN: tests moved to unit/scan.tcl. --- tests/test_helper.tcl | 1 + tests/unit/basic.tcl | 54 ------------------------------------------ tests/unit/scan.tcl | 55 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 56 insertions(+), 54 deletions(-) create mode 100644 tests/unit/scan.tcl diff --git a/tests/test_helper.tcl b/tests/test_helper.tcl index 1f95129533b..344fabb8a4d 100644 --- a/tests/test_helper.tcl +++ b/tests/test_helper.tcl @@ -16,6 +16,7 @@ set ::all_tests { unit/auth unit/protocol unit/basic + unit/scan unit/type/list unit/type/list-2 unit/type/list-3 diff --git a/tests/unit/basic.tcl b/tests/unit/basic.tcl index 67651666e4e..1f46ba66668 100644 --- a/tests/unit/basic.tcl +++ b/tests/unit/basic.tcl @@ -761,58 +761,4 @@ start_server {tags {"basic"}} { r keys * r keys * } {dlskeriewrioeuwqoirueioqwrueoqwrueqw} - - test "SCAN basic" { - r flushdb - r debug populate 1000 - - set cur 0 - set keys {} - while 1 { - set res [r scan $cur] - set cur [lindex $res 0] - set k [lindex $res 1] - lappend keys $k - if {$cur == 0} break - } - - set keys [lsort -unique [concat {*}$keys]] - assert_equal 1000 [llength $keys] - } - - test "SCAN COUNT" { - r flushdb - r debug populate 1000 - - set cur 0 - set keys {} - while 1 { - set res [r scan $cur count 5] - set cur [lindex $res 0] - set k [lindex $res 1] - lappend keys $k - if {$cur == 0} break - } - - set keys [lsort -unique [concat {*}$keys]] - assert_equal 1000 [llength $keys] - } - - test "SCAN MATCH" { - r flushdb - r debug populate 1000 - - set cur 0 - set keys {} - while 1 { - set res [r scan $cur match "key:1??"] - set cur [lindex $res 0] - set k [lindex $res 1] - lappend keys $k - if {$cur == 0} break - } - - set keys [lsort -unique [concat {*}$keys]] - assert_equal 100 [llength $keys] - } } diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl new file mode 100644 index 00000000000..275a4d65624 --- /dev/null +++ b/tests/unit/scan.tcl @@ -0,0 +1,55 @@ +start_server {tags {"scan"}} { + test "SCAN basic" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys $k + if {$cur == 0} break + } + + set keys [lsort -unique [concat {*}$keys]] + assert_equal 1000 [llength $keys] + } + + test "SCAN COUNT" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur count 5] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys $k + if {$cur == 0} break + } + + set keys [lsort -unique [concat {*}$keys]] + assert_equal 1000 [llength $keys] + } + + test "SCAN MATCH" { + r flushdb + r debug populate 1000 + + set cur 0 + set keys {} + while 1 { + set res [r scan $cur match "key:1??"] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys $k + if {$cur == 0} break + } + + set keys [lsort -unique [concat {*}$keys]] + assert_equal 100 [llength $keys] + } +} From bd1962acc4a11f5ee5c8379ede4a09c125f9a904 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2013 11:36:12 +0100 Subject: [PATCH 0835/1752] SCAN test keys sorting turned into more idiomatic Tcl. --- tests/unit/scan.tcl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 275a4d65624..af4bb766751 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -9,11 +9,11 @@ start_server {tags {"scan"}} { set res [r scan $cur] set cur [lindex $res 0] set k [lindex $res 1] - lappend keys $k + lappend keys {*}$k if {$cur == 0} break } - set keys [lsort -unique [concat {*}$keys]] + set keys [lsort -unique $keys] assert_equal 1000 [llength $keys] } @@ -27,11 +27,11 @@ start_server {tags {"scan"}} { set res [r scan $cur count 5] set cur [lindex $res 0] set k [lindex $res 1] - lappend keys $k + lappend keys {*}$k if {$cur == 0} break } - set keys [lsort -unique [concat {*}$keys]] + set keys [lsort -unique $keys] assert_equal 1000 [llength $keys] } @@ -45,11 +45,11 @@ start_server {tags {"scan"}} { set res [r scan $cur match "key:1??"] set cur [lindex $res 0] set k [lindex $res 1] - lappend keys $k + lappend keys {*}$k if {$cur == 0} break } - set keys [lsort -unique [concat {*}$keys]] + set keys [lsort -unique $keys] assert_equal 100 [llength $keys] } } From 9ea69a58ac610af1f2594f3688dceaca9be9d3bc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2013 11:58:04 +0100 Subject: [PATCH 0836/1752] Test: added SSCAN test. --- tests/unit/scan.tcl | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index af4bb766751..280d3141131 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -52,4 +52,38 @@ start_server {tags {"scan"}} { set keys [lsort -unique $keys] assert_equal 100 [llength $keys] } + + foreach enc {intset hashtable} { + test "SSCAN with encoding $enc" { + # Create the Set + r del set + if {$enc eq {intset}} { + set prefix "" + } else { + set prefix "ele:" + } + set elements {} + for {set j 0} {$j < 100} {incr j} { + lappend elements ${prefix}${j} + } + r sadd set {*}$elements + + # Verify that the encoding matches. + assert {[r object encoding set] eq $enc} + + # Test SSCAN + set cur 0 + set keys {} + while 1 { + set res [r sscan set $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys [lsort -unique $keys] + assert_equal 100 [llength $keys] + } + } } From 13e879c9cf2a3fdbe835f6bdd07805aec20bc0d9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2013 16:24:39 +0100 Subject: [PATCH 0837/1752] Test: added HSCAN test. --- tests/unit/scan.tcl | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 280d3141131..02519ff7c8e 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -86,4 +86,44 @@ start_server {tags {"scan"}} { assert_equal 100 [llength $keys] } } + + foreach enc {ziplist hashtable} { + test "HSCAN with encoding $enc" { + # Create the Hash + r del hash + if {$enc eq {ziplist}} { + set count 30 + } else { + set count 1000 + } + set elements {} + for {set j 0} {$j < $count} {incr j} { + lappend elements key:$j $j + } + r hmset hash {*}$elements + + # Verify that the encoding matches. + assert {[r object encoding hash] eq $enc} + + # Test HSCAN + set cur 0 + set keys {} + while 1 { + set res [r hscan hash $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys2 {} + foreach {k v} $keys { + assert {$k eq "key:$v"} + lappend keys2 $k + } + + set keys2 [lsort -unique $keys2] + assert_equal $count [llength $keys2] + } + } } From ab6f4195dd2dc41e21d6a3ceeea90c083f106964 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2013 16:25:47 +0100 Subject: [PATCH 0838/1752] Test: added ZSCAN test. --- tests/unit/scan.tcl | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 02519ff7c8e..8951bcdeeb0 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -126,4 +126,44 @@ start_server {tags {"scan"}} { assert_equal $count [llength $keys2] } } + + foreach enc {ziplist skiplist} { + test "ZSCAN with encoding $enc" { + # Create the Sorted Set + r del zset + if {$enc eq {ziplist}} { + set count 30 + } else { + set count 1000 + } + set elements {} + for {set j 0} {$j < $count} {incr j} { + lappend elements $j key:$j + } + r zadd zset {*}$elements + + # Verify that the encoding matches. + assert {[r object encoding zset] eq $enc} + + # Test ZSCAN + set cur 0 + set keys {} + while 1 { + set res [r zscan zset $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + } + + set keys2 {} + foreach {k v} $keys { + assert {$k eq "key:$v"} + lappend keys2 $k + } + + set keys2 [lsort -unique $keys2] + assert_equal $count [llength $keys2] + } + } } From c53cab5dde035b66a677083ae312becba1665eaa Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 30 Oct 2013 16:50:25 +0100 Subject: [PATCH 0839/1752] Test: added a SCAN test trying to trigger HT resize. --- tests/unit/scan.tcl | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 8951bcdeeb0..71c4e365516 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -166,4 +166,33 @@ start_server {tags {"scan"}} { assert_equal $count [llength $keys2] } } + + test "SCAN guarantees check under write load" { + r flushdb + r debug populate 100 + + # We start scanning here, so keys from 0 to 99 should all be + # reported at the end of the iteration. + set keys {} + while 1 { + set res [r scan $cur] + set cur [lindex $res 0] + set k [lindex $res 1] + lappend keys {*}$k + if {$cur == 0} break + # Write 10 random keys at every SCAN iteration. + for {set j 0} {$j < 10} {incr j} { + r set addedkey:[randomInt 1000] foo + } + } + + set keys2 {} + foreach k $keys { + if {[string length $k] > 6} continue + lappend keys2 $k + } + + set keys2 [lsort -unique $keys2] + assert_equal 100 [llength $keys2] + } } From 141f30f4210fea70b2251aaeb3337085e669d6d3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2013 09:43:21 +0100 Subject: [PATCH 0840/1752] Regression test added for [SHZ]SCAN issue #1354. --- tests/unit/scan.tcl | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 71c4e365516..511dd1fc770 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -195,4 +195,12 @@ start_server {tags {"scan"}} { set keys2 [lsort -unique $keys2] assert_equal 100 [llength $keys2] } + + test "SSCAN with integer encoded object (issue #1345)" { + set objects {1 a} + r del set + r sadd set {*}$objects + set res [r sscan set 0 MATCH *a* COUNT 100] + assert_equal [lsort -unique [lindex $res 1]] {a} + } } From 06455014aa4545c1a26cd3b2317297f3c682cfd5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2013 10:32:33 +0100 Subject: [PATCH 0841/1752] scanGenericCommand() refactoring and handling of integer encoded elements. This commit fixes issue #1354. --- src/db.c | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/src/db.c b/src/db.c index 1c91c3c12b7..720f5e4aab2 100644 --- a/src/db.c +++ b/src/db.c @@ -470,15 +470,28 @@ void scanGenericCommand(redisClient *c, robj *o) { while (node) { robj *kobj = listNodeValue(node); nextnode = listNextNode(node); + int filter = 0; + + /* Filter element if it does not match the pattern. */ + if (!filter && !patnoop) { + if (sdsEncodedObject(kobj)) { + if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) + filter = 1; + } else { + char buf[REDIS_LONGSTR_SIZE]; + int len; + + redisAssert(kobj->encoding == REDIS_ENCODING_INT); + len = ll2string(buf,sizeof(buf),(long)kobj->ptr); + if (!stringmatchlen(pat, patlen, buf, len, 0)) filter = 1; + } + } - /* Keep key iff pattern matches and, if we are iterating the key - * space, check that the key hasn't expired. */ - if ((patnoop || - stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) && - (o != NULL || expireIfNeeded(c->db, kobj) == 0)) - { - /* Keep */ - } else { + /* Filter element if it is an expired key. */ + if (!filter && o == NULL && expireIfNeeded(c->db, kobj)) filter = 1; + + /* Remove the element and its associted value if needed. */ + if (filter) { decrRefCount(kobj); listDelNode(keys, node); /* Also remove the value for hashes and sorted sets. */ From 3cc7d486bc7cae3bf963ee9a9c7206b824939435 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2013 10:35:56 +0100 Subject: [PATCH 0842/1752] Inverted variable boolean value and name after scanGenericCommand() refactoring. --- src/db.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index 720f5e4aab2..29895efe63d 100644 --- a/src/db.c +++ b/src/db.c @@ -360,7 +360,7 @@ void scanGenericCommand(redisClient *c, robj *o) { unsigned long cursor = 0; long count = 10; sds pat; - int patlen, patnoop = 1; + int patlen, use_pattern = 0; dict *ht; /* Object must be NULL (to iterate keys names), or the type of the object @@ -398,8 +398,9 @@ void scanGenericCommand(redisClient *c, robj *o) { pat = c->argv[i+1]->ptr; patlen = sdslen(pat); - /* The pattern is a no-op iff == "*" */ - patnoop = (pat[0] == '*' && patlen == 1); + /* The pattern always matches if it is exactly "*", so it is + * equivalent to disabling it. */ + use_pattern = !(pat[0] == '*' && patlen == 1); i += 2; } else { @@ -473,7 +474,7 @@ void scanGenericCommand(redisClient *c, robj *o) { int filter = 0; /* Filter element if it does not match the pattern. */ - if (!filter && !patnoop) { + if (!filter && use_pattern) { if (sdsEncodedObject(kobj)) { if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) filter = 1; From 3d3e350b128afcb1e1f1865f7dc2423b2e6f4372 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2013 10:37:27 +0100 Subject: [PATCH 0843/1752] SSCAN with integer encoded object test improved. --- tests/unit/scan.tcl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 511dd1fc770..45498a5148e 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -202,5 +202,7 @@ start_server {tags {"scan"}} { r sadd set {*}$objects set res [r sscan set 0 MATCH *a* COUNT 100] assert_equal [lsort -unique [lindex $res 1]] {a} + set res [r sscan set 0 MATCH *1* COUNT 100] + assert_equal [lsort -unique [lindex $res 1]] {1} } } From 59b30b327dfd07e75be90486ff0313c272495ae3 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 31 Oct 2013 18:22:29 +0100 Subject: [PATCH 0844/1752] SCAN: no sdsEncodedObject() API in Redis 2.8. --- src/db.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index 29895efe63d..5f357aa9a8a 100644 --- a/src/db.c +++ b/src/db.c @@ -475,16 +475,16 @@ void scanGenericCommand(redisClient *c, robj *o) { /* Filter element if it does not match the pattern. */ if (!filter && use_pattern) { - if (sdsEncodedObject(kobj)) { - if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) - filter = 1; - } else { + if (kobj->encoding == REDIS_ENCODING_INT) { char buf[REDIS_LONGSTR_SIZE]; int len; redisAssert(kobj->encoding == REDIS_ENCODING_INT); len = ll2string(buf,sizeof(buf),(long)kobj->ptr); if (!stringmatchlen(pat, patlen, buf, len, 0)) filter = 1; + } else { + if (!stringmatchlen(pat, patlen, kobj->ptr, sdslen(kobj->ptr), 0)) + filter = 1; } } From 736e343509feba822fa41b0fac89c093a0e40e9e Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 11:56:11 +0100 Subject: [PATCH 0845/1752] removed not used vars in dictScan(). --- src/dict.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/dict.c b/src/dict.c index d3b83d52d27..54f87773a7c 100644 --- a/src/dict.c +++ b/src/dict.c @@ -751,7 +751,6 @@ unsigned long dictScan(dict *d, { dictht *t0, *t1; const dictEntry *de; - unsigned long s0, s1; unsigned long m0, m1; if (dictSize(d) == 0) return 0; @@ -777,8 +776,6 @@ unsigned long dictScan(dict *d, t1 = &d->ht[0]; } - s0 = t0->size; - s1 = t1->size; m0 = t0->sizemask; m1 = t1->sizemask; From bebbc7f9f7aeee883a48dd82e74fc2980433b168 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 11:57:30 +0100 Subject: [PATCH 0846/1752] Pass int64_t to intsetGet() instead of long long. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 5f357aa9a8a..dd02526579e 100644 --- a/src/db.c +++ b/src/db.c @@ -445,7 +445,7 @@ void scanGenericCommand(redisClient *c, robj *o) { } while (cursor && listLength(keys) < count); } else if (o->type == REDIS_SET) { int pos = 0; - long long ll; + int64_t ll; while(intsetGet(o->ptr,pos++,&ll)) listAddNodeTail(keys,createStringObjectFromLongLong(ll)); From 54a5a7dff8832a59575c45c7bbbb4ea5a77cd000 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 12:16:29 +0100 Subject: [PATCH 0847/1752] HSCAN/ZSCAN: skip value when matching. This fixes issue #1360 and #1362. --- src/db.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index dd02526579e..66b0e6a9f73 100644 --- a/src/db.c +++ b/src/db.c @@ -495,11 +495,16 @@ void scanGenericCommand(redisClient *c, robj *o) { if (filter) { decrRefCount(kobj); listDelNode(keys, node); - /* Also remove the value for hashes and sorted sets. */ - if (o && (o->type == REDIS_ZSET || o->type == REDIS_HASH)) { - node = nextnode; + } + + /* If this is an hash or a sorted set, we have a flat list of + * key-value elements, so if this element was filtered, remove the + * value, or skip it if it was not filtered: we only match keys. */ + if (o && (o->type == REDIS_ZSET || o->type == REDIS_HASH)) { + node = nextnode; + nextnode = listNextNode(node); + if (filter) { kobj = listNodeValue(node); - nextnode = listNextNode(node); decrRefCount(kobj); listDelNode(keys, node); } From 4b615eea8ffb59e47f4bce1fa9411b8d147e79a6 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 15:19:44 +0100 Subject: [PATCH 0848/1752] Added tests for [SHZ]SCAN with MATCH. --- tests/unit/scan.tcl | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/unit/scan.tcl b/tests/unit/scan.tcl index 45498a5148e..2b1033e398a 100644 --- a/tests/unit/scan.tcl +++ b/tests/unit/scan.tcl @@ -205,4 +205,25 @@ start_server {tags {"scan"}} { set res [r sscan set 0 MATCH *1* COUNT 100] assert_equal [lsort -unique [lindex $res 1]] {1} } + + test "SSCAN with PATTERN" { + r del mykey + r sadd mykey foo fab fiz foobar 1 2 3 4 + set res [r sscan mykey 0 MATCH foo* COUNT 10000] + lsort -unique [lindex $res 1] + } {foo foobar} + + test "HSCAN with PATTERN" { + r del mykey + r hmset mykey foo 1 fab 2 fiz 3 foobar 10 1 a 2 b 3 c 4 d + set res [r hscan mykey 0 MATCH foo* COUNT 10000] + lsort -unique [lindex $res 1] + } {1 10 foo foobar} + + test "ZSCAN with PATTERN" { + r del mykey + r zadd mykey 1 foo 2 fab 3 fiz 10 foobar + set res [r zscan mykey 0 MATCH foo* COUNT 10000] + lsort -unique [lindex $res 1] + } } From 406be0ea286a6e7be72f2526fc86cfe2a2779500 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 15:30:21 +0100 Subject: [PATCH 0849/1752] Use strtoul() instead of sscanf() in SCAN implementation. --- src/db.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/db.c b/src/db.c index 66b0e6a9f73..70d7a430e42 100644 --- a/src/db.c +++ b/src/db.c @@ -354,7 +354,7 @@ void scanCallback(void *privdata, const dictEntry *de) { void scanGenericCommand(redisClient *c, robj *o) { int rv; int i, j; - char buf[REDIS_LONGSTR_SIZE]; + char buf[REDIS_LONGSTR_SIZE], *eptr; list *keys = listCreate(); listNode *node, *nextnode; unsigned long cursor = 0; @@ -371,9 +371,12 @@ void scanGenericCommand(redisClient *c, robj *o) { /* Set i to the first option argument. The previous one is the cursor. */ i = (o == NULL) ? 2 : 3; /* Skip the key argument if needed. */ - /* Use sscanf because we need an *unsigned* long */ - rv = sscanf(c->argv[i-1]->ptr, "%lu", &cursor); - if (rv != 1) { + /* Use strtoul() because we need an *unsigned* long, so + * getLongLongFromObject() does not cover the whole cursor space. */ + errno = 0; + cursor = strtoul(c->argv[i-1]->ptr, &eptr, 10); + if (isspace(((char*)c->argv[i-1])[0]) || eptr[0] != '\0' || errno == ERANGE) + { addReplyError(c, "invalid cursor"); goto cleanup; } From 162acd8ac2bebe88b476d9253e49279d73ff305b Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 15:32:21 +0100 Subject: [PATCH 0850/1752] SCAN: when iterating ziplists or intsets always return cursor of 0. The previous implementation assumed that the first call always happens with cursor set to 0, this may not be the case, and we want to return 0 anyway otherwise the (broken) client code will loop forever. --- src/db.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/db.c b/src/db.c index 70d7a430e42..1286f824777 100644 --- a/src/db.c +++ b/src/db.c @@ -452,6 +452,7 @@ void scanGenericCommand(redisClient *c, robj *o) { while(intsetGet(o->ptr,pos++,&ll)) listAddNodeTail(keys,createStringObjectFromLongLong(ll)); + cursor = 0; } else if (o->type == REDIS_HASH || o->type == REDIS_ZSET) { unsigned char *p = ziplistIndex(o->ptr,0); unsigned char *vstr; @@ -465,6 +466,7 @@ void scanGenericCommand(redisClient *c, robj *o) { createStringObjectFromLongLong(vll)); p = ziplistNext(o->ptr,p); } + cursor = 0; } else { redisPanic("Not handled encoding in SCAN."); } From 060d56e7eb96ee58bf5e5881fdfb16839f0c65a5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 15:47:50 +0100 Subject: [PATCH 0851/1752] SCAN code refactored to parse cursor first. The previous implementation of SCAN parsed the cursor in the generic function implementing SCAN, SSCAN, HSCAN and ZSCAN. The actual higher-level command implementation only checked for empty keys and return ASAP in that case. The result was that inverting the arguments of, for instance, SSCAN for example and write: SSCAN 0 key Instead of SSCAN key 0 Resulted into no error, since 0 is a non-existing key name very likely. Just the iterator returned no elements at all. In order to fix this issue the code was refactored to extract the function to parse the cursor and return the error. Every higher level command implementation now parses the cursor and later checks if the key exist or not. --- src/db.c | 38 ++++++++++++++++++++++++-------------- src/redis.h | 3 ++- src/t_hash.c | 4 +++- src/t_set.c | 4 +++- src/t_zset.c | 4 +++- 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/src/db.c b/src/db.c index 1286f824777..75505f8480a 100644 --- a/src/db.c +++ b/src/db.c @@ -340,6 +340,25 @@ void scanCallback(void *privdata, const dictEntry *de) { if (val) listAddNodeTail(keys, val); } +/* Try to parse a SCAN cursor stored ad object 'o': + * if the cursor is valid, store it as unsigned integer into *cursor and + * returns REDIS_OK. Otherwise return REDIS_ERR and send an error to the + * client. */ +int parseScanCursorOrReply(redisClient *c, robj *o, unsigned long *cursor) { + char *eptr; + + /* Use strtoul() because we need an *unsigned* long, so + * getLongLongFromObject() does not cover the whole cursor space. */ + errno = 0; + *cursor = strtoul(o->ptr, &eptr, 10); + if (isspace(((char*)o->ptr)[0]) || eptr[0] != '\0' || errno == ERANGE) + { + addReplyError(c, "invalid cursor"); + return REDIS_ERR; + } + return REDIS_OK; +} + /* This command implements SCAN, HSCAN and SSCAN commands. * If object 'o' is passed, then it must be an Hash or Set object, otherwise * if 'o' is NULL the command will operate on the dictionary associated with @@ -351,13 +370,12 @@ void scanCallback(void *privdata, const dictEntry *de) { * * In the case of an Hash object the function returns both the field and value * of every element on the Hash. */ -void scanGenericCommand(redisClient *c, robj *o) { +void scanGenericCommand(redisClient *c, robj *o, unsigned long cursor) { int rv; int i, j; - char buf[REDIS_LONGSTR_SIZE], *eptr; + char buf[REDIS_LONGSTR_SIZE]; list *keys = listCreate(); listNode *node, *nextnode; - unsigned long cursor = 0; long count = 10; sds pat; int patlen, use_pattern = 0; @@ -371,16 +389,6 @@ void scanGenericCommand(redisClient *c, robj *o) { /* Set i to the first option argument. The previous one is the cursor. */ i = (o == NULL) ? 2 : 3; /* Skip the key argument if needed. */ - /* Use strtoul() because we need an *unsigned* long, so - * getLongLongFromObject() does not cover the whole cursor space. */ - errno = 0; - cursor = strtoul(c->argv[i-1]->ptr, &eptr, 10); - if (isspace(((char*)c->argv[i-1])[0]) || eptr[0] != '\0' || errno == ERANGE) - { - addReplyError(c, "invalid cursor"); - goto cleanup; - } - /* Step 1: Parse options. */ while (i < c->argc) { j = c->argc - i; @@ -538,7 +546,9 @@ void scanGenericCommand(redisClient *c, robj *o) { /* The SCAN command completely relies on scanGenericCommand. */ void scanCommand(redisClient *c) { - scanGenericCommand(c,NULL); + unsigned long cursor; + if (parseScanCursorOrReply(c,c->argv[1],&cursor) == REDIS_ERR) return; + scanGenericCommand(c,NULL,cursor); } void dbsizeCommand(redisClient *c) { diff --git a/src/redis.h b/src/redis.h index aecbcb86af8..3a0e9df008a 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1167,7 +1167,8 @@ int selectDb(redisClient *c, int id); void signalModifiedKey(redisDb *db, robj *key); void signalFlushedDb(int dbid); unsigned int GetKeysInSlot(unsigned int hashslot, robj **keys, unsigned int count); -void scanGenericCommand(redisClient *c, robj *o); +void scanGenericCommand(redisClient *c, robj *o, unsigned long cursor); +int parseScanCursorOrReply(redisClient *c, robj *o, unsigned long *cursor); /* API to get key arguments from commands */ #define REDIS_GETKEYS_ALL 0 diff --git a/src/t_hash.c b/src/t_hash.c index 23fe487b65d..6bec7068685 100644 --- a/src/t_hash.c +++ b/src/t_hash.c @@ -765,8 +765,10 @@ void hexistsCommand(redisClient *c) { void hscanCommand(redisClient *c) { robj *o; + unsigned long cursor; + if (parseScanCursorOrReply(c,c->argv[2],&cursor) == REDIS_ERR) return; if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || checkType(c,o,REDIS_HASH)) return; - scanGenericCommand(c,o); + scanGenericCommand(c,o,cursor); } diff --git a/src/t_set.c b/src/t_set.c index ad17d46108f..7b586f6ddc0 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -914,8 +914,10 @@ void sdiffstoreCommand(redisClient *c) { void sscanCommand(redisClient *c) { robj *set; + unsigned long cursor; + if (parseScanCursorOrReply(c,c->argv[2],&cursor) == REDIS_ERR) return; if ((set = lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || checkType(c,set,REDIS_SET)) return; - scanGenericCommand(c,set); + scanGenericCommand(c,set,cursor); } diff --git a/src/t_zset.c b/src/t_zset.c index 809fae5944c..cbc4abd9bda 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -2207,8 +2207,10 @@ void zrevrankCommand(redisClient *c) { void zscanCommand(redisClient *c) { robj *o; + unsigned long cursor; + if (parseScanCursorOrReply(c,c->argv[2],&cursor) == REDIS_ERR) return; if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptyscan)) == NULL || checkType(c,o,REDIS_ZSET)) return; - scanGenericCommand(c,o); + scanGenericCommand(c,o,cursor); } From 3a66e0c157771f2f542949969e05b968be2b744d Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 17:23:11 +0100 Subject: [PATCH 0852/1752] Fixed typo in parseScanCursorOrReply(): ad -> at. Thanks to @badboy for reporting it. --- src/db.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/db.c b/src/db.c index 75505f8480a..117ebb7bce6 100644 --- a/src/db.c +++ b/src/db.c @@ -340,7 +340,7 @@ void scanCallback(void *privdata, const dictEntry *de) { if (val) listAddNodeTail(keys, val); } -/* Try to parse a SCAN cursor stored ad object 'o': +/* Try to parse a SCAN cursor stored at object 'o': * if the cursor is valid, store it as unsigned integer into *cursor and * returns REDIS_OK. Otherwise return REDIS_ERR and send an error to the * client. */ From f7f97bf730efe1fdda565298bab7340844437ff9 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Nov 2013 09:24:26 +0100 Subject: [PATCH 0853/1752] Redis 2.7.106 (2.8 Release Candidate 6). --- 00-RELEASENOTES | 8 ++++++++ src/version.h | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index c060974549e..8a3ac32a42e 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,14 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8 Release Candidate 5 (2.7.106) ] Release date: 6 Nov 2013 + +This is the 6th release candidate of Redis 2.8 (official version is 2.7.106). + +# UPGRADE URGENCY: LOW, only new features back ported, no fixes. + +* [NEW] SCAN, SSCAN, HSCAN, ZSCAN commands. + --[ Redis 2.8 Release Candidate 5 (2.7.105) ] Release date: 9 Oct 2013 This is the 5th release candidate of Redis 2.8 (official version is 2.7.105). diff --git a/src/version.h b/src/version.h index a1ef1977fc4..a3fb3c93ae6 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.105" +#define REDIS_VERSION "2.7.106" From 97810c45e8ffa1a8b2bb2646d5927f7f44c50dfc Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 5 Nov 2013 17:23:11 +0100 Subject: [PATCH 0854/1752] Sentinel: always send CONFIG REWRITE when changing instance role. This change makes Sentinel less fragile about a number of failure modes. This commit also fixes a different bug as a side effect, SLAVEOF command was sent multiple times without incrementing the pending commands count. --- src/sentinel.c | 64 +++++++++++++++++++++++++++++++------------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index b257ad685ed..0267f37ea27 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1520,8 +1520,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) { int retval; - retval = redisAsyncCommand(ri->cc, - sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %d", + retval = sentinelSendSlaveOf(ri, ri->master->addr->ip, ri->master->addr->port); if (retval == REDIS_OK) @@ -2523,6 +2522,39 @@ char *sentinelGetObjectiveLeader(sentinelRedisInstance *master) { return winner; } +/* Send SLAVEOF to the specified instance, always followed by a + * CONFIG REWRITE command in order to store the new configuration on disk + * when possible (that is, if the Redis instance is recent enough to support + * config rewriting, and if the server was started with a configuration file). + * + * If Host is NULL the function sends "SLAVEOF NO ONE". + * + * The command returns REDIS_OK if the SLAVEOF command was accepted for + * (later) delivery otherwise REDIS_ERR. The command replies are just + * discarded. */ +int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { + char portstr[32]; + + ll2string(portstr,sizeof(portstr),port); + + if (host == NULL) { + host = "NO"; + memcpy(portstr,"ONE",4); + } + + retval = redisAsyncCommand(ri->cc, + sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %s", host, portstr); + if (retval == REDIS_ERR) return retval; + + ri->pending_commands++; + if (redisAsyncCommand(ri->cc, + sentinelDiscardReplyCallback, NULL, "CONFIG REWRITE") == REDIS_OK) + { + ri->pending_commands++; + } + return REDIS_OK; +} + /* Setup the master state to start a failover as a leader. * * State can be either: @@ -2752,10 +2784,8 @@ void sentinelFailoverSendSlaveOfNoOne(sentinelRedisInstance *ri) { * We actually register a generic callback for this command as we don't * really care about the reply. We check if it worked indirectly observing * if INFO returns a different role (master instead of slave). */ - retval = redisAsyncCommand(ri->promoted_slave->cc, - sentinelDiscardReplyCallback, NULL, "SLAVEOF NO ONE"); + retval = sentinelSendSlaveOf(ri->promoted_slave,NULL,0); if (retval != REDIS_OK) return; - ri->promoted_slave->pending_commands++; sentinelEvent(REDIS_NOTICE, "+failover-state-wait-promotion", ri->promoted_slave,"%@"); ri->failover_state = SENTINEL_FAILOVER_STATE_WAIT_PROMOTION; @@ -2825,10 +2855,6 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) { if (timeout && (master->flags & SRI_I_AM_THE_LEADER)) { dictIterator *di; dictEntry *de; - char master_port[32]; - - ll2string(master_port,sizeof(master_port), - master->promoted_slave->addr->port); di = dictGetIterator(master->slaves); while((de = dictNext(di)) != NULL) { @@ -2838,10 +2864,9 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) { if (slave->flags & (SRI_RECONF_DONE|SRI_RECONF_SENT|SRI_DISCONNECTED)) continue; - retval = redisAsyncCommand(slave->cc, - sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %s", + retval = sentinelSendSlaveOf(slave, master->promoted_slave->addr->ip, - master_port); + master->promoted_slave->addr->port); if (retval == REDIS_OK) { sentinelEvent(REDIS_NOTICE,"+slave-reconf-sent-be",slave,"%@"); slave->flags |= SRI_RECONF_SENT; @@ -2894,15 +2919,11 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) { continue; /* Send SLAVEOF . */ - ll2string(master_port,sizeof(master_port), - master->promoted_slave->addr->port); - retval = redisAsyncCommand(slave->cc, - sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %s", + retval = sentinelSendSlaveOf(slave, master->promoted_slave->addr->ip, - master_port); + master->promoted_slave->addr->port); if (retval == REDIS_OK) { slave->flags |= SRI_RECONF_SENT; - slave->pending_commands++; slave->slave_reconf_sent_time = mstime(); sentinelEvent(REDIS_NOTICE,"+slave-reconf-sent",slave,"%@"); in_progress++; @@ -2984,13 +3005,11 @@ void sentinelFailoverStateMachine(sentinelRedisInstance *ri) { * back to the master as well, sending a best effort SLAVEOF command. */ void sentinelAbortFailover(sentinelRedisInstance *ri) { - char master_port[32]; dictIterator *di; dictEntry *de; int sentinel_role; redisAssert(ri->flags & SRI_FAILOVER_IN_PROGRESS); - ll2string(master_port,sizeof(master_port),ri->addr->port); /* Clear failover related flags from slaves. * Also if we are the leader make sure to send SLAVEOF commands to all the @@ -3006,10 +3025,7 @@ void sentinelAbortFailover(sentinelRedisInstance *ri) { { int retval; - retval = redisAsyncCommand(slave->cc, - sentinelDiscardReplyCallback, NULL, "SLAVEOF %s %s", - ri->addr->ip, - master_port); + retval = sentinelSendSlaveOf(slave,ri->addr->ip,ri->addr->port); if (retval == REDIS_OK) sentinelEvent(REDIS_NOTICE,"-slave-reconf-undo",slave,"%@"); } From 176799875175a4d99d639424bf67c6e2daa69cec Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Nov 2013 11:21:44 +0100 Subject: [PATCH 0855/1752] Sentinel: increment pending_commands counter in two more places. AUTH and SCRIPT KILL were sent without incrementing the pending commands counter. Clearly this needs some kind of wrapper doing it for the caller in order to be less bug prone. --- src/sentinel.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 0267f37ea27..2fdcc225e05 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1300,9 +1300,10 @@ void sentinelSendAuthIfNeeded(sentinelRedisInstance *ri, redisAsyncContext *c) { char *auth_pass = (ri->flags & SRI_MASTER) ? ri->auth_pass : ri->master->auth_pass; - if (auth_pass) - redisAsyncCommand(c, sentinelDiscardReplyCallback, NULL, "AUTH %s", - auth_pass); + if (auth_pass) { + if (redisAsyncCommand(c, sentinelDiscardReplyCallback, NULL, "AUTH %s", + auth_pass) == REDIS_OK) ri->pending_commands++; + } } /* Create the async connections for the specified instance if the instance @@ -1691,8 +1692,10 @@ void sentinelPingReplyCallback(redisAsyncContext *c, void *reply, void *privdata (ri->flags & SRI_S_DOWN) && !(ri->flags & SRI_SCRIPT_KILL_SENT)) { - redisAsyncCommand(ri->cc, - sentinelDiscardReplyCallback, NULL, "SCRIPT KILL"); + if (redisAsyncCommand(ri->cc, + sentinelDiscardReplyCallback, NULL, + "SCRIPT KILL") == REDIS_OK) + ri->pending_commands++; ri->flags |= SRI_SCRIPT_KILL_SENT; } } From c874e39c45a225c4121f54c424b8c155921989dc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 6 Nov 2013 11:23:49 +0100 Subject: [PATCH 0856/1752] Sentinel: sentinelSendSlaveOf() was missing a var and the prototype. --- src/sentinel.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 2fdcc225e05..eccbc7df1e0 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -329,6 +329,7 @@ sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master); void sentinelScheduleScriptExecution(char *path, ...); void sentinelStartFailover(sentinelRedisInstance *master, int state); void sentinelDiscardReplyCallback(redisAsyncContext *c, void *reply, void *privdata); +int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port); /* ========================= Dictionary types =============================== */ @@ -2537,6 +2538,7 @@ char *sentinelGetObjectiveLeader(sentinelRedisInstance *master) { * discarded. */ int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { char portstr[32]; + int retval; ll2string(portstr,sizeof(portstr),port); @@ -2901,7 +2903,6 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) { { sentinelRedisInstance *slave = dictGetVal(de); int retval; - char master_port[32]; /* Skip the promoted slave, and already configured slaves. */ if (slave->flags & (SRI_PROMOTED|SRI_RECONF_DONE)) continue; From 875bc519090c102cd11ca9a8440f9c10132d60dd Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 7 Nov 2013 23:53:18 +0100 Subject: [PATCH 0857/1752] Fix broken rdbWriteRaw() return value check in rdb.c. Thanks to @PhoneLi for reporting. --- src/rdb.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rdb.c b/src/rdb.c index de21c75fa3e..f7dcc171945 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -99,7 +99,7 @@ int rdbSaveLen(rio *rdb, uint32_t len) { buf[0] = (REDIS_RDB_32BITLEN<<6); if (rdbWriteRaw(rdb,buf,1) == -1) return -1; len = htonl(len); - if (rdbWriteRaw(rdb,&len,4) == -4) return -1; + if (rdbWriteRaw(rdb,&len,4) == -1) return -1; nwritten = 1+4; } return nwritten; From 7d30b3121df799810310aebbe226b894abb178df Mon Sep 17 00:00:00 2001 From: Ryan Biesemeyer Date: Wed, 6 Nov 2013 08:31:57 +0000 Subject: [PATCH 0858/1752] Deprecate utils/redis-copy.rb in favor of redis-copy gem --- utils/redis-copy.rb | 67 ++++++++------------------------------------- 1 file changed, 12 insertions(+), 55 deletions(-) diff --git a/utils/redis-copy.rb b/utils/redis-copy.rb index d892e377f38..7c5c52dd6c7 100644 --- a/utils/redis-copy.rb +++ b/utils/redis-copy.rb @@ -3,66 +3,23 @@ # # Copy the whole dataset from one Redis instance to another one # -# WARNING: currently hashes and sorted sets are not supported! This -# program should be updated. +# WARNING: this utility is deprecated and serves as a legacy adapter +# for the more-robust redis-copy gem. -require 'rubygems' -require 'redis' -require 'digest/sha1' +require 'shellwords' def redisCopy(opts={}) - sha1="" - src = Redis.new(:host => opts[:srchost], :port => opts[:srcport]) - dst = Redis.new(:host => opts[:dsthost], :port => opts[:dstport]) - puts "Loading key names..." - keys = src.keys('*') - puts "Copying #{keys.length} keys..." - c = 0 - keys.each{|k| - vtype = src.type?(k) - ttl = src.ttl(k).to_i if vtype != "none" - - if vtype == "string" - dst[k] = src[k] - elsif vtype == "list" - list = src.lrange(k,0,-1) - if list.length == 0 - # Empty list special case - dst.lpush(k,"") - dst.lpop(k) - else - list.each{|ele| - dst.rpush(k,ele) - } - end - elsif vtype == "set" - set = src.smembers(k) - if set.length == 0 - # Empty set special case - dst.sadd(k,"") - dst.srem(k,"") - else - set.each{|ele| - dst.sadd(k,ele) - } - end - elsif vtype == "none" - puts "WARNING: key '#{k}' was removed in the meanwhile." - end - - # Handle keys with an expire time set - if ttl != -1 and vtype != "none" - dst.expire(k,ttl) - end - - c = c+1 - if (c % 1000) == 0 - puts "#{c}/#{keys.length} completed" - end - } - puts "DONE!" + src = "#{opts[:srchost]}:#{opts[:srcport]}" + dst = "#{opts[:dsthost]}:#{opts[:dstport]}" + `redis-copy #{src.shellescape} #{dst.shellescape}` +rescue Errno::ENOENT + $stderr.puts 'This utility requires the redis-copy executable', + 'from the redis-copy gem on https://rubygems.org', + 'To install it, run `gem install redis-copy`.' + exit 1 end +$stderr.puts "This utility is deprecated. Use the redis-copy gem instead." if ARGV.length != 4 puts "Usage: redis-copy.rb " exit 1 From 68b911478fdf06ac7df6b33d69cea360426cf7ae Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 10 Nov 2013 10:25:09 +0100 Subject: [PATCH 0859/1752] Fixed typo in release candidate notes. --- 00-RELEASENOTES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 8a3ac32a42e..1336e7c49a2 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,7 +14,7 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- ---[ Redis 2.8 Release Candidate 5 (2.7.106) ] Release date: 6 Nov 2013 +--[ Redis 2.8 Release Candidate 6 (2.7.106) ] Release date: 6 Nov 2013 This is the 6th release candidate of Redis 2.8 (official version is 2.7.106). From 98682cd178b62602d371e3e4e517cda606ecfdc6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Nov 2013 09:25:36 +0100 Subject: [PATCH 0860/1752] Log to what master a slave is going to connect to. --- src/replication.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index c0018cec84b..2c99a6624d0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1499,7 +1499,8 @@ void replicationCron(void) { /* Check if we should connect to a MASTER */ if (server.repl_state == REDIS_REPL_CONNECT) { - redisLog(REDIS_NOTICE,"Connecting to MASTER..."); + redisLog(REDIS_NOTICE,"Connecting to MASTER %s:%d", + server.masterhost, server.masterport); if (connectWithMaster() == REDIS_OK) { redisLog(REDIS_NOTICE,"MASTER <-> SLAVE sync started"); } From be2ef1b59fd43259c67e909e1f940b66c0f574ae Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Nov 2013 11:05:58 +0100 Subject: [PATCH 0861/1752] Sentinel: epoch introduced. Sentinel state now includes the idea of current epoch and config epoch. In the Hello message, that is now published both on masters and slaves, a Sentinel no longer just advertises itself but also broadcasts its current view of the configuration: the master name / ip / port and its current epoch. Sentinels receiving such information switch to the new master if the configuration epoch received is newer and the ip / port of the master are indeed different compared to the previos ones. --- src/sentinel.c | 87 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 62 insertions(+), 25 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index eccbc7df1e0..edaf284d552 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -135,6 +135,7 @@ typedef struct sentinelRedisInstance { int flags; /* See SRI_... defines */ char *name; /* Master name from the point of view of this sentinel. */ char *runid; /* run ID of this instance. */ + uint64_t config_epoch; /* Configuration epoch. */ sentinelAddr *addr; /* Master host. */ redisAsyncContext *cc; /* Hiredis context for commands. */ redisAsyncContext *pc; /* Hiredis context for Pub / Sub. */ @@ -193,13 +194,14 @@ typedef struct sentinelRedisInstance { /* Main state. */ struct sentinelState { + uint64_t current_epoch; /* Current epoch. */ dict *masters; /* Dictionary of master sentinelRedisInstances. Key is the instance name, value is the sentinelRedisInstance structure pointer. */ int tilt; /* Are we in TILT mode? */ int running_scripts; /* Number of scripts in execution right now. */ mstime_t tilt_start_time; /* When TITL started. */ - mstime_t previous_time; /* Time last time we ran the time handler. */ + mstime_t previous_time; /* Last time we ran the time handler. */ list *scripts_queue; /* Queue of user scripts to execute. */ } sentinel; @@ -404,6 +406,7 @@ void initSentinel(void) { } /* Initialize various data structures. */ + sentinel.current_epoch = 0; sentinel.masters = dictCreate(&instancesDictType,NULL); sentinel.tilt = 0; sentinel.tilt_start_time = 0; @@ -863,6 +866,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * ri->flags = flags | SRI_DISCONNECTED; ri->name = sdsname; ri->runid = NULL; + ri->config_epoch = 0; ri->addr = addr; ri->cc = NULL; ri->pc = NULL; @@ -1747,24 +1751,28 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd if (strstr(r->element[2]->str,server.runid) != NULL) return; { - int numtokens, port, removed, canfailover; - /* Separator changed from ":" to "," in recent versions in order to - * play well with IPv6 addresses. For now we make sure to parse both - * correctly detecting if there is "," inside the string. */ - char *sep = strchr(r->element[2]->str,',') ? "," : ":"; + /* Format is composed of 9 tokens: + * 0=ip,1=port,2=runid,3=can_failover,4=current_epoch, + * 5=master_name,6=master_ip,7=master_port,8=master_config_epoch. */ + int numtokens, port, removed, canfailover, master_port; + uint64_t current_epoch, master_config_epoch; char **token = sdssplitlen(r->element[2]->str, r->element[2]->len, - sep,1,&numtokens); - sentinelRedisInstance *sentinel; + ",",1,&numtokens); + sentinelRedisInstance *si; - if (numtokens == 4) { + if (numtokens == 9) { /* First, try to see if we already have this sentinel. */ port = atoi(token[1]); + master_port = atoi(token[7]); canfailover = atoi(token[3]); - sentinel = getSentinelRedisInstanceByAddrAndRunID( + si = getSentinelRedisInstanceByAddrAndRunID( ri->sentinels,token[0],port,token[2]); + current_epoch = strtoull(token[4],NULL,10); + master_config_epoch = strtoull(token[8],NULL,10); + sentinelRedisInstance *master; - if (!sentinel) { + if (!si) { /* If not, remove all the sentinels that have the same runid * OR the same ip/port, because it's either a restart or a * network topology change. */ @@ -1777,24 +1785,45 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd } /* Add the new sentinel. */ - sentinel = createSentinelRedisInstance(NULL,SRI_SENTINEL, + si = createSentinelRedisInstance(NULL,SRI_SENTINEL, token[0],port,ri->quorum,ri); - if (sentinel) { - sentinelEvent(REDIS_NOTICE,"+sentinel",sentinel,"%@"); + if (si) { + sentinelEvent(REDIS_NOTICE,"+sentinel",si,"%@"); /* The runid is NULL after a new instance creation and * for Sentinels we don't have a later chance to fill it, * so do it now. */ - sentinel->runid = sdsnew(token[2]); + si->runid = sdsnew(token[2]); + } + } + + /* Update local current_epoch if received current_epoch is greater. */ + if (current_epoch > sentinel.current_epoch) + sentinel.current_epoch = current_epoch; + + /* Update master info if received configuration is newer. */ + if ((master = sentinelGetMasterByName(token[5])) != NULL) { + if (master->config_epoch < master_config_epoch) { + master->config_epoch = master_config_epoch; + if (master_port != master->addr->port || + !strcmp(master->addr->ip, token[6])) + { + sentinelEvent(REDIS_WARNING,"+switch-master", + master,"%s %s %d %s %d", + master->name, master->addr->ip, master->addr->port, + token[6], master_port); + sentinelResetMasterAndChangeAddress(ri, + token[6], master_port); + } } } /* Update the state of the Sentinel. */ - if (sentinel) { - sentinel->last_hello_time = mstime(); + if (si) { + si->last_hello_time = mstime(); if (canfailover) - sentinel->flags |= SRI_CAN_FAILOVER; + si->flags |= SRI_CAN_FAILOVER; else - sentinel->flags &= ~SRI_CAN_FAILOVER; + si->flags &= ~SRI_CAN_FAILOVER; } } sdsfreesplitres(token,numtokens); @@ -1844,20 +1873,28 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { sentinelPingReplyCallback, NULL, "PING"); if (retval != REDIS_OK) return; ri->pending_commands++; - } else if ((ri->flags & SRI_MASTER) && + } else if ((ri->flags & SRI_SENTINEL) == 0 && (now - ri->last_pub_time) > SENTINEL_PUBLISH_PERIOD) { - /* PUBLISH hello messages only to masters. */ + /* PUBLISH hello messages to masters and slaves. */ char ip[REDIS_IP_STR_LEN]; if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { - char myaddr[REDIS_IP_STR_LEN+128]; + char payload[REDIS_IP_STR_LEN+1024]; + sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? + NULL : ri->master; - snprintf(myaddr,sizeof(myaddr),"%s,%d,%s,%d", + snprintf(payload,sizeof(payload), + "%s,%d,%s,%d,%llu," /* Info about this sentinel. */ + "%s,%s,%d,%lld", /* Info about current master. */ ip, server.port, server.runid, - (ri->flags & SRI_CAN_FAILOVER) != 0); + (ri->flags & SRI_CAN_FAILOVER) != 0, + (unsigned long long) sentinel.current_epoch, + /* --- */ + master->name,master->addr->ip,master->addr->port, + master->config_epoch); retval = redisAsyncCommand(ri->cc, sentinelPublishReplyCallback, NULL, "PUBLISH %s %s", - SENTINEL_HELLO_CHANNEL,myaddr); + SENTINEL_HELLO_CHANNEL,payload); if (retval != REDIS_OK) return; ri->pending_commands++; } From fe7f96f18c740a1312459e46808538b28ec6f672 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Nov 2013 12:05:16 +0100 Subject: [PATCH 0862/1752] Sentinel: remove code not useful in the new design. --- src/sentinel.c | 171 ++++++++++--------------------------------------- 1 file changed, 33 insertions(+), 138 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index edaf284d552..5d2feb9e938 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -74,8 +74,6 @@ typedef struct sentinelAddr { #define SRI_RECONF_DONE (1<<13) /* Slave synchronized with new master. */ #define SRI_FORCE_FAILOVER (1<<14) /* Force failover with master up. */ #define SRI_SCRIPT_KILL_SENT (1<<15) /* SCRIPT KILL already sent on -BUSY */ -#define SRI_DEMOTE (1<<16) /* If the instance claims to be a master, demote - it into a slave sending SLAVEOF. */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 @@ -92,7 +90,6 @@ typedef struct sentinelAddr { #define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000 #define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*15*1000) #define SENTINEL_MAX_PENDING_COMMANDS 100 -#define SENTINEL_EXTENDED_SDOWN_MULTIPLIER 10 /* How many milliseconds is an information valid? This applies for instance * to the reply to SENTINEL IS-MASTER-DOWN-BY-ADDR replies. */ @@ -1504,108 +1501,44 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Handle slave -> master role switch. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { - if (!sentinel.tilt && ri->flags & SRI_DEMOTE) { - /* If this sentinel was partitioned from the slave's master, - * or tilted recently, wait some time before to act, - * so that DOWN and roles INFO will be refreshed. */ - mstime_t wait_time = SENTINEL_INFO_PERIOD*2 + - ri->master->down_after_period*2; + /* If this is a promoted slave we can change state to the + * failover state machine. */ + if (!sentinel.tilt && + (ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && + (ri->master->flags & SRI_I_AM_THE_LEADER) && + (ri->master->failover_state == + SENTINEL_FAILOVER_STATE_WAIT_PROMOTION)) + { + ri->master->failover_state = SENTINEL_FAILOVER_STATE_RECONF_SLAVES; + ri->master->failover_state_change_time = mstime(); + sentinelEvent(REDIS_WARNING,"+promoted-slave",ri,"%@"); + sentinelEvent(REDIS_WARNING,"+failover-state-reconf-slaves", + ri->master,"%@"); + sentinelCallClientReconfScript(ri->master,SENTINEL_LEADER, + "start",ri->master->addr,ri->addr); + } else if (!sentinel.tilt) { + /* A slave turned into a master. We want to force our view and + * reconfigure as slave, but make sure to wait some time before + * doing this in order to make sure to receive an updated + * configuratio via Pub/Sub if any. */ + mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; if (!sentinelRedisInstanceNoDownFor(ri->master,wait_time) || (mstime()-sentinel.tilt_start_time) < wait_time) return; - /* Old master returned back? Turn it into a slave ASAP if - * we can reach what we believe is the new master now, and - * have a recent role information for it. - * - * Note: we'll clear the DEMOTE flag only when we have the - * acknowledge that it's a slave again. */ + /* Make sure the master is sane before reconfiguring this instance + * into a slave. */ if (ri->master->flags & SRI_MASTER && (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) { - int retval; - retval = sentinelSendSlaveOf(ri, + int retval = sentinelSendSlaveOf(ri, ri->master->addr->ip, ri->master->addr->port); if (retval == REDIS_OK) sentinelEvent(REDIS_NOTICE,"+demote-old-slave",ri,"%@"); - } else { - /* Otherwise if there are not the conditions to demote, we - * no longer trust the DEMOTE flag and remove it. */ - ri->flags &= ~SRI_DEMOTE; - sentinelEvent(REDIS_NOTICE,"-demote-flag-cleared",ri,"%@"); - } - } else if (!(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && - (runid_changed || first_runid)) - { - /* If a slave turned into master but: - * - * 1) Failover not in progress. - * 2) RunID has changed or its the first time we see an INFO output. - * - * We assume this is a reboot with a wrong configuration. - * Log the event and remove the slave. Note that this is processed - * in tilt mode as well, otherwise we lose the information that the - * runid changed (reboot?) and when the tilt mode ends a fake - * failover will be detected. */ - int retval; - - sentinelEvent(REDIS_WARNING,"-slave-restart-as-master",ri,"%@ #removing it from the attached slaves"); - retval = dictDelete(ri->master->slaves,ri->name); - redisAssert(retval == REDIS_OK); - return; - } else if (!sentinel.tilt && ri->flags & SRI_PROMOTED) { - /* If this is a promoted slave we can change state to the - * failover state machine. */ - if ((ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && - (ri->master->flags & SRI_I_AM_THE_LEADER) && - (ri->master->failover_state == - SENTINEL_FAILOVER_STATE_WAIT_PROMOTION)) - { - ri->master->failover_state = SENTINEL_FAILOVER_STATE_RECONF_SLAVES; - ri->master->failover_state_change_time = mstime(); - sentinelEvent(REDIS_WARNING,"+promoted-slave",ri,"%@"); - sentinelEvent(REDIS_WARNING,"+failover-state-reconf-slaves", - ri->master,"%@"); - sentinelCallClientReconfScript(ri->master,SENTINEL_LEADER, - "start",ri->master->addr,ri->addr); } - } else if (!sentinel.tilt && ( - !(ri->master->flags & SRI_FAILOVER_IN_PROGRESS) || - ((ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && - (ri->master->flags & SRI_I_AM_THE_LEADER) && - ri->master->failover_state == - SENTINEL_FAILOVER_STATE_WAIT_START))) - { - /* No failover in progress? Then it is the start of a failover - * and we are an observer. - * - * We also do that if we are a leader doing a failover, in wait - * start, but well, somebody else started before us. */ - - if (ri->master->flags & SRI_FAILOVER_IN_PROGRESS) { - sentinelEvent(REDIS_WARNING,"-failover-abort-race", - ri->master, "%@"); - sentinelAbortFailover(ri->master); - } - - ri->master->flags |= SRI_FAILOVER_IN_PROGRESS; - sentinelEvent(REDIS_WARNING,"+failover-detected",ri->master,"%@"); - ri->master->failover_state = SENTINEL_FAILOVER_STATE_DETECT_END; - ri->master->failover_state_change_time = mstime(); - ri->master->promoted_slave = ri; - ri->flags |= SRI_PROMOTED; - ri->flags &= ~SRI_DEMOTE; - sentinelCallClientReconfScript(ri->master,SENTINEL_OBSERVER, - "start", ri->master->addr,ri->addr); - /* We are an observer, so we can only assume that the leader - * is reconfiguring the slave instances. For this reason we - * set all the instances as RECONF_SENT waiting for progresses - * on this side. */ - sentinelAddFlagsToDictOfRedisInstances(ri->master->slaves, - SRI_RECONF_SENT); } } @@ -1643,13 +1576,6 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->failover_state_change_time = mstime(); } } - - /* Detect if the old master was demoted as slave and generate the - * +slave event. */ - if (role == SRI_SLAVE && ri->flags & SRI_DEMOTE) { - sentinelEvent(REDIS_NOTICE,"+slave",ri,"%@"); - ri->flags &= ~SRI_DEMOTE; - } } void sentinelInfoReplyCallback(redisAsyncContext *c, void *reply, void *privdata) { @@ -1958,7 +1884,6 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { if (ri->flags & SRI_RECONF_SENT) flags = sdscat(flags,"reconf_sent,"); if (ri->flags & SRI_RECONF_INPROG) flags = sdscat(flags,"reconf_inprog,"); if (ri->flags & SRI_RECONF_DONE) flags = sdscat(flags,"reconf_done,"); - if (ri->flags & SRI_DEMOTE) flags = sdscat(flags,"demote,"); if (sdslen(flags) != 0) sdsrange(flags,0,-2); /* remove last "," */ addReplyBulkCString(c,flags); @@ -2750,7 +2675,6 @@ sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master) { mstime_t info_validity_time = mstime()-SENTINEL_INFO_VALIDITY_TIME; if (slave->flags & (SRI_S_DOWN|SRI_O_DOWN|SRI_DISCONNECTED)) continue; - if (slave->flags & SRI_DEMOTE) continue; /* Old master not yet ready. */ if (slave->last_avail_time < info_validity_time) continue; if (slave->slave_priority == 0) continue; @@ -2994,16 +2918,14 @@ void sentinelFailoverSwitchToPromotedSlave(sentinelRedisInstance *master) { old_master_ip = sdsdup(master->addr->ip); old_master_port = master->addr->port; sentinelResetMasterAndChangeAddress(master,ref->addr->ip,ref->addr->port); - /* If this is a real switch, that is, we have master->promoted_slave not - * NULL, then we want to add the old master as a slave of the new master, - * but flagging it with SRI_DEMOTE so that we know we'll need to send - * SLAVEOF once the old master is reachable again. */ + /* If this is a real switch and not just a user requested reset, we want + * to add all the known instances as slaves, and also all the sentinels + * back to this master. */ if (master != ref) { - /* Add the new slave, but don't generate a Sentinel event as it will - * happen later when finally the instance will claim to be a slave - * in the INFO output. */ - createSentinelRedisInstance(NULL,SRI_SLAVE|SRI_DEMOTE, + /* TODO: + createSentinelRedisInstance(NULL,SRI_SLAVE old_master_ip, old_master_port, master->quorum, master); + */ } sdsfree(old_master_ip); } @@ -3036,15 +2958,9 @@ void sentinelFailoverStateMachine(sentinelRedisInstance *ri) { } /* Abort a failover in progress with the following steps: - * 1) If this instance is the leaer send a SLAVEOF command to all the already - * reconfigured slaves if any to configure them to replicate with the - * original master. - * 2) For both leaders and observers: clear the failover flags and state in - * the master instance. - * 3) If there is already a promoted slave and we are the leader, and this - * slave is not DISCONNECTED, try to reconfigure it to replicate - * back to the master as well, sending a best effort SLAVEOF command. - */ + * 1) Set the master back to the original one, increment the config epoch. + * 2) Reconfig slaves to replicate to the old master. + * 3) Reconfig the promoted slave as a slave as well. */ void sentinelAbortFailover(sentinelRedisInstance *ri) { dictIterator *di; dictEntry *de; @@ -3087,26 +3003,6 @@ void sentinelAbortFailover(sentinelRedisInstance *ri) { } } -/* The following is called only for master instances and will abort the - * failover process if: - * - * 1) The failover is in progress. - * 2) We already promoted a slave. - * 3) The promoted slave is in extended SDOWN condition. - */ -void sentinelAbortFailoverIfNeeded(sentinelRedisInstance *ri) { - /* Failover is in progress? Do we have a promoted slave? */ - if (!(ri->flags & SRI_FAILOVER_IN_PROGRESS) || !ri->promoted_slave) return; - - /* Is the promoted slave into an extended SDOWN state? */ - if (!(ri->promoted_slave->flags & SRI_S_DOWN) || - (mstime() - ri->promoted_slave->s_down_since_time) < - (ri->down_after_period * SENTINEL_EXTENDED_SDOWN_MULTIPLIER)) return; - - sentinelEvent(REDIS_WARNING,"-failover-abort-x-sdown",ri->promoted_slave,"%@"); - sentinelAbortFailover(ri); -} - /* ======================== SENTINEL timer handler ========================== * This is the "main" our Sentinel, being sentinel completely non blocking * in design. The function is called every second. @@ -3152,7 +3048,6 @@ void sentinelHandleRedisInstance(sentinelRedisInstance *ri) { sentinelCheckObjectivelyDown(ri); sentinelStartFailoverIfNeeded(ri); sentinelFailoverStateMachine(ri); - sentinelAbortFailoverIfNeeded(ri); } } From b72985d7f7736b79dcaf34a262bd30cc802e9cb4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Nov 2013 17:10:28 +0100 Subject: [PATCH 0863/1752] Sentinel: handle Hello messages received via slaves correctly. Even when messages are received via the slave, we should perform operations (like adding a new Sentinel) in the context of the master. --- src/sentinel.c | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 5d2feb9e938..ac23c817093 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1653,12 +1653,14 @@ void sentinelPublishReplyCallback(redisAsyncContext *c, void *reply, void *privd /* This is our Pub/Sub callback for the Hello channel. It's useful in order * to discover other sentinels attached at the same master. */ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privdata) { - sentinelRedisInstance *ri = c->data; + sentinelRedisInstance *ri = c->data, *master; redisReply *r; if (!reply || !ri) return; r = reply; + master = (ri->flags & SRI_MASTER) ? ri : ri->master; + /* Update the last activity in the pubsub channel. Note that since we * receive our messages as well this timestamp can be used to detect * if the link is probably disconnected even if it seems otherwise. */ @@ -1693,26 +1695,26 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd master_port = atoi(token[7]); canfailover = atoi(token[3]); si = getSentinelRedisInstanceByAddrAndRunID( - ri->sentinels,token[0],port,token[2]); + master->sentinels,token[0],port,token[2]); current_epoch = strtoull(token[4],NULL,10); master_config_epoch = strtoull(token[8],NULL,10); - sentinelRedisInstance *master; + sentinelRedisInstance *msgmaster; if (!si) { /* If not, remove all the sentinels that have the same runid * OR the same ip/port, because it's either a restart or a * network topology change. */ - removed = removeMatchingSentinelsFromMaster(ri,token[0],port, + removed = removeMatchingSentinelsFromMaster(master,token[0],port, token[2]); if (removed) { - sentinelEvent(REDIS_NOTICE,"-dup-sentinel",ri, + sentinelEvent(REDIS_NOTICE,"-dup-sentinel",master, "%@ #duplicate of %s:%d or %s", token[0],port,token[2]); } /* Add the new sentinel. */ si = createSentinelRedisInstance(NULL,SRI_SENTINEL, - token[0],port,ri->quorum,ri); + token[0],port,master->quorum,master); if (si) { sentinelEvent(REDIS_NOTICE,"+sentinel",si,"%@"); /* The runid is NULL after a new instance creation and @@ -1727,17 +1729,18 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd sentinel.current_epoch = current_epoch; /* Update master info if received configuration is newer. */ - if ((master = sentinelGetMasterByName(token[5])) != NULL) { - if (master->config_epoch < master_config_epoch) { - master->config_epoch = master_config_epoch; - if (master_port != master->addr->port || - !strcmp(master->addr->ip, token[6])) + if ((msgmaster = sentinelGetMasterByName(token[5])) != NULL) { + if (msgmaster->config_epoch < master_config_epoch) { + msgmaster->config_epoch = master_config_epoch; + if (master_port != msgmaster->addr->port || + !strcmp(msgmaster->addr->ip, token[6])) { sentinelEvent(REDIS_WARNING,"+switch-master", - master,"%s %s %d %s %d", - master->name, master->addr->ip, master->addr->port, + msgmaster,"%s %s %d %s %d", + msgmaster->name, + msgmaster->addr->ip, msgmaster->addr->port, token[6], master_port); - sentinelResetMasterAndChangeAddress(ri, + sentinelResetMasterAndChangeAddress(msgmaster, token[6], master_port); } } From 663d79c0d5ed9c0f7cd2b28c5eaf593ee87b2243 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 11 Nov 2013 18:30:11 +0100 Subject: [PATCH 0864/1752] Sentinel: leadership handling changes WIP. Changes to leadership handling. Now the leader gets selected by every Sentinel, for a specified epoch, when the SENTINEL is-master-down-by-addr is sent. This command now includes the runid and the currentEpoch of the instance seeking for a vote. The Sentinel only votes a single time in a given epoch. Still a work in progress, does not even compile at this stage. --- src/sentinel.c | 137 +++++++++++++++++++++++++++++-------------------- 1 file changed, 80 insertions(+), 57 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index ac23c817093..0c242ec8387 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -175,9 +175,8 @@ typedef struct sentinelRedisInstance { char *leader; /* If this is a master instance, this is the runid of the Sentinel that should perform the failover. If this is a Sentinel, this is the runid of the Sentinel - that this other Sentinel is voting as leader. - This field is valid only if SRI_MASTER_DOWN is - set on the Sentinel instance. */ + that this Sentinel voted as leader. */ + uint64_t leader_epoch; /* Epoch of the 'leader' field. */ int failover_state; /* See SENTINEL_FAILOVER_STATE_* defines. */ mstime_t failover_state_change_time; mstime_t failover_start_time; /* When to start to failover if leader. */ @@ -329,6 +328,7 @@ void sentinelScheduleScriptExecution(char *path, ...); void sentinelStartFailover(sentinelRedisInstance *master, int state); void sentinelDiscardReplyCallback(redisAsyncContext *c, void *reply, void *privdata); int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port); +char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch); /* ========================= Dictionary types =============================== */ @@ -896,6 +896,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * /* Failover state. */ ri->leader = NULL; + ri->leader_epoch = 0; ri->failover_state = SENTINEL_FAILOVER_STATE_NONE; ri->failover_state_change_time = 0; ri->failover_start_time = 0; @@ -1033,7 +1034,7 @@ sentinelRedisInstance *getSentinelRedisInstanceByAddrAndRunID(dict *instances, c return instance; } -/* Simple master lookup by name */ +/* Master lookup by name */ sentinelRedisInstance *sentinelGetMasterByName(char *name) { sentinelRedisInstance *ri; sds sdsname = sdsnew(name); @@ -1043,6 +1044,24 @@ sentinelRedisInstance *sentinelGetMasterByName(char *name) { return ri; } +/* Senitnel lookup by runid */ +sentinelRedisInstance *sentinelGetSentinelByRunid(sentinelRedisInstance *master, char *runid) { + sentinelRedisInstance *retval = NULL; + dictIterator *di; + dictEntry *de; + + di = dictGetIterator(master->sentinels); + while((de = dictNext(di)) != NULL) { + sentinelRedisInstance *ri = dictGetVal(de); + if (!strcmp(ri->runid,runid)) { + retval = ri; + break; + } + } + dictReleaseIterator(di); + return retval; +} + /* Add the specified flags to all the instances in the specified dictionary. */ void sentinelAddFlagsToDictOfRedisInstances(dict *instances, int flags) { dictIterator *di; @@ -1981,11 +2000,13 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { addReplyBulkLongLong(c,(ri->flags & SRI_CAN_FAILOVER) != 0); fields++; - if (ri->flags & SRI_MASTER_DOWN) { - addReplyBulkCString(c,"subjective-leader"); - addReplyBulkCString(c,ri->leader ? ri->leader : "?"); - fields++; - } + addReplyBulkCString(c,"voted-leader"); + addReplyBulkCString(c,ri->leader ? ri->leader : "?"); + fields++; + + addReplyBulkCString(c,"voted-leader-epoch"); + addReplyBulkLongLong(c,ri->leader_epoch); + fields++; } setDeferredMultiBulkLength(c,mbl,fields*2); @@ -2046,14 +2067,18 @@ void sentinelCommand(redisClient *c) { return; addReplyDictOfRedisInstances(c,ri->sentinels); } else if (!strcasecmp(c->argv[1]->ptr,"is-master-down-by-addr")) { - /* SENTINEL IS-MASTER-DOWN-BY-ADDR */ + /* SENTINEL IS-MASTER-DOWN-BY-ADDR */ sentinelRedisInstance *ri; + long long req_epoch; + uint64_t leader_epoch = 0; char *leader = NULL; long port; int isdown = 0; - if (c->argc != 4) goto numargserr; - if (getLongFromObjectOrReply(c,c->argv[3],&port,NULL) != REDIS_OK) + if (c->argc != 6) goto numargserr; + if (getLongFromObjectOrReply(c,c->argv[3],&port,NULL) != REDIS_OK || + getLongLongFromObjectOrReply(c,c->argv[4],&req_epoch,NULL) + != REDIS_OK) return; ri = getSentinelRedisInstanceByAddrAndRunID(sentinel.masters, c->argv[2]->ptr,port,NULL); @@ -2063,12 +2088,20 @@ void sentinelCommand(redisClient *c) { if (!sentinel.tilt && ri && (ri->flags & SRI_S_DOWN) && (ri->flags & SRI_MASTER)) isdown = 1; - if (ri) leader = sentinelGetSubjectiveLeader(ri); - /* Reply with a two-elements multi-bulk reply: down state, leader. */ - addReplyMultiBulkLen(c,2); + /* Vote for the master (or fetch the previous vote) */ + if (ri && ri->flags & SRI_MASTER) { + leader = sentinelVoteLeader(ri,(uint64_t)req_epoch, + c->argv[5]->ptr, + &leader_epoch); + } + + /* Reply with a three-elements multi-bulk reply: + * down state, leader, vote epoch. */ + addReplyMultiBulkLen(c,3); addReply(c, isdown ? shared.cone : shared.czero); addReplyBulkCString(c, leader ? leader : "?"); + addReplyLongLong(c, (long long)leader_epoch); if (leader) sdsfree(leader); } else if (!strcasecmp(c->argv[1]->ptr,"reset")) { /* SENTINEL RESET */ @@ -2291,9 +2324,10 @@ void sentinelReceiveIsMasterDownReply(redisAsyncContext *c, void *reply, void *p /* Ignore every error or unexpected reply. * Note that if the command returns an error for any reason we'll * end clearing the SRI_MASTER_DOWN flag for timeout anyway. */ - if (r->type == REDIS_REPLY_ARRAY && r->elements == 2 && + if (r->type == REDIS_REPLY_ARRAY && r->elements == 3 && r->element[0]->type == REDIS_REPLY_INTEGER && - r->element[1]->type == REDIS_REPLY_STRING) + r->element[1]->type == REDIS_REPLY_STRING && + r->element[2]->type == REDIS_REPLY_INTEGER) { ri->last_master_down_reply_time = mstime(); if (r->element[0]->integer == 1) { @@ -2303,6 +2337,7 @@ void sentinelReceiveIsMasterDownReply(redisAsyncContext *c, void *reply, void *p } sdsfree(ri->leader); ri->leader = sdsnew(r->element[1]->str); + ri->leader_epoch = r->element[2]->integer; } } @@ -2343,8 +2378,8 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master) { ll2string(port,sizeof(port),master->addr->port); retval = redisAsyncCommand(ri->cc, sentinelReceiveIsMasterDownReply, NULL, - "SENTINEL is-master-down-by-addr %s %s", - master->addr->ip, port); + "SENTINEL is-master-down-by-addr %s %s %llu %s", + master->addr->ip, port, sentinel.current_epoch, server.runid); if (retval == REDIS_OK) ri->pending_commands++; } dictReleaseIterator(di); @@ -2371,41 +2406,25 @@ int compareRunID(const void *a, const void *b) { return strcasecmp(*aptrptr, *bptrptr); } -char *sentinelGetSubjectiveLeader(sentinelRedisInstance *master) { - dictIterator *di; - dictEntry *de; - char **instance = - zmalloc(sizeof(char*)*(dictSize(master->sentinels)+1)); - int instances = 0; - char *leader = NULL; - - if (master->flags & SRI_CAN_FAILOVER) { - /* Add myself if I'm a Sentinel that can failover this master. */ - instance[instances++] = server.runid; - } +/* Vote for the sentinel with 'req_runid' or return the old vote if already + * voted for the specifed 'req_epoch' or one greater. + * + * If a vote is not available returns NULL, otherwise return the Sentinel + * runid and populate the leader_epoch with the epoch of the last vote. */ +char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch) { + sentinelRedisInstance *si = sentinelGetSentinelByRunid(master,req_runid); - di = dictGetIterator(master->sentinels); - while((de = dictNext(di)) != NULL) { - sentinelRedisInstance *ri = dictGetVal(de); - mstime_t lag = mstime() - ri->last_avail_time; + if (req_epoch > sentinel.current_epoch) + sentinel.current_epoch = req_epoch; - if (lag > SENTINEL_INFO_VALIDITY_TIME || - !(ri->flags & SRI_CAN_FAILOVER) || - (ri->flags & SRI_DISCONNECTED) || - ri->runid == NULL) - continue; - instance[instances++] = ri->runid; + if (si && master->leader_epoch < req_epoch) { + sdsfree(master->leader); + master->leader = sdsnew(req_runid); + master->leader_epoch = sentinel.current_epoch; } - dictReleaseIterator(di); - /* If we have at least one instance passing our checks, order the array - * by runid. */ - if (instances) { - qsort(instance,instances,sizeof(char*),compareRunID); - leader = sdsnew(instance[0]); - } - zfree(instance); - return leader; + *leader_epoch = master->leader_epoch; + return master->leader; } struct sentinelLeader { @@ -2413,9 +2432,9 @@ struct sentinelLeader { unsigned long votes; }; -/* Helper function for sentinelGetObjectiveLeader, increment the counter +/* Helper function for sentinelGetLeader, increment the counter * relative to the specified runid. */ -void sentinelObjectiveLeaderIncr(dict *counters, char *runid) { +void sentinelLeaderIncr(dict *counters, char *runid) { dictEntry *de = dictFind(counters,runid); uint64_t oldval; @@ -2429,9 +2448,13 @@ void sentinelObjectiveLeaderIncr(dict *counters, char *runid) { } } -/* Scan all the Sentinels attached to this master to check what is the - * most voted leader among Sentinels. */ -char *sentinelGetObjectiveLeader(sentinelRedisInstance *master) { +/* Scan all the Sentinels attached to this master to check if there + * is a leader for a given term, and return it if any. + * + * To be a leader for a given epoch, we should have the majorify of + * the Sentinels we know about that reported the same instance as + * leader for the same epoch. */ +char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t term) { dict *counters; dictIterator *di; dictEntry *de; @@ -2445,7 +2468,7 @@ char *sentinelGetObjectiveLeader(sentinelRedisInstance *master) { /* Count my vote. */ myvote = sentinelGetSubjectiveLeader(master); if (myvote) { - sentinelObjectiveLeaderIncr(counters,myvote); + sentinelLeaderIncr(counters,myvote); voters++; } @@ -2460,7 +2483,7 @@ char *sentinelGetObjectiveLeader(sentinelRedisInstance *master) { * leader fails. In that case we consider all the voters. */ if (!(master->flags & SRI_FAILOVER_IN_PROGRESS) && !(ri->flags & SRI_MASTER_DOWN)) continue; - sentinelObjectiveLeaderIncr(counters,ri->leader); + sentinelLeaderIncr(counters,ri->leader); voters++; } dictReleaseIterator(di); From b95c6ed7b777aae98c6cb78d7b96fc9df860e3d0 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 11:09:35 +0100 Subject: [PATCH 0865/1752] Sentinel: epoch introduced in leader vote. --- src/sentinel.c | 213 ++++++++++++++++--------------------------------- 1 file changed, 67 insertions(+), 146 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 0c242ec8387..d80d154619a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -67,13 +67,12 @@ typedef struct sentinelAddr { #define SRI_CAN_FAILOVER (1<<7) #define SRI_FAILOVER_IN_PROGRESS (1<<8) /* Failover is in progress for this master. */ -#define SRI_I_AM_THE_LEADER (1<<9) /* We are the leader for this master. */ -#define SRI_PROMOTED (1<<10) /* Slave selected for promotion. */ -#define SRI_RECONF_SENT (1<<11) /* SLAVEOF sent. */ -#define SRI_RECONF_INPROG (1<<12) /* Slave synchronization in progress. */ -#define SRI_RECONF_DONE (1<<13) /* Slave synchronized with new master. */ -#define SRI_FORCE_FAILOVER (1<<14) /* Force failover with master up. */ -#define SRI_SCRIPT_KILL_SENT (1<<15) /* SCRIPT KILL already sent on -BUSY */ +#define SRI_PROMOTED (1<<9) /* Slave selected for promotion. */ +#define SRI_RECONF_SENT (1<<10) /* SLAVEOF sent. */ +#define SRI_RECONF_INPROG (1<<11) /* Slave synchronization in progress. */ +#define SRI_RECONF_DONE (1<<12) /* Slave synchronized with new master. */ +#define SRI_FORCE_FAILOVER (1<<13) /* Force failover with master up. */ +#define SRI_SCRIPT_KILL_SENT (1<<14) /* SCRIPT KILL already sent on -BUSY */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 @@ -177,6 +176,7 @@ typedef struct sentinelRedisInstance { this is a Sentinel, this is the runid of the Sentinel that this Sentinel voted as leader. */ uint64_t leader_epoch; /* Epoch of the 'leader' field. */ + uint64_t failover_epoch; /* Epoch of the currently started failover. */ int failover_state; /* See SENTINEL_FAILOVER_STATE_* defines. */ mstime_t failover_state_change_time; mstime_t failover_start_time; /* When to start to failover if leader. */ @@ -325,7 +325,7 @@ void sentinelAbortFailover(sentinelRedisInstance *ri); void sentinelEvent(int level, char *type, sentinelRedisInstance *ri, const char *fmt, ...); sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master); void sentinelScheduleScriptExecution(char *path, ...); -void sentinelStartFailover(sentinelRedisInstance *master, int state); +void sentinelStartFailover(sentinelRedisInstance *master); void sentinelDiscardReplyCallback(redisAsyncContext *c, void *reply, void *privdata); int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port); char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch); @@ -897,6 +897,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * /* Failover state. */ ri->leader = NULL; ri->leader_epoch = 0; + ri->failover_epoch = 0; ri->failover_state = SENTINEL_FAILOVER_STATE_NONE; ri->failover_state_change_time = 0; ri->failover_start_time = 0; @@ -1524,7 +1525,6 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * failover state machine. */ if (!sentinel.tilt && (ri->master->flags & SRI_FAILOVER_IN_PROGRESS) && - (ri->master->flags & SRI_I_AM_THE_LEADER) && (ri->master->failover_state == SENTINEL_FAILOVER_STATE_WAIT_PROMOTION)) { @@ -1900,8 +1900,6 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { if (ri->flags & SRI_MASTER_DOWN) flags = sdscat(flags,"master_down,"); if (ri->flags & SRI_FAILOVER_IN_PROGRESS) flags = sdscat(flags,"failover_in_progress,"); - if (ri->flags & SRI_I_AM_THE_LEADER) - flags = sdscat(flags,"i_am_the_leader,"); if (ri->flags & SRI_PROMOTED) flags = sdscat(flags,"promoted,"); if (ri->flags & SRI_RECONF_SENT) flags = sdscat(flags,"reconf_sent,"); if (ri->flags & SRI_RECONF_INPROG) flags = sdscat(flags,"reconf_inprog,"); @@ -2149,7 +2147,7 @@ void sentinelCommand(redisClient *c) { addReplySds(c,sdsnew("-NOGOODSLAVE No suitable slave to promote\r\n")); return; } - sentinelStartFailover(ri,SENTINEL_FAILOVER_STATE_WAIT_START); + sentinelStartFailover(ri); ri->flags |= SRI_FORCE_FAILOVER; addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"pending-scripts")) { @@ -2349,6 +2347,14 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master) { dictIterator *di; dictEntry *de; + /* Vote for myself if I see the master is already in ODOWN state. */ + if (master->flags & SRI_O_DOWN) { + uint64_t leader_epoch; + + sentinelVoteLeader(master,sentinel.current_epoch,server.runid, + &leader_epoch); + } + di = dictGetIterator(master->sentinels); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *ri = dictGetVal(de); @@ -2368,8 +2374,7 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master) { * 1) We believe it is down, or there is a failover in progress. * 2) Sentinel is connected. * 3) We did not received the info within SENTINEL_ASK_PERIOD ms. */ - if ((master->flags & (SRI_S_DOWN|SRI_FAILOVER_IN_PROGRESS)) == 0) - continue; + if ((master->flags & SRI_S_DOWN) == 0) continue; if (ri->flags & SRI_DISCONNECTED) continue; if (mstime() - ri->last_master_down_reply_time < SENTINEL_ASK_PERIOD) continue; @@ -2379,7 +2384,9 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master) { retval = redisAsyncCommand(ri->cc, sentinelReceiveIsMasterDownReply, NULL, "SENTINEL is-master-down-by-addr %s %s %llu %s", - master->addr->ip, port, sentinel.current_epoch, server.runid); + master->addr->ip, port, + sentinel.current_epoch, + server.runid); if (retval == REDIS_OK) ri->pending_commands++; } dictReleaseIterator(di); @@ -2417,7 +2424,9 @@ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char if (req_epoch > sentinel.current_epoch) sentinel.current_epoch = req_epoch; - if (si && master->leader_epoch < req_epoch) { + if (si && master->leader_epoch < req_epoch && + sentinel.current_epoch <= req_epoch) + { sdsfree(master->leader); master->leader = sdsnew(req_runid); master->leader_epoch = sentinel.current_epoch; @@ -2449,25 +2458,27 @@ void sentinelLeaderIncr(dict *counters, char *runid) { } /* Scan all the Sentinels attached to this master to check if there - * is a leader for a given term, and return it if any. + * is a leader for the specified epoch. * * To be a leader for a given epoch, we should have the majorify of - * the Sentinels we know about that reported the same instance as + * the Sentinels we know that reported the same instance as * leader for the same epoch. */ -char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t term) { +char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t epoch) { dict *counters; dictIterator *di; dictEntry *de; unsigned int voters = 0, voters_quorum; char *myvote; char *winner = NULL; + uint64_t leader_epoch; redisAssert(master->flags & (SRI_O_DOWN|SRI_FAILOVER_IN_PROGRESS)); counters = dictCreate(&leaderVotesDictType,NULL); - /* Count my vote. */ - myvote = sentinelGetSubjectiveLeader(master); - if (myvote) { + /* Count my vote (and vote for myself if I still did not voted for + * the currnet epoch). */ + myvote = sentinelVoteLeader(master,epoch,server.runid,&leader_epoch); + if (myvote && leader_epoch == epoch) { sentinelLeaderIncr(counters,myvote); voters++; } @@ -2476,13 +2487,8 @@ char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t term) { di = dictGetIterator(master->sentinels); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *ri = dictGetVal(de); - if (ri->leader == NULL) continue; - /* If the failover is not already in progress we are only interested - * in Sentinels that believe the master is down. Otherwise the leader - * selection is useful for the "failover-takedown" when the original - * leader fails. In that case we consider all the voters. */ - if (!(master->flags & SRI_FAILOVER_IN_PROGRESS) && - !(ri->flags & SRI_MASTER_DOWN)) continue; + if (ri->leader == NULL || ri->leader_epoch != sentinel.current_epoch) + continue; sentinelLeaderIncr(counters,ri->leader); voters++; } @@ -2548,32 +2554,14 @@ int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port) { return REDIS_OK; } -/* Setup the master state to start a failover as a leader. - * - * State can be either: - * - * SENTINEL_FAILOVER_STATE_WAIT_START: starts a failover from scratch. - * SENTINEL_FAILOVER_STATE_RECONF_SLAVES: takedown a failed failover. - */ -void sentinelStartFailover(sentinelRedisInstance *master, int state) { +/* Setup the master state to start a failover. */ +void sentinelStartFailover(sentinelRedisInstance *master) { redisAssert(master->flags & SRI_MASTER); - redisAssert(state == SENTINEL_FAILOVER_STATE_WAIT_START || - state == SENTINEL_FAILOVER_STATE_RECONF_SLAVES); - master->failover_state = state; - master->flags |= SRI_FAILOVER_IN_PROGRESS|SRI_I_AM_THE_LEADER; + master->failover_state = SENTINEL_FAILOVER_STATE_WAIT_START; + master->flags |= SRI_FAILOVER_IN_PROGRESS; + master->failover_epoch = ++sentinel.current_epoch; sentinelEvent(REDIS_WARNING,"+failover-triggered",master,"%@"); - - /* Pick a random delay if it's a fresh failover (WAIT_START), and not - * a recovery of a failover started by another sentinel. */ - if (master->failover_state == SENTINEL_FAILOVER_STATE_WAIT_START) { - master->failover_start_time = mstime() + - SENTINEL_FAILOVER_FIXED_DELAY + - (rand() % SENTINEL_FAILOVER_MAX_RANDOM_DELAY); - sentinelEvent(REDIS_WARNING,"+failover-state-wait-start",master, - "%@ #starting in %lld milliseconds", - master->failover_start_time-mstime()); - } master->failover_state_change_time = mstime(); } @@ -2582,66 +2570,18 @@ void sentinelStartFailover(sentinelRedisInstance *master, int state) { * * 1) Enough time has passed since O_DOWN. * 2) The master is marked as SRI_CAN_FAILOVER, so we can failover it. - * 3) We are the objectively leader for this master. - * - * If the conditions are met we flag the master as SRI_FAILOVER_IN_PROGRESS - * and SRI_I_AM_THE_LEADER. - */ + * + * We still don't know if we'll win the election so it is possible that we + * start the failover but that we'll not be able to act. */ void sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { - char *leader; - int isleader; - - /* We can't failover if the master is not in O_DOWN state or if - * there is not already a failover in progress (to perform the - * takedown if the leader died) or if this Sentinel is not allowed - * to start a failover. */ + /* We can't failover if the master is not in O_DOWN state. */ if (!(master->flags & SRI_CAN_FAILOVER) || - !(master->flags & (SRI_O_DOWN|SRI_FAILOVER_IN_PROGRESS))) return; + !(master->flags & SRI_O_DOWN)) return; - leader = sentinelGetObjectiveLeader(master); - isleader = leader && strcasecmp(leader,server.runid) == 0; - sdsfree(leader); - - /* If I'm not the leader, I can't failover for sure. */ - if (!isleader) return; + /* Failover already in progress? */ + if (master->flags & SRI_FAILOVER_IN_PROGRESS) return; - /* If the failover is already in progress there are two options... */ - if (master->flags & SRI_FAILOVER_IN_PROGRESS) { - if (master->flags & SRI_I_AM_THE_LEADER) { - /* 1) I'm flagged as leader so I already started the failover. - * Just return. */ - return; - } else { - mstime_t elapsed = mstime() - master->failover_state_change_time; - - /* 2) I'm the new leader, but I'm not flagged as leader in the - * master: I did not started the failover, but the original - * leader has no longer the leadership. - * - * In this case if the failover appears to be lagging - * for at least 25% of the configured failover timeout, - * I can assume I can take control. Otherwise - * it's better to return and wait more. */ - if (elapsed < (master->failover_timeout/4)) return; - sentinelEvent(REDIS_WARNING,"+failover-takedown",master,"%@"); - /* We have already an elected slave if we are in - * FAILOVER_IN_PROGRESS state, that is, the slave that we - * observed turning into a master. */ - sentinelStartFailover(master,SENTINEL_FAILOVER_STATE_RECONF_SLAVES); - /* As an observer we flagged all the slaves as RECONF_SENT but - * now we are in charge of actually sending the reconfiguration - * command so let's clear this flag for all the instances. */ - sentinelDelFlagsToDictOfRedisInstances(master->slaves, - SRI_RECONF_SENT); - } - } else { - /* Brand new failover as SRI_FAILOVER_IN_PROGRESS was not set. - * - * Do we have a slave to promote? Otherwise don't start a failover - * at all. */ - if (sentinelSelectSlave(master) == NULL) return; - sentinelStartFailover(master,SENTINEL_FAILOVER_STATE_WAIT_START); - } + sentinelStartFailover(master); } /* Select a suitable slave to promote. The current algorithm only uses @@ -2725,29 +2665,22 @@ sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master) { /* ---------------- Failover state machine implementation ------------------- */ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { - /* If we in "wait start" but the master is no longer in ODOWN nor in - * SDOWN condition we abort the failover. This is important as it - * prevents a useless failover in a a notable case of netsplit, where - * the sentinels are split from the redis instances. In this case - * the failover will not start while there is the split because no - * good slave can be reached. However when the split is resolved, we - * can go to waitstart if the slave is back reachable a few milliseconds - * before the master is. In that case when the master is back online - * we cancel the failover. */ - if ((ri->flags & (SRI_S_DOWN|SRI_O_DOWN|SRI_FORCE_FAILOVER)) == 0) { - sentinelEvent(REDIS_WARNING,"-failover-abort-master-is-back", - ri,"%@"); - sentinelAbortFailover(ri); - return; - } + char *leader; + int isleader; + + /* Check if we are the leader for the failover epoch. */ + leader = sentinelGetLeader(ri, ri->failover_epoch); + isleader = leader && strcasecmp(leader,server.runid) == 0; + sdsfree(leader); + + /* If I'm not the leader, I can't continue with the failover. */ + if (!isleader) return; /* Start the failover going to the next state if enough time has * elapsed. */ - if (mstime() >= ri->failover_start_time) { - ri->failover_state = SENTINEL_FAILOVER_STATE_SELECT_SLAVE; - ri->failover_state_change_time = mstime(); - sentinelEvent(REDIS_WARNING,"+failover-state-select-slave",ri,"%@"); - } + ri->failover_state = SENTINEL_FAILOVER_STATE_SELECT_SLAVE; + ri->failover_state_change_time = mstime(); + sentinelEvent(REDIS_WARNING,"+failover-state-select-slave",ri,"%@"); } void sentinelFailoverSelectSlave(sentinelRedisInstance *ri) { @@ -2831,8 +2764,7 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) { } if (not_reconfigured == 0) { - int role = (master->flags & SRI_I_AM_THE_LEADER) ? SENTINEL_LEADER : - SENTINEL_OBSERVER; + int role = SENTINEL_LEADER; sentinelEvent(REDIS_WARNING,"+failover-end",master,"%@"); master->failover_state = SENTINEL_FAILOVER_STATE_UPDATE_CONFIG; @@ -2844,7 +2776,7 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) { /* If I'm the leader it is a good idea to send a best effort SLAVEOF * command to all the slaves still not reconfigured to replicate with * the new master. */ - if (timeout && (master->flags & SRI_I_AM_THE_LEADER)) { + if (timeout) { dictIterator *di; dictEntry *de; @@ -3001,8 +2933,7 @@ void sentinelAbortFailover(sentinelRedisInstance *ri) { di = dictGetIterator(ri->slaves); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *slave = dictGetVal(de); - if ((ri->flags & SRI_I_AM_THE_LEADER) && - !(slave->flags & SRI_DISCONNECTED) && + if (!(slave->flags & SRI_DISCONNECTED) && (slave->flags & (SRI_PROMOTED|SRI_RECONF_SENT|SRI_RECONF_INPROG| SRI_RECONF_DONE))) { @@ -3016,9 +2947,8 @@ void sentinelAbortFailover(sentinelRedisInstance *ri) { } dictReleaseIterator(di); - sentinel_role = (ri->flags & SRI_I_AM_THE_LEADER) ? SENTINEL_LEADER : - SENTINEL_OBSERVER; - ri->flags &= ~(SRI_FAILOVER_IN_PROGRESS|SRI_I_AM_THE_LEADER|SRI_FORCE_FAILOVER); + sentinel_role = SENTINEL_LEADER; + ri->flags &= ~(SRI_FAILOVER_IN_PROGRESS|SRI_FORCE_FAILOVER); ri->failover_state = SENTINEL_FAILOVER_STATE_NONE; ri->failover_state_change_time = mstime(); if (ri->promoted_slave) { @@ -3041,16 +2971,6 @@ void sentinelHandleRedisInstance(sentinelRedisInstance *ri) { sentinelReconnectInstance(ri); sentinelPingInstance(ri); - /* Masters and slaves */ - if (ri->flags & (SRI_MASTER|SRI_SLAVE)) { - /* Nothing so far. */ - } - - /* Only masters */ - if (ri->flags & SRI_MASTER) { - sentinelAskMasterStateToOtherSentinels(ri); - } - /* ============== ACTING HALF ============= */ /* We don't proceed with the acting half if we are in TILT mode. * TILT happens when we find something odd with the time, like a @@ -3074,6 +2994,7 @@ void sentinelHandleRedisInstance(sentinelRedisInstance *ri) { sentinelCheckObjectivelyDown(ri); sentinelStartFailoverIfNeeded(ri); sentinelFailoverStateMachine(ri); + sentinelAskMasterStateToOtherSentinels(ri); } } From eba4775b5d5bcb66e78f93d8b592076d712ed2cd Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 11:10:10 +0100 Subject: [PATCH 0866/1752] Sentinel: fix PUBLISH to masters and slaves. --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index d80d154619a..294c9cb18ef 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1829,13 +1829,13 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { char payload[REDIS_IP_STR_LEN+1024]; sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? - NULL : ri->master; + ri : ri->master; snprintf(payload,sizeof(payload), "%s,%d,%s,%d,%llu," /* Info about this sentinel. */ "%s,%s,%d,%lld", /* Info about current master. */ ip, server.port, server.runid, - (ri->flags & SRI_CAN_FAILOVER) != 0, + (master->flags & SRI_CAN_FAILOVER) != 0, (unsigned long long) sentinel.current_epoch, /* --- */ master->name,master->addr->ip,master->addr->port, From 447c2787a058d139c4f70203b182f6b30c3a0654 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 11:32:40 +0100 Subject: [PATCH 0867/1752] Sentinel: allow to vote for myself. --- src/sentinel.c | 26 +++----------------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 294c9cb18ef..90c5ef2ed5e 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1045,24 +1045,6 @@ sentinelRedisInstance *sentinelGetMasterByName(char *name) { return ri; } -/* Senitnel lookup by runid */ -sentinelRedisInstance *sentinelGetSentinelByRunid(sentinelRedisInstance *master, char *runid) { - sentinelRedisInstance *retval = NULL; - dictIterator *di; - dictEntry *de; - - di = dictGetIterator(master->sentinels); - while((de = dictNext(di)) != NULL) { - sentinelRedisInstance *ri = dictGetVal(de); - if (!strcmp(ri->runid,runid)) { - retval = ri; - break; - } - } - dictReleaseIterator(di); - return retval; -} - /* Add the specified flags to all the instances in the specified dictionary. */ void sentinelAddFlagsToDictOfRedisInstances(dict *instances, int flags) { dictIterator *di; @@ -2419,17 +2401,15 @@ int compareRunID(const void *a, const void *b) { * If a vote is not available returns NULL, otherwise return the Sentinel * runid and populate the leader_epoch with the epoch of the last vote. */ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch) { - sentinelRedisInstance *si = sentinelGetSentinelByRunid(master,req_runid); - if (req_epoch > sentinel.current_epoch) sentinel.current_epoch = req_epoch; - if (si && master->leader_epoch < req_epoch && - sentinel.current_epoch <= req_epoch) - { + if (master->leader_epoch < req_epoch && sentinel.current_epoch <= req_epoch) { sdsfree(master->leader); master->leader = sdsnew(req_runid); master->leader_epoch = sentinel.current_epoch; + printf("Selected leader %s for epoch %llu\n", master->leader, + (unsigned long long) master->leader_epoch); } *leader_epoch = master->leader_epoch; From 48acc675dd2cdc20f7e24a8a6cf909e3467ab628 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 13:30:31 +0100 Subject: [PATCH 0868/1752] Sentinel: wait some time between failover attempts. --- src/sentinel.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 90c5ef2ed5e..1edcf763708 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -179,7 +179,7 @@ typedef struct sentinelRedisInstance { uint64_t failover_epoch; /* Epoch of the currently started failover. */ int failover_state; /* See SENTINEL_FAILOVER_STATE_* defines. */ mstime_t failover_state_change_time; - mstime_t failover_start_time; /* When to start to failover if leader. */ + mstime_t failover_start_time; /* Last failover attempt start time. */ mstime_t failover_timeout; /* Max time to refresh failover state. */ struct sentinelRedisInstance *promoted_slave; /* Promoted slave instance. */ /* Scripts executed to notify admin or reconfigure clients: when they @@ -2413,7 +2413,7 @@ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char } *leader_epoch = master->leader_epoch; - return master->leader; + return master->leader ? sdsnew(master->leader) : NULL; } struct sentinelLeader { @@ -2542,6 +2542,7 @@ void sentinelStartFailover(sentinelRedisInstance *master) { master->flags |= SRI_FAILOVER_IN_PROGRESS; master->failover_epoch = ++sentinel.current_epoch; sentinelEvent(REDIS_WARNING,"+failover-triggered",master,"%@"); + master->failover_start_time = mstime(); master->failover_state_change_time = mstime(); } @@ -2561,6 +2562,10 @@ void sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { /* Failover already in progress? */ if (master->flags & SRI_FAILOVER_IN_PROGRESS) return; + /* Last failover attempt started too little time ago? */ + if (mstime() - master->failover_start_time < + SENTINEL_PUBLISH_PERIOD*4) return; + sentinelStartFailover(master); } From 6593f332224f405a8f255452f675685d55d88282 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 13:35:25 +0100 Subject: [PATCH 0869/1752] Sentinel: +new-epoch events. --- src/sentinel.c | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 1edcf763708..5b1716092cd 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1726,8 +1726,11 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd } /* Update local current_epoch if received current_epoch is greater. */ - if (current_epoch > sentinel.current_epoch) + if (current_epoch > sentinel.current_epoch) { sentinel.current_epoch = current_epoch; + sentinelEvent(REDIS_WARNING,"+new-epoch",ri,"%llu", + (unsigned long long) sentinel.current_epoch); + } /* Update master info if received configuration is newer. */ if ((msgmaster = sentinelGetMasterByName(token[5])) != NULL) { @@ -2401,15 +2404,18 @@ int compareRunID(const void *a, const void *b) { * If a vote is not available returns NULL, otherwise return the Sentinel * runid and populate the leader_epoch with the epoch of the last vote. */ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch) { - if (req_epoch > sentinel.current_epoch) + if (req_epoch > sentinel.current_epoch) { sentinel.current_epoch = req_epoch; + sentinelEvent(REDIS_WARNING,"+new-epoch",master,"%llu", + (unsigned long long) sentinel.current_epoch); + } if (master->leader_epoch < req_epoch && sentinel.current_epoch <= req_epoch) { sdsfree(master->leader); master->leader = sdsnew(req_runid); master->leader_epoch = sentinel.current_epoch; - printf("Selected leader %s for epoch %llu\n", master->leader, - (unsigned long long) master->leader_epoch); + sentinelEvent(REDIS_WARNING,"+vote-for-leader",master,"%s %llu", + master->leader, (unsigned long long) master->leader_epoch); } *leader_epoch = master->leader_epoch; @@ -2541,7 +2547,9 @@ void sentinelStartFailover(sentinelRedisInstance *master) { master->failover_state = SENTINEL_FAILOVER_STATE_WAIT_START; master->flags |= SRI_FAILOVER_IN_PROGRESS; master->failover_epoch = ++sentinel.current_epoch; - sentinelEvent(REDIS_WARNING,"+failover-triggered",master,"%@"); + sentinelEvent(REDIS_WARNING,"+new-epoch",master,"%llu", + (unsigned long long) sentinel.current_epoch); + sentinelEvent(REDIS_WARNING,"+try-failover",master,"%@"); master->failover_start_time = mstime(); master->failover_state_change_time = mstime(); } @@ -2660,6 +2668,7 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { /* If I'm not the leader, I can't continue with the failover. */ if (!isleader) return; + sentinelEvent(REDIS_WARNING,"+elected-leader",ri,"%@"); /* Start the failover going to the next state if enough time has * elapsed. */ From 2488257ad81abd52d6a98df0072746b44e822cb1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 16:38:02 +0100 Subject: [PATCH 0870/1752] Sentinel: when starting failover seek for votes ASAP. --- src/sentinel.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 5b1716092cd..55f0adf8cc6 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -74,6 +74,7 @@ typedef struct sentinelAddr { #define SRI_FORCE_FAILOVER (1<<13) /* Force failover with master up. */ #define SRI_SCRIPT_KILL_SENT (1<<14) /* SCRIPT KILL already sent on -BUSY */ +#define SENTINEL_NO_FLAGS 0 /* Generic no flags define. */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 #define SENTINEL_ASK_PERIOD 1000 @@ -2328,7 +2329,8 @@ void sentinelReceiveIsMasterDownReply(redisAsyncContext *c, void *reply, void *p * SENTINEL IS-MASTER-DOWN-BY-ADDR requests to other sentinels * in order to get the replies that allow to reach the quorum and * possibly also mark the master as objectively down. */ -void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master) { +#define SENTINEL_ASK_FORCED (1<<0) +void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int flags) { dictIterator *di; dictEntry *de; @@ -2361,7 +2363,8 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master) { * 3) We did not received the info within SENTINEL_ASK_PERIOD ms. */ if ((master->flags & SRI_S_DOWN) == 0) continue; if (ri->flags & SRI_DISCONNECTED) continue; - if (mstime() - ri->last_master_down_reply_time < SENTINEL_ASK_PERIOD) + if (!(flags & SENTINEL_ASK_FORCED) && + mstime() - ri->last_master_down_reply_time < SENTINEL_ASK_PERIOD) continue; /* Ask */ @@ -2561,20 +2564,23 @@ void sentinelStartFailover(sentinelRedisInstance *master) { * 2) The master is marked as SRI_CAN_FAILOVER, so we can failover it. * * We still don't know if we'll win the election so it is possible that we - * start the failover but that we'll not be able to act. */ -void sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { + * start the failover but that we'll not be able to act. + * + * Return non-zero if a failover was started. */ +int sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { /* We can't failover if the master is not in O_DOWN state. */ if (!(master->flags & SRI_CAN_FAILOVER) || - !(master->flags & SRI_O_DOWN)) return; + !(master->flags & SRI_O_DOWN)) return 0; /* Failover already in progress? */ - if (master->flags & SRI_FAILOVER_IN_PROGRESS) return; + if (master->flags & SRI_FAILOVER_IN_PROGRESS) return 0; /* Last failover attempt started too little time ago? */ if (mstime() - master->failover_start_time < - SENTINEL_PUBLISH_PERIOD*4) return; + SENTINEL_PUBLISH_PERIOD*4) return 0; sentinelStartFailover(master); + return 1; } /* Select a suitable slave to promote. The current algorithm only uses @@ -2986,9 +2992,10 @@ void sentinelHandleRedisInstance(sentinelRedisInstance *ri) { /* Only masters */ if (ri->flags & SRI_MASTER) { sentinelCheckObjectivelyDown(ri); - sentinelStartFailoverIfNeeded(ri); + if (sentinelStartFailoverIfNeeded(ri)) + sentinelAskMasterStateToOtherSentinels(ri,SENTINEL_ASK_FORCED); sentinelFailoverStateMachine(ri); - sentinelAskMasterStateToOtherSentinels(ri); + sentinelAskMasterStateToOtherSentinels(ri,SENTINEL_NO_FLAGS); } } From 9f20780de61d35393b4431ce5dabdb81c8a6d7d1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 17:07:31 +0100 Subject: [PATCH 0871/1752] Sentinel: new failover algo, desync slaves and update config epoch. --- src/sentinel.c | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 55f0adf8cc6..444c6400322 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1511,6 +1511,12 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { (ri->master->failover_state == SENTINEL_FAILOVER_STATE_WAIT_PROMOTION)) { + /* Now that we are sure the slave was reconfigured as a master + * set the master configuration epoch to the epoch we won the + * election to perform this failover. This will force the other + * Sentinels to update their config (assuming there is not + * a newer one already available). */ + ri->master->config_epoch = ri->master->failover_epoch; ri->master->failover_state = SENTINEL_FAILOVER_STATE_RECONF_SLAVES; ri->master->failover_state_change_time = mstime(); sentinelEvent(REDIS_WARNING,"+promoted-slave",ri,"%@"); @@ -2419,6 +2425,13 @@ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char master->leader_epoch = sentinel.current_epoch; sentinelEvent(REDIS_WARNING,"+vote-for-leader",master,"%s %llu", master->leader, (unsigned long long) master->leader_epoch); + /* If we did not voted for ourselves, set the master failover start + * time to now, in order to force a delay before we can start a + * failover for the same master. + * + * The random addition is useful to desynchronize a bit the slaves + * and reduce the chance that no slave gets majority. */ + master->failover_start_time = mstime() + rand() % 2000; } *leader_epoch = master->leader_epoch; @@ -2673,7 +2686,14 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { sdsfree(leader); /* If I'm not the leader, I can't continue with the failover. */ - if (!isleader) return; + if (!isleader) { + /* Abort the failover if I'm not the leader after some time. */ + if (mstime() - ri->failover_start_time > 10000) { + sentinelEvent(REDIS_WARNING,"-failover-abort-not-elected",ri,"%@"); + sentinelAbortFailover(ri); + } + return; + } sentinelEvent(REDIS_WARNING,"+elected-leader",ri,"%@"); /* Start the failover going to the next state if enough time has From 1569326277a602bfb1d124132470cd30d4f66bbf Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 17:21:48 +0100 Subject: [PATCH 0872/1752] Sentinel: added config-epoch to SENTINEL masters output. --- src/sentinel.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 444c6400322..800c6d81aa0 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1941,6 +1941,10 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { /* Only masters */ if (ri->flags & SRI_MASTER) { + addReplyBulkCString(c,"config-epoch"); + addReplyBulkLongLong(c,ri->config_epoch); + fields++; + addReplyBulkCString(c,"num-slaves"); addReplyBulkLongLong(c,dictSize(ri->slaves)); fields++; From f7604b4c07518b118bcc3f6ca6be3058e2b006f1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 23:00:14 +0100 Subject: [PATCH 0873/1752] Sentinel: change event name when converting master to slave. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 800c6d81aa0..2de5942168c 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1545,7 +1545,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->master->addr->ip, ri->master->addr->port); if (retval == REDIS_OK) - sentinelEvent(REDIS_NOTICE,"+demote-old-slave",ri,"%@"); + sentinelEvent(REDIS_NOTICE,"+convert-to-slave",ri,"%@"); } } } From 9f0e52a13d6c454a02fbffd6496d22886d017a5f Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 12 Nov 2013 23:07:33 +0100 Subject: [PATCH 0874/1752] Sentinel: receive Pub/Sub messages from slaves. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 2de5942168c..250ef61a0b3 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1336,7 +1336,7 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) { } } /* Pub / Sub */ - if ((ri->flags & SRI_MASTER) && ri->pc == NULL) { + if ((ri->flags & (SRI_MASTER|SRI_SLAVE)) && ri->pc == NULL) { ri->pc = redisAsyncConnect(ri->addr->ip,ri->addr->port); if (ri->pc->err) { sentinelEvent(REDIS_DEBUG,"-pubsub-link-reconnection",ri,"%@ #%s", From 1a1dc3de3836df54040d829de5e940efea8509e0 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 10:30:45 +0100 Subject: [PATCH 0875/1752] Sentinel: sentinelResetMaster() new flag to avoid removing set of sentinels. This commit also removes some dead code and cleanup generic flags. --- src/sentinel.c | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 250ef61a0b3..41b5a6aa14a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -74,7 +74,6 @@ typedef struct sentinelAddr { #define SRI_FORCE_FAILOVER (1<<13) /* Force failover with master up. */ #define SRI_SCRIPT_KILL_SENT (1<<14) /* SCRIPT KILL already sent on -BUSY */ -#define SENTINEL_NO_FLAGS 0 /* Generic no flags define. */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 #define SENTINEL_ASK_PERIOD 1000 @@ -113,11 +112,13 @@ typedef struct sentinelAddr { #define SENTINEL_MASTER_LINK_STATUS_UP 0 #define SENTINEL_MASTER_LINK_STATUS_DOWN 1 -/* Generic flags that can be used with different functions. */ +/* Generic flags that can be used with different functions. + * They use higher bits to avoid colliding with the function specific + * flags. */ #define SENTINEL_NO_FLAGS 0 -#define SENTINEL_GENERATE_EVENT 1 -#define SENTINEL_LEADER 2 -#define SENTINEL_OBSERVER 4 +#define SENTINEL_GENERATE_EVENT (1<<16) +#define SENTINEL_LEADER (1<<17) +#define SENTINEL_OBSERVER (1<<18) /* Script execution flags and limits. */ #define SENTINEL_SCRIPT_NONE 0 @@ -1081,12 +1082,16 @@ void sentinelDelFlagsToDictOfRedisInstances(dict *instances, int flags) { * 5) In the process of doing this undo the failover if in progress. * 6) Disconnect the connections with the master (will reconnect automatically). */ + +#define SENTINEL_RESET_NO_SENTINELS (1<<0) void sentinelResetMaster(sentinelRedisInstance *ri, int flags) { redisAssert(ri->flags & SRI_MASTER); dictRelease(ri->slaves); - dictRelease(ri->sentinels); ri->slaves = dictCreate(&instancesDictType,NULL); - ri->sentinels = dictCreate(&instancesDictType,NULL); + if (!(flags & SENTINEL_RESET_NO_SENTINELS)) { + dictRelease(ri->sentinels); + ri->sentinels = dictCreate(&instancesDictType,NULL); + } if (ri->cc) sentinelKillLink(ri,ri->cc); if (ri->pc) sentinelKillLink(ri,ri->pc); ri->flags &= SRI_MASTER|SRI_CAN_FAILOVER|SRI_DISCONNECTED; @@ -1136,17 +1141,13 @@ int sentinelResetMastersByPattern(char *pattern, int flags) { * This is used to handle the +switch-master and +redirect-to-master events. * * The function returns REDIS_ERR if the address can't be resolved for some - * reason. Otherwise REDIS_OK is returned. - * - * TODO: make this reset so that original sentinels are re-added with - * same ip / port / runid. - */ + * reason. Otherwise REDIS_OK is returned. */ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, int port) { sentinelAddr *oldaddr, *newaddr; newaddr = createSentinelAddr(ip,port); if (newaddr == NULL) return REDIS_ERR; - sentinelResetMaster(master,SENTINEL_NO_FLAGS); + sentinelResetMaster(master,SENTINEL_RESET_NO_SENTINELS); oldaddr = master->addr; master->addr = newaddr; master->o_down_since_time = 0; @@ -2900,15 +2901,6 @@ void sentinelFailoverSwitchToPromotedSlave(sentinelRedisInstance *master) { old_master_ip = sdsdup(master->addr->ip); old_master_port = master->addr->port; sentinelResetMasterAndChangeAddress(master,ref->addr->ip,ref->addr->port); - /* If this is a real switch and not just a user requested reset, we want - * to add all the known instances as slaves, and also all the sentinels - * back to this master. */ - if (master != ref) { - /* TODO: - createSentinelRedisInstance(NULL,SRI_SLAVE - old_master_ip, old_master_port, master->quorum, master); - */ - } sdsfree(old_master_ip); } From b02ef3d59a2d7d6d271f3ae7551c78006f642e32 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 13:01:11 +0100 Subject: [PATCH 0876/1752] Sentinel: readd slaves back after a master reset. --- src/sentinel.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 41b5a6aa14a..b3f27b7d325 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -445,6 +445,11 @@ void releaseSentinelAddr(sentinelAddr *sa) { zfree(sa); } +/* Return non-zero if two addresses are equal. */ +int sentinelAddrIsEqual(sentinelAddr *a, sentinelAddr *b) { + return a->port == b->port && !strcasecmp(a->ip,b->ip); +} + /* =========================== Events notification ========================== */ /* Send an event to log, pub/sub, user notification script. @@ -1144,15 +1149,54 @@ int sentinelResetMastersByPattern(char *pattern, int flags) { * reason. Otherwise REDIS_OK is returned. */ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, int port) { sentinelAddr *oldaddr, *newaddr; + sentinelAddr **slaves = NULL; + int numslaves = 0, j; + dictIterator *di; + dictEntry *de; newaddr = createSentinelAddr(ip,port); if (newaddr == NULL) return REDIS_ERR; + + /* Make a list of slaves to add back after the reset. + * Don't include the one having the address we are switching to. */ + di = dictGetIterator(master->slaves); + while((de = dictNext(di)) != NULL) { + sentinelRedisInstance *slave = dictGetVal(de); + + if (sentinelAddrIsEqual(slave->addr,newaddr)) continue; + slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1)); + slaves[numslaves++] = createSentinelAddr(slave->addr->ip, + slave->addr->port); + } + dictReleaseIterator(di); + + /* If we are switching to a different address, include the old address + * as a slave as well, so that we'll be able to sense / reconfigure + * the old master. */ + if (!sentinelAddrIsEqual(newaddr,master->addr)) { + slaves = zrealloc(slaves,sizeof(sentinelAddr*)*(numslaves+1)); + slaves[numslaves++] = createSentinelAddr(master->addr->ip, + master->addr->port); + } + + /* Reset and switch address. */ sentinelResetMaster(master,SENTINEL_RESET_NO_SENTINELS); oldaddr = master->addr; master->addr = newaddr; master->o_down_since_time = 0; master->s_down_since_time = 0; + /* Add slaves back. */ + for (j = 0; j < numslaves; j++) { + sentinelRedisInstance *slave; + + slave = createSentinelRedisInstance(NULL,SRI_SLAVE,slaves[j]->ip, + slaves[j]->port, master->quorum, master); + releaseSentinelAddr(slaves[j]); + if (slave) sentinelEvent(REDIS_NOTICE,"+slave",slave,"%@"); + } + zfree(slaves); + /* Release the old address at the end so we are safe even if the function * gets the master->addr->ip and master->addr->port as arguments. */ releaseSentinelAddr(oldaddr); From fc10fb17dac833972c0f53c05ccde599ae58b64b Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 13:43:59 +0100 Subject: [PATCH 0877/1752] Sentinel: fix no-down check in master->slave conversion code. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index b3f27b7d325..35880f7ee55 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1576,7 +1576,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * configuratio via Pub/Sub if any. */ mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; - if (!sentinelRedisInstanceNoDownFor(ri->master,wait_time) || + if (!sentinelRedisInstanceNoDownFor(ri,wait_time) || (mstime()-sentinel.tilt_start_time) < wait_time) return; From 9577fed8a3ec33aa65cfc0653b11f122b27d84b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 16:18:23 +0100 Subject: [PATCH 0878/1752] Sentinel: track role change time. Wait before reconfigurations. --- src/sentinel.c | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 35880f7ee55..29c2c71fd3c 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -157,6 +157,14 @@ typedef struct sentinelRedisInstance { mstime_t down_after_period; /* Consider it down after that period. */ mstime_t info_refresh; /* Time at which we received INFO output from it. */ + /* Role and the first time we observed it. + * This is useful in order to delay replacing what the instance reports + * with our own configuration. We need to always wait some time in order + * to give a chance to the leader to report the new configuration before + * we do silly things. */ + int role_reported; + mstime_t role_reported_time; + /* Master specific. */ dict *sentinels; /* Other sentinels monitoring the same master. */ dict *slaves; /* Slaves for this master instance. */ @@ -913,6 +921,10 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * ri->notification_script = NULL; ri->client_reconfig_script = NULL; + /* Role */ + ri->role_reported = ri->flags & (SRI_MASTER|SRI_SLAVE); + ri->role_reported_time = mstime(); + /* Add into the right table. */ dictAdd(table, ri->name, ri); return ri; @@ -1538,6 +1550,11 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * master, always. */ if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE && ri->slave_master_host) { + if (ri->role_reported != SRI_MASTER) { + ri->role_reported_time = mstime(); + ri->role_reported = SRI_MASTER; + } + sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, "%s %s %d %s %d", ri->name, ri->addr->ip, ri->addr->port, @@ -1549,6 +1566,11 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Handle slave -> master role switch. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { + if (ri->role_reported != SRI_SLAVE) { + ri->role_reported_time = mstime(); + ri->role_reported = SRI_SLAVE; + } + /* If this is a promoted slave we can change state to the * failover state machine. */ if (!sentinel.tilt && @@ -1577,7 +1599,8 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; if (!sentinelRedisInstanceNoDownFor(ri,wait_time) || - (mstime()-sentinel.tilt_start_time) < wait_time) + mstime() - ri->role_reported_time < wait_time || + mstime() - sentinel.tilt_start_time < wait_time) return; /* Make sure the master is sane before reconfiguring this instance From 8c551d65a11f5836b34f7f65ba5e2213d211b2d1 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 16:21:58 +0100 Subject: [PATCH 0879/1752] Sentinel: make sure role_reported is always updated. --- src/sentinel.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 29c2c71fd3c..f7d4b914447 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1548,20 +1548,21 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* When what we believe is our master, turned into a slave, the wiser * thing we can do is to follow the events and redirect to the new * master, always. */ - if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE && ri->slave_master_host) - { + if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) { if (ri->role_reported != SRI_MASTER) { ri->role_reported_time = mstime(); ri->role_reported = SRI_MASTER; } - sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, - "%s %s %d %s %d", - ri->name, ri->addr->ip, ri->addr->port, - ri->slave_master_host, ri->slave_master_port); - sentinelResetMasterAndChangeAddress(ri,ri->slave_master_host, - ri->slave_master_port); - return; /* Don't process anything after this event. */ + if (ri->slave_master_host) { + sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, + "%s %s %d %s %d", + ri->name, ri->addr->ip, ri->addr->port, + ri->slave_master_host, ri->slave_master_port); + sentinelResetMasterAndChangeAddress(ri,ri->slave_master_host, + ri->slave_master_port); + return; /* Don't process anything after this event. */ + } } /* Handle slave -> master role switch. */ From be19e5450cf088d93efe0410f4774d5ca31420ec Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 16:28:52 +0100 Subject: [PATCH 0880/1752] Sentinel: being a master and reporting as slave is considered SDOWN. --- src/sentinel.c | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index f7d4b914447..6858683bd5d 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2321,8 +2321,18 @@ void sentinelCheckSubjectivelyDown(sentinelRedisInstance *ri) { sentinelKillLink(ri,ri->pc); } - /* Update the subjectively down flag. */ - if (elapsed > ri->down_after_period) { + /* Update the subjectively down flag. We believe the instance is in SDOWN + * state if: + * 1) It is not replying. + * 2) We believe it is a master, it reports to be a slave for enough time + * to meet the down_after_period, plus enough time to get two times + * INFO report from the instance. */ + if (elapsed > ri->down_after_period || + (ri->flags & SRI_MASTER && + ri->role_reported == SRI_SLAVE && + mstime() - ri->role_reported_time > + (ri->down_after_period+SENTINEL_INFO_PERIOD*2))) + { /* Is subjectively down */ if ((ri->flags & SRI_S_DOWN) == 0) { sentinelEvent(REDIS_WARNING,"+sdown",ri,"%@"); From e98d82c6398724b88242b0e65a83a9e2d0e3bb9a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 16:36:40 +0100 Subject: [PATCH 0881/1752] Sentinel: role reporting fixed and added in SENTINEL output. --- src/sentinel.c | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 6858683bd5d..d1e844a7d32 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1549,9 +1549,9 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * thing we can do is to follow the events and redirect to the new * master, always. */ if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) { - if (ri->role_reported != SRI_MASTER) { + if (ri->role_reported != SRI_SLAVE) { ri->role_reported_time = mstime(); - ri->role_reported = SRI_MASTER; + ri->role_reported = SRI_SLAVE; } if (ri->slave_master_host) { @@ -1567,9 +1567,9 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Handle slave -> master role switch. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { - if (ri->role_reported != SRI_SLAVE) { + if (ri->role_reported != SRI_MASTER) { ri->role_reported_time = mstime(); - ri->role_reported = SRI_SLAVE; + ri->role_reported = SRI_MASTER; } /* If this is a promoted slave we can change state to the @@ -2006,6 +2006,15 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { addReplyBulkCString(c,"info-refresh"); addReplyBulkLongLong(c,mstime() - ri->info_refresh); fields++; + + addReplyBulkCString(c,"role-reported"); + addReplyBulkCString(c, (ri->role_reported == SRI_MASTER) ? "master" : + "slave"); + fields++; + + addReplyBulkCString(c,"role-reported-time"); + addReplyBulkLongLong(c,mstime() - ri->role_reported_time); + fields++; } /* Only masters */ From 4ccf807abc16e05323ab05a00938277fe17aaaee Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 17:02:09 +0100 Subject: [PATCH 0882/1752] Sentinel: safer slave reconfig, master reported role should match. --- src/sentinel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentinel.c b/src/sentinel.c index d1e844a7d32..ab16e61a97c 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1607,6 +1607,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Make sure the master is sane before reconfiguring this instance * into a slave. */ if (ri->master->flags & SRI_MASTER && + ri->master->role_reported == SRI_MASTER && (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) { From 612dbb2a9189967d7399101906fa065488722b6f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 13 Nov 2013 17:03:42 +0100 Subject: [PATCH 0883/1752] Sentinel: redirect-to-master is not ok with new algorithm. Now Sentinel believe the current configuration is always the winner and should be applied by Sentinels instead of trying to adapt our view of the cluster based on what we observe. So the only way to modify what a Sentinel believe to be the truth is to win an election and advertise the new configuration via Pub / Sub with a greater configuration epoch. --- src/sentinel.c | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index ab16e61a97c..67d4a6d7ecc 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1545,24 +1545,12 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * Some things will not happen if sentinel.tilt is true, but some will * still be processed. */ - /* When what we believe is our master, turned into a slave, the wiser - * thing we can do is to follow the events and redirect to the new - * master, always. */ + /* Handle master -> slave role switch. */ if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) { if (ri->role_reported != SRI_SLAVE) { ri->role_reported_time = mstime(); ri->role_reported = SRI_SLAVE; } - - if (ri->slave_master_host) { - sentinelEvent(REDIS_WARNING,"+redirect-to-master",ri, - "%s %s %d %s %d", - ri->name, ri->addr->ip, ri->addr->port, - ri->slave_master_host, ri->slave_master_port); - sentinelResetMasterAndChangeAddress(ri,ri->slave_master_host, - ri->slave_master_port); - return; /* Don't process anything after this event. */ - } } /* Handle slave -> master role switch. */ From 7dbc0a63f5591ed9a1e5aaaecd817b44d27e6bac Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2013 00:08:13 +0100 Subject: [PATCH 0884/1752] Sentinel: remember last time slave changed master. --- src/sentinel.c | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 67d4a6d7ecc..e577df11697 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -164,6 +164,7 @@ typedef struct sentinelRedisInstance { * we do silly things. */ int role_reported; mstime_t role_reported_time; + mstime_t slave_conf_change_time; /* Last time slave master addr changed. */ /* Master specific. */ dict *sentinels; /* Other sentinels monitoring the same master. */ @@ -924,6 +925,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * /* Role */ ri->role_reported = ri->flags & (SRI_MASTER|SRI_SLAVE); ri->role_reported_time = mstime(); + ri->slave_conf_change_time = mstime(); /* Add into the right table. */ dictAdd(table, ri->name, ri); @@ -1517,13 +1519,24 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { if (role == SRI_SLAVE) { /* master_host: */ if (sdslen(l) >= 12 && !memcmp(l,"master_host:",12)) { - sdsfree(ri->slave_master_host); - ri->slave_master_host = sdsnew(l+12); + if (ri->slave_master_host == NULL || + strcasecmp(l+12,ri->slave_master_host)) + { + sdsfree(ri->slave_master_host); + ri->slave_master_host = sdsnew(l+12); + ri->slave_conf_change_time = mstime(); + } } /* master_port: */ - if (sdslen(l) >= 12 && !memcmp(l,"master_port:",12)) - ri->slave_master_port = atoi(l+12); + if (sdslen(l) >= 12 && !memcmp(l,"master_port:",12)) { + int slave_master_port = atoi(l+12); + + if (ri->slave_master_port != slave_master_port) { + ri->slave_master_port = slave_master_port; + ri->slave_conf_change_time = mstime(); + } + } /* master_link_status: */ if (sdslen(l) >= 19 && !memcmp(l,"master_link_status:",19)) { @@ -1550,6 +1563,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { if (ri->role_reported != SRI_SLAVE) { ri->role_reported_time = mstime(); ri->role_reported = SRI_SLAVE; + ri->slave_conf_change_time = mstime(); } } From 782f9cacaf95c4e812e5a1aed3af6bd49c5a9a0d Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2013 00:29:38 +0100 Subject: [PATCH 0885/1752] Sentinel: reconfigure slaves to right master. --- src/sentinel.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index e577df11697..f40b1ae94b8 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1598,7 +1598,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* A slave turned into a master. We want to force our view and * reconfigure as slave, but make sure to wait some time before * doing this in order to make sure to receive an updated - * configuratio via Pub/Sub if any. */ + * configuration via Pub/Sub if any. */ mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; if (!sentinelRedisInstanceNoDownFor(ri,wait_time) || @@ -1622,6 +1622,32 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { } } + /* Handle slaves replicating to a different master address. */ + if ((ri->flags & SRI_SLAVE) && !sentinel.tilt && + (ri->slave_master_port != ri->master->addr->port || + strcasecmp(ri->slave_master_host,ri->master->addr->ip))) + { + mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; + + if (!sentinelRedisInstanceNoDownFor(ri,wait_time) || + mstime() - ri->slave_conf_change_time < wait_time) + return; + + /* Make sure the master is sane before reconfiguring this instance + * into a slave. */ + if (ri->master->flags & SRI_MASTER && + ri->master->role_reported == SRI_MASTER && + (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && + (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) + { + int retval = sentinelSendSlaveOf(ri, + ri->master->addr->ip, + ri->master->addr->port); + if (retval == REDIS_OK) + sentinelEvent(REDIS_NOTICE,"+fix-slave-config",ri,"%@"); + } + } + /* None of the following conditions are processed when in tilt mode, so * return asap. */ if (sentinel.tilt) return; From 64c8de86576adf3850cea69beb48014f84fbb614 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2013 00:36:43 +0100 Subject: [PATCH 0886/1752] Sentinel: simplify and refactor slave reconfig code. --- src/sentinel.c | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index f40b1ae94b8..a1660b0b5ad 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1433,6 +1433,19 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) { /* ======================== Redis instances pinging ======================== */ +/* Return true if master looks "sane", that is: + * 1) It is actually a master in the current configuration. + * 2) It reports itself as a master. + * 3) It is not SDOWN or ODOWN. + * 4) We obtained last INFO no more than two times the INFO period of time ago. */ +int sentinelMasterLooksSane(sentinelRedisInstance *master) { + return + master->flags & SRI_MASTER && + master->role_reported == SRI_MASTER && + (master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && + (mstime() - master->info_refresh) < SENTINEL_INFO_PERIOD*2; +} + /* Process the INFO output from masters. */ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { sds *lines; @@ -1596,22 +1609,13 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { "start",ri->master->addr,ri->addr); } else if (!sentinel.tilt) { /* A slave turned into a master. We want to force our view and - * reconfigure as slave, but make sure to wait some time before - * doing this in order to make sure to receive an updated - * configuration via Pub/Sub if any. */ + * reconfigure as slave. Wait some time after the change before + * going forward, to receive new configs if any. */ mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; - if (!sentinelRedisInstanceNoDownFor(ri,wait_time) || - mstime() - ri->role_reported_time < wait_time || - mstime() - sentinel.tilt_start_time < wait_time) - return; - - /* Make sure the master is sane before reconfiguring this instance - * into a slave. */ - if (ri->master->flags & SRI_MASTER && - ri->master->role_reported == SRI_MASTER && - (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && - (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) + if (sentinelMasterLooksSane(ri->master) && + sentinelRedisInstanceNoDownFor(ri,wait_time) && + mstime() - ri->role_reported_time > wait_time) { int retval = sentinelSendSlaveOf(ri, ri->master->addr->ip, @@ -1629,16 +1633,11 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { { mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; - if (!sentinelRedisInstanceNoDownFor(ri,wait_time) || - mstime() - ri->slave_conf_change_time < wait_time) - return; - /* Make sure the master is sane before reconfiguring this instance * into a slave. */ - if (ri->master->flags & SRI_MASTER && - ri->master->role_reported == SRI_MASTER && - (ri->master->flags & (SRI_S_DOWN|SRI_O_DOWN)) == 0 && - (mstime() - ri->master->info_refresh) < SENTINEL_INFO_PERIOD*2) + if (sentinelMasterLooksSane(ri->master) && + sentinelRedisInstanceNoDownFor(ri,wait_time) && + mstime() - ri->slave_conf_change_time > wait_time) { int retval = sentinelSendSlaveOf(ri, ri->master->addr->ip, From 0eeb0a0782b18a19e6d724068faba97f32054235 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2013 10:23:05 +0100 Subject: [PATCH 0887/1752] Sentinel: fix conditional to only affect slaves with wrong master. --- src/sentinel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentinel.c b/src/sentinel.c index a1660b0b5ad..f225dc76ef2 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1628,6 +1628,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* Handle slaves replicating to a different master address. */ if ((ri->flags & SRI_SLAVE) && !sentinel.tilt && + role == SRI_SLAVE && (ri->slave_master_port != ri->master->addr->port || strcasecmp(ri->slave_master_host,ri->master->addr->ip))) { From 1a6abe7d79689b43debad0cf55b91063a1fd2119 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2013 10:23:54 +0100 Subject: [PATCH 0888/1752] Sentinel: master address selection in get-master-address refactored. --- src/sentinel.c | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index f225dc76ef2..eff6f0d3f82 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1228,6 +1228,24 @@ int sentinelRedisInstanceNoDownFor(sentinelRedisInstance *ri, mstime_t ms) { return most_recent == 0 || (mstime() - most_recent) > ms; } +/* Return the current master address, that is, its address or the address + * of the promoted slave if already operational. */ +sentinelAddr *sentinelGetCurrentMasterAddress(sentinelRedisInstance *master) { + /* If we are failing over the master, and the state is already + * SENTINEL_FAILOVER_STATE_RECONF_SLAVES or greater, it means that we + * already have the new configuration epoch in the master, and the + * slave acknowledged the configuration switch. Advertise the new + * address. */ + if ((master->flags & SRI_FAILOVER_IN_PROGRESS) && + master->promoted_slave && + master->failover_state >= SENTINEL_FAILOVER_STATE_RECONF_SLAVES) + { + return master->promoted_slave->addr; + } else { + return master->addr; + } +} + /* ============================ Config handling ============================= */ char *sentinelHandleConfiguration(char **argv, int argc) { sentinelRedisInstance *ri; @@ -2219,18 +2237,8 @@ void sentinelCommand(redisClient *c) { } else if (ri->info_refresh == 0) { addReplySds(c,sdsnew("-IDONTKNOW I have not enough information to reply. Please ask another Sentinel.\r\n")); } else { - sentinelAddr *addr = ri->addr; - - /* If we are in the middle of a failover, and the slave was - * already successfully switched to master role, we can advertise - * the new address as slave in order to allow clients to talk - * with the new master ASAP. */ - if ((ri->flags & SRI_FAILOVER_IN_PROGRESS) && - ri->promoted_slave && - ri->failover_state >= SENTINEL_FAILOVER_STATE_RECONF_SLAVES) - { - addr = ri->promoted_slave->addr; - } + sentinelAddr *addr = sentinelGetCurrentMasterAddress(ri); + addReplyMultiBulkLen(c,2); addReplyBulkCString(c,addr->ip); addReplyBulkLongLong(c,addr->port); From e15ba6a6972f7e20fe92340b6d92fd6b73119d52 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 14 Nov 2013 10:25:55 +0100 Subject: [PATCH 0889/1752] Sentinel: fix address of master in Hello messages. Once we switched configuration during a failover, we should advertise the new address. This was a serious race condition as the Sentinel performing the failover for a moment advertised the old address with the new configuration epoch: once trasmitted to the other Sentinels the broken configuration would remain there forever, until the next failover (because a greater configuration epoch is required to overwrite an older one). --- src/sentinel.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index eff6f0d3f82..ae1887e97a5 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1938,6 +1938,7 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { char payload[REDIS_IP_STR_LEN+1024]; sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? ri : ri->master; + sentinelAddr *master_addr = sentinelGetCurrentMasterAddress(master); snprintf(payload,sizeof(payload), "%s,%d,%s,%d,%llu," /* Info about this sentinel. */ @@ -1946,7 +1947,7 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { (master->flags & SRI_CAN_FAILOVER) != 0, (unsigned long long) sentinel.current_epoch, /* --- */ - master->name,master->addr->ip,master->addr->port, + master->name,master_addr->ip,master_addr->port, master->config_epoch); retval = redisAsyncCommand(ri->cc, sentinelPublishReplyCallback, NULL, "PUBLISH %s %s", From 3c4497e83c1766f88c3e5503b808c18523c5f41f Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 10:08:06 +0100 Subject: [PATCH 0890/1752] Sentinel: election timeout define. --- src/sentinel.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index ae1887e97a5..381a6154c9a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -89,6 +89,7 @@ typedef struct sentinelAddr { #define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000 #define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*15*1000) #define SENTINEL_MAX_PENDING_COMMANDS 100 +#define SENTINEL_ELECTION_TIMEOUT 10000 /* How many milliseconds is an information valid? This applies for instance * to the reply to SENTINEL IS-MASTER-DOWN-BY-ADDR replies. */ @@ -2818,7 +2819,7 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { /* If I'm not the leader, I can't continue with the failover. */ if (!isleader) { /* Abort the failover if I'm not the leader after some time. */ - if (mstime() - ri->failover_start_time > 10000) { + if (mstime() - ri->failover_start_time > SENTINEL_ELECTION_TIMEOUT) { sentinelEvent(REDIS_WARNING,"-failover-abort-not-elected",ri,"%@"); sentinelAbortFailover(ri); } From ccaba966bc9d72150229f398ca98dafec25fe3ff Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 11:12:58 +0100 Subject: [PATCH 0891/1752] Sentinel: state machine and timeouts simplified. --- src/sentinel.c | 95 +++++++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 381a6154c9a..415c64863a8 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -87,7 +87,7 @@ typedef struct sentinelAddr { #define SENTINEL_SLAVE_RECONF_RETRY_PERIOD 10000 #define SENTINEL_DEFAULT_PARALLEL_SYNCS 1 #define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000 -#define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*15*1000) +#define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*5*1000) #define SENTINEL_MAX_PENDING_COMMANDS 100 #define SENTINEL_ELECTION_TIMEOUT 10000 @@ -107,8 +107,7 @@ typedef struct sentinelAddr { #define SENTINEL_FAILOVER_STATE_WAIT_NEXT_SLAVE 6 /* wait replication */ #define SENTINEL_FAILOVER_STATE_ALERT_CLIENTS 7 /* Run user script. */ #define SENTINEL_FAILOVER_STATE_WAIT_ALERT_SCRIPT 8 /* Wait script exec. */ -#define SENTINEL_FAILOVER_STATE_DETECT_END 9 /* Check for failover end. */ -#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 10 /* Monitor promoted slave. */ +#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 9 /* Monitor promoted slave. */ #define SENTINEL_MASTER_LINK_STATUS_UP 0 #define SENTINEL_MASTER_LINK_STATUS_DOWN 1 @@ -1695,10 +1694,6 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->flags &= ~SRI_RECONF_INPROG; ri->flags |= SRI_RECONF_DONE; sentinelEvent(REDIS_NOTICE,"+slave-reconf-done",ri,"%@"); - /* If we are moving forward (a new slave is now configured) - * we update the change_time as we are conceptually passing - * to the next slave. */ - ri->failover_state_change_time = mstime(); } } } @@ -1970,7 +1965,6 @@ const char *sentinelFailoverStateStr(int state) { case SENTINEL_FAILOVER_STATE_WAIT_PROMOTION: return "wait_promotion"; case SENTINEL_FAILOVER_STATE_RECONF_SLAVES: return "reconf_slaves"; case SENTINEL_FAILOVER_STATE_ALERT_CLIENTS: return "alert_clients"; - case SENTINEL_FAILOVER_STATE_DETECT_END: return "detect_end"; case SENTINEL_FAILOVER_STATE_UPDATE_CONFIG: return "update_config"; default: return "unknown"; } @@ -2818,17 +2812,20 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { /* If I'm not the leader, I can't continue with the failover. */ if (!isleader) { + int election_timeout = SENTINEL_ELECTION_TIMEOUT; + + /* The election timeout is the MIN between SENTINEL_ELECTION_TIMEOUT + * and the configured failover timeout. */ + if (election_timeout > ri->failover_timeout) + election_timeout = ri->failover_timeout; /* Abort the failover if I'm not the leader after some time. */ - if (mstime() - ri->failover_start_time > SENTINEL_ELECTION_TIMEOUT) { + if (mstime() - ri->failover_start_time > election_timeout) { sentinelEvent(REDIS_WARNING,"-failover-abort-not-elected",ri,"%@"); sentinelAbortFailover(ri); } return; } sentinelEvent(REDIS_WARNING,"+elected-leader",ri,"%@"); - - /* Start the failover going to the next state if enough time has - * elapsed. */ ri->failover_state = SENTINEL_FAILOVER_STATE_SELECT_SLAVE; ri->failover_state_change_time = mstime(); sentinelEvent(REDIS_WARNING,"+failover-state-select-slave",ri,"%@"); @@ -2837,6 +2834,8 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { void sentinelFailoverSelectSlave(sentinelRedisInstance *ri) { sentinelRedisInstance *slave = sentinelSelectSlave(ri); + /* We don't handle the timeout in this state as the function aborts + * the failover or go forward in the next state. */ if (slave == NULL) { sentinelEvent(REDIS_WARNING,"-failover-abort-no-good-slave",ri,"%@"); sentinelAbortFailover(ri); @@ -2854,7 +2853,16 @@ void sentinelFailoverSelectSlave(sentinelRedisInstance *ri) { void sentinelFailoverSendSlaveOfNoOne(sentinelRedisInstance *ri) { int retval; - if (ri->promoted_slave->flags & SRI_DISCONNECTED) return; + /* We can't send the command to the promoted slave if it is now + * disconnected. Retry again and again with this state until the timeout + * is reached, then abort the failover. */ + if (ri->promoted_slave->flags & SRI_DISCONNECTED) { + if (mstime() - ri->failover_state_change_time > ri->failover_timeout) { + sentinelEvent(REDIS_WARNING,"-failover-abort-slave-timeout",ri,"%@"); + sentinelAbortFailover(ri); + } + return; + } /* Send SLAVEOF NO ONE command to turn the slave into a master. * We actually register a generic callback for this command as we don't @@ -2871,16 +2879,11 @@ void sentinelFailoverSendSlaveOfNoOne(sentinelRedisInstance *ri) { /* We actually wait for promotion indirectly checking with INFO when the * slave turns into a master. */ void sentinelFailoverWaitPromotion(sentinelRedisInstance *ri) { - mstime_t elapsed = mstime() - ri->failover_state_change_time; - - if (elapsed >= SENTINEL_PROMOTION_RETRY_PERIOD) { - sentinelEvent(REDIS_WARNING,"-promotion-timeout",ri->promoted_slave, - "%@"); - sentinelEvent(REDIS_WARNING,"+failover-state-select-slave",ri,"%@"); - ri->failover_state = SENTINEL_FAILOVER_STATE_SELECT_SLAVE; - ri->failover_state_change_time = mstime(); - ri->promoted_slave->flags &= ~SRI_PROMOTED; - ri->promoted_slave = NULL; + /* Just handle the timeout. Switching to the next state is handled + * by the function parsing the INFO command of the promoted slave. */ + if (mstime() - ri->failover_state_change_time > ri->failover_timeout) { + sentinelEvent(REDIS_WARNING,"-failover-abort-slave-timeout",ri,"%@"); + sentinelAbortFailover(ri); } } @@ -3004,6 +3007,8 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) { } } dictReleaseIterator(di); + + /* Check if all the slaves are reconfigured and handle timeout. */ sentinelFailoverDetectEnd(master); } @@ -3051,50 +3056,46 @@ void sentinelFailoverStateMachine(sentinelRedisInstance *ri) { case SENTINEL_FAILOVER_STATE_RECONF_SLAVES: sentinelFailoverReconfNextSlave(ri); break; - case SENTINEL_FAILOVER_STATE_DETECT_END: - sentinelFailoverDetectEnd(ri); - break; } } -/* Abort a failover in progress with the following steps: - * 1) Set the master back to the original one, increment the config epoch. - * 2) Reconfig slaves to replicate to the old master. - * 3) Reconfig the promoted slave as a slave as well. */ +/* Abort a failover in progress: + * + * This function can only be called before the promoted slave acknowledged + * the slave -> master switch. Otherwise the failover can't be aborted and + * will reach its end. + * + * If there is a promoted slave and we already got acknowledge of the + * slave -> master switch, we clear our flags and redirect to the + * new master. Eventually the config will be propagated if it is the one + * with the greater config epoch for this master. + * + * Otherwise if we still did not received the acknowledgement from the + * promoted slave, or there is no promoted slave at all, we just clear the + * failover-in-progress state as there is nothing to do (if the promoted + * slave for some reason actually received our "SLAVEOF NO ONE" command + * even if we did not received the ACK, it will be reverted to slave again + * by one of the Sentinels). */ void sentinelAbortFailover(sentinelRedisInstance *ri) { dictIterator *di; dictEntry *de; - int sentinel_role; redisAssert(ri->flags & SRI_FAILOVER_IN_PROGRESS); + redisAssert(ri->failover_state <= SENTINEL_FAILOVER_STATE_WAIT_PROMOTION); - /* Clear failover related flags from slaves. - * Also if we are the leader make sure to send SLAVEOF commands to all the - * already reconfigured slaves in order to turn them back into slaves of - * the original master. */ + /* Clear failover related flags from slaves. */ di = dictGetIterator(ri->slaves); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *slave = dictGetVal(de); - if (!(slave->flags & SRI_DISCONNECTED) && - (slave->flags & (SRI_PROMOTED|SRI_RECONF_SENT|SRI_RECONF_INPROG| - SRI_RECONF_DONE))) - { - int retval; - - retval = sentinelSendSlaveOf(slave,ri->addr->ip,ri->addr->port); - if (retval == REDIS_OK) - sentinelEvent(REDIS_NOTICE,"-slave-reconf-undo",slave,"%@"); - } slave->flags &= ~(SRI_RECONF_SENT|SRI_RECONF_INPROG|SRI_RECONF_DONE); } dictReleaseIterator(di); - sentinel_role = SENTINEL_LEADER; ri->flags &= ~(SRI_FAILOVER_IN_PROGRESS|SRI_FORCE_FAILOVER); ri->failover_state = SENTINEL_FAILOVER_STATE_NONE; ri->failover_state_change_time = mstime(); if (ri->promoted_slave) { - sentinelCallClientReconfScript(ri,sentinel_role,"abort", + sentinelCallClientReconfScript(ri,SENTINEL_LEADER,"abort", ri->promoted_slave->addr,ri->addr); ri->promoted_slave->flags &= ~SRI_PROMOTED; ri->promoted_slave = NULL; From 8ba31c218bbe86c98a3bfd8f4c38642ddc48d137 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 11:30:08 +0100 Subject: [PATCH 0892/1752] Sentinel: failover restart time is now multiple of failover timeout. Also defaulf failover timeout changed to 3 minutes as the failover is a fairly fast procedure most of the times, unless there are a very big number of slaves and the user picked to configure them sequentially (in that case the user should change the failover timeout accordingly). --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 415c64863a8..e599f004620 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -87,7 +87,7 @@ typedef struct sentinelAddr { #define SENTINEL_SLAVE_RECONF_RETRY_PERIOD 10000 #define SENTINEL_DEFAULT_PARALLEL_SYNCS 1 #define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000 -#define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*5*1000) +#define SENTINEL_DEFAULT_FAILOVER_TIMEOUT (60*3*1000) #define SENTINEL_MAX_PENDING_COMMANDS 100 #define SENTINEL_ELECTION_TIMEOUT 10000 @@ -2715,7 +2715,7 @@ int sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { /* Last failover attempt started too little time ago? */ if (mstime() - master->failover_start_time < - SENTINEL_PUBLISH_PERIOD*4) return 0; + master->failover_timeout*2) return 0; sentinelStartFailover(master); return 1; From 66b03c1a4023ba73b3713227f1301423a61ec13d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 11:37:24 +0100 Subject: [PATCH 0893/1752] Sentinel: slaves reconfig delay modified. The time Sentinel waits since the slave is detected to be configured to the wrong master, before reconfiguring it, is now the failover_timeout time as this makes more sense in order to give the Sentinel performing the failover enoung time to reconfigure the slaves slowly (if required by the configuration). Also we now PUBLISH more frequently the new configuraiton as this allows to switch the reapprearing master back to slave faster. --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index e599f004620..6b39803cb89 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -77,7 +77,7 @@ typedef struct sentinelAddr { #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 #define SENTINEL_ASK_PERIOD 1000 -#define SENTINEL_PUBLISH_PERIOD 5000 +#define SENTINEL_PUBLISH_PERIOD 2000 #define SENTINEL_DOWN_AFTER_PERIOD 30000 #define SENTINEL_HELLO_CHANNEL "__sentinel__:hello" #define SENTINEL_TILT_TRIGGER 2000 @@ -1650,7 +1650,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { (ri->slave_master_port != ri->master->addr->port || strcasecmp(ri->slave_master_host,ri->master->addr->ip))) { - mstime_t wait_time = SENTINEL_PUBLISH_PERIOD*4; + mstime_t wait_time = ri->master->failover_timeout; /* Make sure the master is sane before reconfiguring this instance * into a slave. */ From 737062745de934549823c945df97a4dfef034f0d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 11:43:35 +0100 Subject: [PATCH 0894/1752] Sentinel: failover abort function simplified. --- src/sentinel.c | 29 ++--------------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 6b39803cb89..4198afb9570 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1157,7 +1157,7 @@ int sentinelResetMastersByPattern(char *pattern, int flags) { /* Reset the specified master with sentinelResetMaster(), and also change * the ip:port address, but take the name of the instance unmodified. * - * This is used to handle the +switch-master and +redirect-to-master events. + * This is used to handle the +switch-master event. * * The function returns REDIS_ERR if the address can't be resolved for some * reason. Otherwise REDIS_OK is returned. */ @@ -3063,40 +3063,15 @@ void sentinelFailoverStateMachine(sentinelRedisInstance *ri) { * * This function can only be called before the promoted slave acknowledged * the slave -> master switch. Otherwise the failover can't be aborted and - * will reach its end. - * - * If there is a promoted slave and we already got acknowledge of the - * slave -> master switch, we clear our flags and redirect to the - * new master. Eventually the config will be propagated if it is the one - * with the greater config epoch for this master. - * - * Otherwise if we still did not received the acknowledgement from the - * promoted slave, or there is no promoted slave at all, we just clear the - * failover-in-progress state as there is nothing to do (if the promoted - * slave for some reason actually received our "SLAVEOF NO ONE" command - * even if we did not received the ACK, it will be reverted to slave again - * by one of the Sentinels). */ + * will reach its end (possibly by timeout). */ void sentinelAbortFailover(sentinelRedisInstance *ri) { - dictIterator *di; - dictEntry *de; - redisAssert(ri->flags & SRI_FAILOVER_IN_PROGRESS); redisAssert(ri->failover_state <= SENTINEL_FAILOVER_STATE_WAIT_PROMOTION); - /* Clear failover related flags from slaves. */ - di = dictGetIterator(ri->slaves); - while((de = dictNext(di)) != NULL) { - sentinelRedisInstance *slave = dictGetVal(de); - slave->flags &= ~(SRI_RECONF_SENT|SRI_RECONF_INPROG|SRI_RECONF_DONE); - } - dictReleaseIterator(di); - ri->flags &= ~(SRI_FAILOVER_IN_PROGRESS|SRI_FORCE_FAILOVER); ri->failover_state = SENTINEL_FAILOVER_STATE_NONE; ri->failover_state_change_time = mstime(); if (ri->promoted_slave) { - sentinelCallClientReconfScript(ri,SENTINEL_LEADER,"abort", - ri->promoted_slave->addr,ri->addr); ri->promoted_slave->flags &= ~SRI_PROMOTED; ri->promoted_slave = NULL; } From 0b9853ecdc6898bab2fae0764e8aa90e0ca49e83 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 16:02:58 +0100 Subject: [PATCH 0895/1752] Sentinel: added config options useful to take state on config rewrite. We'll use CONFIG REWRITE (internally) in order to store the new configuration of a Sentinel after the internal state changes. In order to do so, we need configuration options (that usually the user will not touch at all) about config epoch of the master, Sentinels and Slaves known for this master, and so forth. --- src/sentinel.c | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 4198afb9570..366f7bcae09 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1314,6 +1314,35 @@ char *sentinelHandleConfiguration(char **argv, int argc) { ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; ri->auth_pass = sdsnew(argv[2]); + } else if (!strcasecmp(argv[0],"config-epoch") && argc == 3) { + /* config-epoch */ + ri = sentinelGetMasterByName(argv[1]); + if (!ri) return "No such master with specified name."; + ri->config_epoch = strtoull(argv[2],NULL,10); + if (ri->config_epoch > sentinel.current_epoch) + sentinel.current_epoch = ri->config_epoch; + } else if (!strcasecmp(argv[0],"slave") && argc == 3) { + sentinelRedisInstance *slave; + + /* slave */ + ri = sentinelGetMasterByName(argv[1]); + if (!ri) return "No such master with specified name."; + if ((slave = createSentinelRedisInstance(NULL,SRI_SLAVE,argv[2], + atoi(argv[3]), ri->quorum, ri)) == NULL) + { + return "Wrong hostname or port for slave."; + } + } else if (!strcasecmp(argv[0],"sentinel") && argc == 3) { + sentinelRedisInstance *si; + + /* sentinel */ + ri = sentinelGetMasterByName(argv[1]); + if (!ri) return "No such master with specified name."; + if ((si = createSentinelRedisInstance(NULL,SRI_SENTINEL,argv[2], + atoi(argv[3]), ri->quorum, ri)) == NULL) + { + return "Wrong hostname or port for sentinel."; + } } else { return "Unrecognized sentinel configuration statement."; } From 04b1fb0b1a1cf512aebc6887c0c29d5d4a985d1e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 18 Nov 2013 18:18:04 +0100 Subject: [PATCH 0896/1752] Fix typo 'configuraiton' in rewriteConfigRewriteLine() comment. --- src/config.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 8c588a06772..235d0dc8306 100644 --- a/src/config.c +++ b/src/config.c @@ -1206,7 +1206,7 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { /* Rewrite the specified configuration option with the new "line". * It progressively uses lines of the file that were already used for the same - * configuraiton option in the old version of the file, removing that line from + * configuration option in the old version of the file, removing that line from * the map of options -> line numbers. * * If there are lines associated with a given configuration option and From 8c3e197040cb735b727cb3d0da8fc565ac3b0bc8 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 09:28:47 +0100 Subject: [PATCH 0897/1752] Sentinel: can-failover option removed, many comments fixed. --- src/sentinel.c | 105 ++++++++++++++----------------------------------- 1 file changed, 29 insertions(+), 76 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 366f7bcae09..5bd562acb14 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -60,25 +60,20 @@ typedef struct sentinelAddr { #define SRI_O_DOWN (1<<5) /* Objectively down (quorum reached). */ #define SRI_MASTER_DOWN (1<<6) /* A Sentinel with this flag set thinks that its master is down. */ -/* SRI_CAN_FAILOVER when set in an SRI_MASTER instance means that we are - * allowed to perform the failover for this master. - * When set in a SRI_SENTINEL instance means that sentinel is allowed to - * perform the failover on its master. */ -#define SRI_CAN_FAILOVER (1<<7) -#define SRI_FAILOVER_IN_PROGRESS (1<<8) /* Failover is in progress for +#define SRI_FAILOVER_IN_PROGRESS (1<<7) /* Failover is in progress for this master. */ -#define SRI_PROMOTED (1<<9) /* Slave selected for promotion. */ -#define SRI_RECONF_SENT (1<<10) /* SLAVEOF sent. */ -#define SRI_RECONF_INPROG (1<<11) /* Slave synchronization in progress. */ -#define SRI_RECONF_DONE (1<<12) /* Slave synchronized with new master. */ -#define SRI_FORCE_FAILOVER (1<<13) /* Force failover with master up. */ -#define SRI_SCRIPT_KILL_SENT (1<<14) /* SCRIPT KILL already sent on -BUSY */ +#define SRI_PROMOTED (1<<8) /* Slave selected for promotion. */ +#define SRI_RECONF_SENT (1<<9) /* SLAVEOF sent. */ +#define SRI_RECONF_INPROG (1<<10) /* Slave synchronization in progress. */ +#define SRI_RECONF_DONE (1<<11) /* Slave synchronized with new master. */ +#define SRI_FORCE_FAILOVER (1<<12) /* Force failover with master up. */ +#define SRI_SCRIPT_KILL_SENT (1<<13) /* SCRIPT KILL already sent on -BUSY */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 #define SENTINEL_ASK_PERIOD 1000 #define SENTINEL_PUBLISH_PERIOD 2000 -#define SENTINEL_DOWN_AFTER_PERIOD 30000 +#define SENTINEL_DEFAULT_DOWN_AFTER 30000 #define SENTINEL_HELLO_CHANNEL "__sentinel__:hello" #define SENTINEL_TILT_TRIGGER 2000 #define SENTINEL_TILT_PERIOD (SENTINEL_PING_PERIOD*30) @@ -895,7 +890,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * ri->s_down_since_time = 0; ri->o_down_since_time = 0; ri->down_after_period = master ? master->down_after_period : - SENTINEL_DOWN_AFTER_PERIOD; + SENTINEL_DEFAULT_DOWN_AFTER; ri->master_link_down_time = 0; ri->auth_pass = NULL; ri->slave_priority = SENTINEL_DEFAULT_SLAVE_PRIORITY; @@ -1113,7 +1108,7 @@ void sentinelResetMaster(sentinelRedisInstance *ri, int flags) { } if (ri->cc) sentinelKillLink(ri,ri->cc); if (ri->pc) sentinelKillLink(ri,ri->pc); - ri->flags &= SRI_MASTER|SRI_CAN_FAILOVER|SRI_DISCONNECTED; + ri->flags &= SRI_MASTER|SRI_DISCONNECTED; if (ri->leader) { sdsfree(ri->leader); ri->leader = NULL; @@ -1278,17 +1273,6 @@ char *sentinelHandleConfiguration(char **argv, int argc) { ri->failover_timeout = atoi(argv[2]); if (ri->failover_timeout <= 0) return "negative or zero time parameter."; - } else if (!strcasecmp(argv[0],"can-failover") && argc == 3) { - /* can-failover */ - int yesno = yesnotoi(argv[2]); - - ri = sentinelGetMasterByName(argv[1]); - if (!ri) return "No such master with specified name."; - if (yesno == -1) return "Argument must be either yes or no."; - if (yesno) - ri->flags |= SRI_CAN_FAILOVER; - else - ri->flags &= ~SRI_CAN_FAILOVER; } else if (!strcasecmp(argv[0],"parallel-syncs") && argc == 3) { /* parallel-syncs */ ri = sentinelGetMasterByName(argv[1]); @@ -1828,25 +1812,24 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd if (strstr(r->element[2]->str,server.runid) != NULL) return; { - /* Format is composed of 9 tokens: - * 0=ip,1=port,2=runid,3=can_failover,4=current_epoch, - * 5=master_name,6=master_ip,7=master_port,8=master_config_epoch. */ - int numtokens, port, removed, canfailover, master_port; + /* Format is composed of 8 tokens: + * 0=ip,1=port,2=runid,3=current_epoch,4=master_name, + * 5=master_ip,6=master_port,7=master_config_epoch. */ + int numtokens, port, removed, master_port; uint64_t current_epoch, master_config_epoch; char **token = sdssplitlen(r->element[2]->str, r->element[2]->len, ",",1,&numtokens); sentinelRedisInstance *si; - if (numtokens == 9) { + if (numtokens == 8) { /* First, try to see if we already have this sentinel. */ port = atoi(token[1]); - master_port = atoi(token[7]); - canfailover = atoi(token[3]); + master_port = atoi(token[6]); si = getSentinelRedisInstanceByAddrAndRunID( master->sentinels,token[0],port,token[2]); - current_epoch = strtoull(token[4],NULL,10); - master_config_epoch = strtoull(token[8],NULL,10); + current_epoch = strtoull(token[3],NULL,10); + master_config_epoch = strtoull(token[7],NULL,10); sentinelRedisInstance *msgmaster; if (!si) { @@ -1873,7 +1856,7 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd } } - /* Update local current_epoch if received current_epoch is greater. */ + /* Update local current_epoch if received current_epoch is greater.*/ if (current_epoch > sentinel.current_epoch) { sentinel.current_epoch = current_epoch; sentinelEvent(REDIS_WARNING,"+new-epoch",ri,"%llu", @@ -1881,31 +1864,25 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd } /* Update master info if received configuration is newer. */ - if ((msgmaster = sentinelGetMasterByName(token[5])) != NULL) { + if ((msgmaster = sentinelGetMasterByName(token[4])) != NULL) { if (msgmaster->config_epoch < master_config_epoch) { msgmaster->config_epoch = master_config_epoch; if (master_port != msgmaster->addr->port || - !strcmp(msgmaster->addr->ip, token[6])) + !strcmp(msgmaster->addr->ip, token[5])) { sentinelEvent(REDIS_WARNING,"+switch-master", msgmaster,"%s %s %d %s %d", msgmaster->name, msgmaster->addr->ip, msgmaster->addr->port, - token[6], master_port); + token[5], master_port); sentinelResetMasterAndChangeAddress(msgmaster, - token[6], master_port); + token[5], master_port); } } } /* Update the state of the Sentinel. */ - if (si) { - si->last_hello_time = mstime(); - if (canfailover) - si->flags |= SRI_CAN_FAILOVER; - else - si->flags &= ~SRI_CAN_FAILOVER; - } + if (si) si->last_hello_time = mstime(); } sdsfreesplitres(token,numtokens); } @@ -1966,10 +1943,9 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { sentinelAddr *master_addr = sentinelGetCurrentMasterAddress(master); snprintf(payload,sizeof(payload), - "%s,%d,%s,%d,%llu," /* Info about this sentinel. */ + "%s,%d,%s,%llu," /* Info about this sentinel. */ "%s,%s,%d,%lld", /* Info about current master. */ ip, server.port, server.runid, - (master->flags & SRI_CAN_FAILOVER) != 0, (unsigned long long) sentinel.current_epoch, /* --- */ master->name,master_addr->ip,master_addr->port, @@ -2140,10 +2116,6 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { addReplyBulkLongLong(c,mstime() - ri->last_hello_time); fields++; - addReplyBulkCString(c,"can-failover-its-master"); - addReplyBulkLongLong(c,(ri->flags & SRI_CAN_FAILOVER) != 0); - fields++; - addReplyBulkCString(c,"voted-leader"); addReplyBulkCString(c,ri->leader ? ri->leader : "?"); fields++; @@ -2542,25 +2514,6 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f /* =============================== FAILOVER ================================= */ -/* Given a master get the "subjective leader", that is, among all the sentinels - * with given characteristics, the one with the lexicographically smaller - * runid. The characteristics required are: - * - * 1) Has SRI_CAN_FAILOVER flag. - * 2) Is not disconnected. - * 3) Recently answered to our ping (no longer than - * SENTINEL_INFO_VALIDITY_TIME milliseconds ago). - * - * The function returns a pointer to an sds string representing the runid of the - * leader sentinel instance (from our point of view). Otherwise NULL is - * returned if there are no suitable sentinels. - */ - -int compareRunID(const void *a, const void *b) { - char **aptrptr = (char**)a, **bptrptr = (char**)b; - return strcasecmp(*aptrptr, *bptrptr); -} - /* Vote for the sentinel with 'req_runid' or return the old vote if already * voted for the specifed 'req_epoch' or one greater. * @@ -2727,8 +2680,9 @@ void sentinelStartFailover(sentinelRedisInstance *master) { /* This function checks if there are the conditions to start the failover, * that is: * - * 1) Enough time has passed since O_DOWN. - * 2) The master is marked as SRI_CAN_FAILOVER, so we can failover it. + * 1) Master must be in ODOWN condition. + * 2) No failover already in progress. + * 3) No failover already attempted recently. * * We still don't know if we'll win the election so it is possible that we * start the failover but that we'll not be able to act. @@ -2736,8 +2690,7 @@ void sentinelStartFailover(sentinelRedisInstance *master) { * Return non-zero if a failover was started. */ int sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { /* We can't failover if the master is not in O_DOWN state. */ - if (!(master->flags & SRI_CAN_FAILOVER) || - !(master->flags & SRI_O_DOWN)) return 0; + if (!(master->flags & SRI_O_DOWN)) return 0; /* Failover already in progress? */ if (master->flags & SRI_FAILOVER_IN_PROGRESS) return 0; From a52909c5f21fc81b8aeff3e67eea96126f911228 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 09:48:12 +0100 Subject: [PATCH 0898/1752] Sentinel: CONFIG REWRITE support for Sentinel config. --- src/config.c | 5 +++ src/redis.h | 2 + src/sentinel.c | 109 +++++++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 112 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index 235d0dc8306..29e1893b6f2 100644 --- a/src/config.c +++ b/src/config.c @@ -1110,6 +1110,10 @@ int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2); void dictSdsDestructor(void *privdata, void *val); void dictListDestructor(void *privdata, void *val); +/* Sentinel config rewriting is implemented inside sentinel.c by + * rewriteConfigSentinelOption(). */ +void rewriteConfigSentinelOption(struct rewriteConfigState *state); + dictType optionToLineDictType = { dictSdsHash, /* hash function */ NULL, /* key dup */ @@ -1680,6 +1684,7 @@ int rewriteConfig(char *path) { rewriteConfigClientoutputbufferlimitOption(state); rewriteConfigNumericalOption(state,"hz",server.hz,REDIS_DEFAULT_HZ); rewriteConfigYesNoOption(state,"aof-rewrite-incremental-fsync",server.aof_rewrite_incremental_fsync,REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC); + if (server.sentinel_mode) rewriteConfigSentinelOption(state); /* Step 3: remove all the orphaned lines in the old file, that is, lines * that were used by a config option and are no longer used, like in case diff --git a/src/redis.h b/src/redis.h index 3a0e9df008a..015d82c4db7 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1144,6 +1144,8 @@ sds keyspaceEventsFlagsToString(int flags); void loadServerConfig(char *filename, char *options); void appendServerSaveParams(time_t seconds, int changes); void resetServerSaveParams(); +struct rewriteConfigState; /* Forward declaration to export API. */ +void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sds line, int force); /* db.c -- Keyspace access API */ int removeExpire(redisDb *db, robj *key); diff --git a/src/sentinel.c b/src/sentinel.c index 5bd562acb14..e71486f276a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1305,10 +1305,10 @@ char *sentinelHandleConfiguration(char **argv, int argc) { ri->config_epoch = strtoull(argv[2],NULL,10); if (ri->config_epoch > sentinel.current_epoch) sentinel.current_epoch = ri->config_epoch; - } else if (!strcasecmp(argv[0],"slave") && argc == 3) { + } else if (!strcasecmp(argv[0],"known-slave") && argc == 3) { sentinelRedisInstance *slave; - /* slave */ + /* known-slave */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; if ((slave = createSentinelRedisInstance(NULL,SRI_SLAVE,argv[2], @@ -1316,10 +1316,10 @@ char *sentinelHandleConfiguration(char **argv, int argc) { { return "Wrong hostname or port for slave."; } - } else if (!strcasecmp(argv[0],"sentinel") && argc == 3) { + } else if (!strcasecmp(argv[0],"known-sentinel") && argc == 3) { sentinelRedisInstance *si; - /* sentinel */ + /* known-sentinel */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; if ((si = createSentinelRedisInstance(NULL,SRI_SENTINEL,argv[2], @@ -1333,6 +1333,107 @@ char *sentinelHandleConfiguration(char **argv, int argc) { return NULL; } +/* Implements CONFIG REWRITE for "sentinel" option. + * This is used not just to rewrite the configuration given by the user + * (the configured masters) but also in order to retain the state of + * Sentinel across restarts: config epoch of masters, associated slaves + * and sentinel instances, and so forth. */ +void rewriteConfigSentinelOption(struct rewriteConfigState *state) { + dictIterator *di, *di2; + dictEntry *de; + + /* For every master emit a "sentinel monitor" config entry. */ + di = dictGetIterator(sentinel.masters); + while((de = dictNext(di)) != NULL) { + sentinelRedisInstance *master, *ri; + sds line; + + /* sentinel monitor */ + master = dictGetVal(de); + line = sdscatprintf(sdsempty(),"sentinel monitor %s %s %d %d", + master->name, master->addr->ip, master->addr->port, + master->quorum); + rewriteConfigRewriteLine(state,"sentinel",line,1); + + /* sentinel down-after-milliseconds */ + if (master->down_after_period != SENTINEL_DEFAULT_DOWN_AFTER) { + line = sdscatprintf(sdsempty(), + "sentinel down-after-milliseconds %s %ld", + master->name, (long) master->down_after_period); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + + /* sentinel failover-timeout */ + if (master->failover_timeout != SENTINEL_DEFAULT_FAILOVER_TIMEOUT) { + line = sdscatprintf(sdsempty(), + "sentinel failover-timeout %s %ld", + master->name, (long) master->failover_timeout); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + + /* sentinel parallel-syncs */ + if (master->parallel_syncs != SENTINEL_DEFAULT_PARALLEL_SYNCS) { + line = sdscatprintf(sdsempty(), + "sentinel parallel-syncs %s %d", + master->name, master->parallel_syncs); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + + /* sentinel notification-script */ + if (master->notification_script) { + line = sdscatprintf(sdsempty(), + "sentinel notification-script %s %s", + master->name, master->notification_script); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + + /* sentinel client-reconfig-script */ + if (master->client_reconfig_script) { + line = sdscatprintf(sdsempty(), + "sentinel client-reconfig-script %s %s", + master->name, master->client_reconfig_script); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + + /* sentinel auth-pass */ + if (master->auth_pass) { + line = sdscatprintf(sdsempty(), + "sentinel auth-pass %s %s", + master->name, master->auth_pass); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + + /* sentinel config-epoch */ + line = sdscatprintf(sdsempty(), + "sentinel config-epoch %s %llu", + master->name, (unsigned long long) master->config_epoch); + rewriteConfigRewriteLine(state,"sentinel",line,1); + + /* sentinel known-slave */ + di2 = dictGetIterator(master->slaves); + while((de = dictNext(di)) != NULL) { + ri = dictGetVal(de); + line = sdscatprintf(sdsempty(), + "sentinel known-slave %s %s %d", + master->name, ri->addr->ip, ri->addr->port); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + dictReleaseIterator(di2); + + /* sentinel known-sentinel */ + di2 = dictGetIterator(master->sentinels); + while((de = dictNext(di)) != NULL) { + ri = dictGetVal(de); + line = sdscatprintf(sdsempty(), + "sentinel known-sentinel %s %s %d", + master->name, ri->addr->ip, ri->addr->port); + rewriteConfigRewriteLine(state,"sentinel",line,1); + } + dictReleaseIterator(di2); + } + dictReleaseIterator(di); +} + /* ====================== hiredis connection handling ======================= */ /* Completely disconnect an hiredis link from an instance. */ From 93d924ff1c70891e3fe93357988d68e158ddeda5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 10:13:04 +0100 Subject: [PATCH 0899/1752] Sentinel: sentinelFlushConfig() to CONFIG REWRITE + fsync. --- src/redis.h | 1 + src/sentinel.c | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/redis.h b/src/redis.h index 015d82c4db7..eb9d9e0794b 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1146,6 +1146,7 @@ void appendServerSaveParams(time_t seconds, int changes); void resetServerSaveParams(); struct rewriteConfigState; /* Forward declaration to export API. */ void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sds line, int force); +int rewriteConfig(char *path); /* db.c -- Keyspace access API */ int removeExpire(redisDb *db, robj *key); diff --git a/src/sentinel.c b/src/sentinel.c index e71486f276a..d7ad3a1c0bb 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -36,6 +36,7 @@ #include #include #include +#include extern char **environ; @@ -1434,6 +1435,27 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { dictReleaseIterator(di); } +/* This function uses the config rewriting Redis engine in order to persist + * the state of the Sentinel in the current configuration file. + * + * Before returning the function calls fsync() against the generated + * configuration file to make sure changes are committed to disk. + * + * On failure the function logs a warning on the Redis log. */ +void sentinelFlushConfig(void) { + int fd; + + if (rewriteConfig(server.configfile) == -1) { + redisLog(REDIS_WARNING,"WARNING: Senitnel was not able to save the new configuration on disk!!!: %s", strerror(errno)); + return; + } + if ((fd = open(server.configfile,O_RDONLY)) != -1) { + fsync(fd); + close(fd); + } + return; +} + /* ====================== hiredis connection handling ======================= */ /* Completely disconnect an hiredis link from an instance. */ From 9a9c0cfaa6e1c292f4d23e62c2516f6f2f120379 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 10:55:43 +0100 Subject: [PATCH 0900/1752] Sentinel: call sentinelFlushConfig() to persist state when needed. Also the sentinel configuration rewriting was modified in order to account for failover in progress, where we need to provide the promoted slave address as master address, and the old master address as one of the slaves address. --- src/sentinel.c | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index d7ad3a1c0bb..fd52d03971a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -336,6 +336,7 @@ void sentinelStartFailover(sentinelRedisInstance *master); void sentinelDiscardReplyCallback(redisAsyncContext *c, void *reply, void *privdata); int sentinelSendSlaveOf(sentinelRedisInstance *ri, char *host, int port); char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch); +void sentinelFlushConfig(void); /* ========================= Dictionary types =============================== */ @@ -1203,13 +1204,17 @@ int sentinelResetMasterAndChangeAddress(sentinelRedisInstance *master, char *ip, slave = createSentinelRedisInstance(NULL,SRI_SLAVE,slaves[j]->ip, slaves[j]->port, master->quorum, master); releaseSentinelAddr(slaves[j]); - if (slave) sentinelEvent(REDIS_NOTICE,"+slave",slave,"%@"); + if (slave) { + sentinelEvent(REDIS_NOTICE,"+slave",slave,"%@"); + sentinelFlushConfig(); + } } zfree(slaves); /* Release the old address at the end so we are safe even if the function * gets the master->addr->ip and master->addr->port as arguments. */ releaseSentinelAddr(oldaddr); + sentinelFlushConfig(); return REDIS_OK; } @@ -1347,12 +1352,14 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { di = dictGetIterator(sentinel.masters); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *master, *ri; + sentinelAddr *master_addr; sds line; /* sentinel monitor */ master = dictGetVal(de); + master_addr = sentinelGetCurrentMasterAddress(master); line = sdscatprintf(sdsempty(),"sentinel monitor %s %s %d %d", - master->name, master->addr->ip, master->addr->port, + master->name, master_addr->ip, master_addr->port, master->quorum); rewriteConfigRewriteLine(state,"sentinel",line,1); @@ -1413,7 +1420,18 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { /* sentinel known-slave */ di2 = dictGetIterator(master->slaves); while((de = dictNext(di)) != NULL) { + sentinelAddr *slave_addr; + ri = dictGetVal(de); + slave_addr = ri->addr; + + /* If master_addr (obtained using sentinelGetCurrentMasterAddress() + * so it may be the address of the promoted slave) is equal to this + * slave's address, a failover is in progress and the slave was + * already successfully promoted. So as the address of this slave + * we use the old master address instead. */ + if (sentinelAddrIsEqual(slave_addr,master_addr)) + slave_addr = master->addr; line = sdscatprintf(sdsempty(), "sentinel known-slave %s %s %d", master->name, ri->addr->ip, ri->addr->port); @@ -1756,6 +1774,7 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { ri->master->config_epoch = ri->master->failover_epoch; ri->master->failover_state = SENTINEL_FAILOVER_STATE_RECONF_SLAVES; ri->master->failover_state_change_time = mstime(); + sentinelFlushConfig(); sentinelEvent(REDIS_WARNING,"+promoted-slave",ri,"%@"); sentinelEvent(REDIS_WARNING,"+failover-state-reconf-slaves", ri->master,"%@"); @@ -1976,6 +1995,7 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd * for Sentinels we don't have a later chance to fill it, * so do it now. */ si->runid = sdsnew(token[2]); + sentinelFlushConfig(); } } From 3e1fc6278dce0cfba22b98d97f70e26e25bf1b23 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 10:59:47 +0100 Subject: [PATCH 0901/1752] Sentinel: rewriteConfigSentinelOption() sub-iterators var typo fixed. --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index fd52d03971a..042ae196cb3 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1419,7 +1419,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { /* sentinel known-slave */ di2 = dictGetIterator(master->slaves); - while((de = dictNext(di)) != NULL) { + while((de = dictNext(di2)) != NULL) { sentinelAddr *slave_addr; ri = dictGetVal(de); @@ -1441,7 +1441,7 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { /* sentinel known-sentinel */ di2 = dictGetIterator(master->sentinels); - while((de = dictNext(di)) != NULL) { + while((de = dictNext(di2)) != NULL) { ri = dictGetVal(de); line = sdscatprintf(sdsempty(), "sentinel known-sentinel %s %s %d", From 679dc8b09df59afa289b49466f38a6e952336f91 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 11:03:47 +0100 Subject: [PATCH 0902/1752] Sentinel: arity of known-sentinel/slave is 4 not 3. --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 042ae196cb3..352b29fcae8 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1311,7 +1311,7 @@ char *sentinelHandleConfiguration(char **argv, int argc) { ri->config_epoch = strtoull(argv[2],NULL,10); if (ri->config_epoch > sentinel.current_epoch) sentinel.current_epoch = ri->config_epoch; - } else if (!strcasecmp(argv[0],"known-slave") && argc == 3) { + } else if (!strcasecmp(argv[0],"known-slave") && argc == 4) { sentinelRedisInstance *slave; /* known-slave */ @@ -1322,7 +1322,7 @@ char *sentinelHandleConfiguration(char **argv, int argc) { { return "Wrong hostname or port for slave."; } - } else if (!strcasecmp(argv[0],"known-sentinel") && argc == 3) { + } else if (!strcasecmp(argv[0],"known-sentinel") && argc == 4) { sentinelRedisInstance *si; /* known-sentinel */ From 1bbce5bf011a455751a79ace157370b91e2561b1 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 11:11:43 +0100 Subject: [PATCH 0903/1752] Sentinel: when writing config on disk, remember sentinels runid. --- src/sentinel.c | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 352b29fcae8..d75990cdc22 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1322,10 +1322,11 @@ char *sentinelHandleConfiguration(char **argv, int argc) { { return "Wrong hostname or port for slave."; } - } else if (!strcasecmp(argv[0],"known-sentinel") && argc == 4) { + } else if (!strcasecmp(argv[0],"known-sentinel") && + (argc == 4 || argc == 5)) { sentinelRedisInstance *si; - /* known-sentinel */ + /* known-sentinel [runid] */ ri = sentinelGetMasterByName(argv[1]); if (!ri) return "No such master with specified name."; if ((si = createSentinelRedisInstance(NULL,SRI_SENTINEL,argv[2], @@ -1333,6 +1334,7 @@ char *sentinelHandleConfiguration(char **argv, int argc) { { return "Wrong hostname or port for sentinel."; } + if (argc == 5) si->runid = sdsnew(argv[4]); } else { return "Unrecognized sentinel configuration statement."; } @@ -1444,8 +1446,10 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { while((de = dictNext(di2)) != NULL) { ri = dictGetVal(de); line = sdscatprintf(sdsempty(), - "sentinel known-sentinel %s %s %d", - master->name, ri->addr->ip, ri->addr->port); + "sentinel known-sentinel %s %s %d%s%s", + master->name, ri->addr->ip, ri->addr->port, + ri->runid ? " " : "", + ri->runid ? ri->runid : ""); rewriteConfigRewriteLine(state,"sentinel",line,1); } dictReleaseIterator(di2); From 6d0400f5694abfc35a9d00c8e19cad41ea994cf7 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 11:24:36 +0100 Subject: [PATCH 0904/1752] Sentinel: no longer used defines removed. --- src/sentinel.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index d75990cdc22..b34f4466b00 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -79,7 +79,6 @@ typedef struct sentinelAddr { #define SENTINEL_TILT_TRIGGER 2000 #define SENTINEL_TILT_PERIOD (SENTINEL_PING_PERIOD*30) #define SENTINEL_DEFAULT_SLAVE_PRIORITY 100 -#define SENTINEL_PROMOTION_RETRY_PERIOD 30000 #define SENTINEL_SLAVE_RECONF_RETRY_PERIOD 10000 #define SENTINEL_DEFAULT_PARALLEL_SYNCS 1 #define SENTINEL_MIN_LINK_RECONNECT_PERIOD 15000 @@ -90,8 +89,6 @@ typedef struct sentinelAddr { /* How many milliseconds is an information valid? This applies for instance * to the reply to SENTINEL IS-MASTER-DOWN-BY-ADDR replies. */ #define SENTINEL_INFO_VALIDITY_TIME 5000 -#define SENTINEL_FAILOVER_FIXED_DELAY 5000 -#define SENTINEL_FAILOVER_MAX_RANDOM_DELAY 10000 /* Failover machine different states. */ #define SENTINEL_FAILOVER_STATE_NONE 0 /* No failover in progress. */ @@ -100,10 +97,7 @@ typedef struct sentinelAddr { #define SENTINEL_FAILOVER_STATE_SEND_SLAVEOF_NOONE 3 /* Slave -> Master */ #define SENTINEL_FAILOVER_STATE_WAIT_PROMOTION 4 /* Wait slave to change role */ #define SENTINEL_FAILOVER_STATE_RECONF_SLAVES 5 /* SLAVEOF newmaster */ -#define SENTINEL_FAILOVER_STATE_WAIT_NEXT_SLAVE 6 /* wait replication */ -#define SENTINEL_FAILOVER_STATE_ALERT_CLIENTS 7 /* Run user script. */ -#define SENTINEL_FAILOVER_STATE_WAIT_ALERT_SCRIPT 8 /* Wait script exec. */ -#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 9 /* Monitor promoted slave. */ +#define SENTINEL_FAILOVER_STATE_UPDATE_CONFIG 6 /* Monitor promoted slave. */ #define SENTINEL_MASTER_LINK_STATUS_UP 0 #define SENTINEL_MASTER_LINK_STATUS_DOWN 1 @@ -2116,7 +2110,6 @@ const char *sentinelFailoverStateStr(int state) { case SENTINEL_FAILOVER_STATE_SEND_SLAVEOF_NOONE: return "send_slaveof_noone"; case SENTINEL_FAILOVER_STATE_WAIT_PROMOTION: return "wait_promotion"; case SENTINEL_FAILOVER_STATE_RECONF_SLAVES: return "reconf_slaves"; - case SENTINEL_FAILOVER_STATE_ALERT_CLIENTS: return "alert_clients"; case SENTINEL_FAILOVER_STATE_UPDATE_CONFIG: return "update_config"; default: return "unknown"; } From 44b3684633534142dd901d012bdc993f4b71a1d5 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 12:34:39 +0100 Subject: [PATCH 0905/1752] Sentinel: failover script execution fixed. --- src/sentinel.c | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index b34f4466b00..33fc7e209c0 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -439,6 +439,16 @@ sentinelAddr *createSentinelAddr(char *hostname, int port) { return sa; } +/* Return a duplicate of the source address. */ +sentinelAddr *dupSentinelAddr(sentinelAddr *src) { + sentinelAddr *sa; + + sa = zmalloc(sizeof(*sa)); + sa->ip = sdsnew(src->ip); + sa->port = src->port; + return sa; +} + /* Free a Sentinel address. Can't fail. */ void releaseSentinelAddr(sentinelAddr *sa) { sdsfree(sa->ip); @@ -785,15 +795,13 @@ void sentinelPendingScriptsCommand(redisClient *c) { * * * - * It is called every time a failover starts, ends, or is aborted. + * It is called every time a failover is performed. * - * is "start", "end" or "abort". + * is currently always "failover". * is either "leader" or "observer". * * from/to fields are respectively master -> promoted slave addresses for - * "start" and "end", or the reverse (promoted slave -> master) in case of - * "abort". - */ + * "start" and "end". */ void sentinelCallClientReconfScript(sentinelRedisInstance *master, int role, char *state, sentinelAddr *from, sentinelAddr *to) { char fromport[32], toport[32]; @@ -2011,13 +2019,21 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd if (master_port != msgmaster->addr->port || !strcmp(msgmaster->addr->ip, token[5])) { + sentinelAddr *old_addr; + sentinelEvent(REDIS_WARNING,"+switch-master", msgmaster,"%s %s %d %s %d", msgmaster->name, msgmaster->addr->ip, msgmaster->addr->port, token[5], master_port); + + old_addr = dupSentinelAddr(msgmaster->addr); sentinelResetMasterAndChangeAddress(msgmaster, token[5], master_port); + sentinelCallClientReconfScript(msgmaster, + SENTINEL_OBSERVER,"start", + old_addr,msgmaster->addr); + releaseSentinelAddr(old_addr); } } } @@ -3040,13 +3056,9 @@ void sentinelFailoverDetectEnd(sentinelRedisInstance *master) { } if (not_reconfigured == 0) { - int role = SENTINEL_LEADER; - sentinelEvent(REDIS_WARNING,"+failover-end",master,"%@"); master->failover_state = SENTINEL_FAILOVER_STATE_UPDATE_CONFIG; master->failover_state_change_time = mstime(); - sentinelCallClientReconfScript(master,role,"end",master->addr, - master->promoted_slave->addr); } /* If I'm the leader it is a good idea to send a best effort SLAVEOF From fc93198ff920945d29625dda88a4509e69feefdf Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 16:20:42 +0100 Subject: [PATCH 0906/1752] Sentinel: various fixes to leader election implementation. --- src/sentinel.c | 68 +++++++++++++++++++++++++++++--------------------- 1 file changed, 40 insertions(+), 28 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 33fc7e209c0..d73114856c2 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2674,7 +2674,7 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f * voted for the specifed 'req_epoch' or one greater. * * If a vote is not available returns NULL, otherwise return the Sentinel - * runid and populate the leader_epoch with the epoch of the last vote. */ + * runid and populate the leader_epoch with the epoch of the vote. */ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char *req_runid, uint64_t *leader_epoch) { if (req_epoch > sentinel.current_epoch) { sentinel.current_epoch = req_epoch; @@ -2682,7 +2682,8 @@ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char (unsigned long long) sentinel.current_epoch); } - if (master->leader_epoch < req_epoch && sentinel.current_epoch <= req_epoch) { + if (master->leader_epoch < req_epoch && sentinel.current_epoch <= req_epoch) + { sdsfree(master->leader); master->leader = sdsnew(req_runid); master->leader_epoch = sentinel.current_epoch; @@ -2694,7 +2695,8 @@ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char * * The random addition is useful to desynchronize a bit the slaves * and reduce the chance that no slave gets majority. */ - master->failover_start_time = mstime() + rand() % 2000; + if (strcasecmp(master->leader,server.runid)) + master->failover_start_time = mstime() + rand() % 2000; } *leader_epoch = master->leader_epoch; @@ -2708,17 +2710,19 @@ struct sentinelLeader { /* Helper function for sentinelGetLeader, increment the counter * relative to the specified runid. */ -void sentinelLeaderIncr(dict *counters, char *runid) { +int sentinelLeaderIncr(dict *counters, char *runid) { dictEntry *de = dictFind(counters,runid); uint64_t oldval; if (de) { oldval = dictGetUnsignedIntegerVal(de); dictSetUnsignedIntegerVal(de,oldval+1); + return oldval+1; } else { de = dictAddRaw(counters,runid); redisAssert(de != NULL); dictSetUnsignedIntegerVal(de,1); + return 1; } } @@ -2736,49 +2740,57 @@ char *sentinelGetLeader(sentinelRedisInstance *master, uint64_t epoch) { char *myvote; char *winner = NULL; uint64_t leader_epoch; + uint64_t max_votes = 0; redisAssert(master->flags & (SRI_O_DOWN|SRI_FAILOVER_IN_PROGRESS)); counters = dictCreate(&leaderVotesDictType,NULL); - /* Count my vote (and vote for myself if I still did not voted for - * the currnet epoch). */ - myvote = sentinelVoteLeader(master,epoch,server.runid,&leader_epoch); - if (myvote && leader_epoch == epoch) { - sentinelLeaderIncr(counters,myvote); - voters++; - } - /* Count other sentinels votes */ di = dictGetIterator(master->sentinels); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *ri = dictGetVal(de); - if (ri->leader == NULL || ri->leader_epoch != sentinel.current_epoch) - continue; - sentinelLeaderIncr(counters,ri->leader); + if (ri->leader != NULL && ri->leader_epoch == sentinel.current_epoch) + sentinelLeaderIncr(counters,ri->leader); voters++; } dictReleaseIterator(di); - voters_quorum = voters/2+1; /* Check what's the winner. For the winner to win, it needs two conditions: * 1) Absolute majority between voters (50% + 1). * 2) And anyway at least master->quorum votes. */ - { - uint64_t max_votes = 0; /* Max votes so far. */ + di = dictGetIterator(counters); + while((de = dictNext(di)) != NULL) { + uint64_t votes = dictGetUnsignedIntegerVal(de); - di = dictGetIterator(counters); - while((de = dictNext(di)) != NULL) { - uint64_t votes = dictGetUnsignedIntegerVal(de); + if (votes > max_votes) { + max_votes = votes; + winner = dictGetKey(de); + } + } + dictReleaseIterator(di); - if (max_votes < votes) { - max_votes = votes; - winner = dictGetKey(de); - } + /* Count this Sentinel vote: + * if this Sentinel did not voted yet, either vote for the most + * common voted sentinel, or for itself if no vote exists at all. */ + if (winner) + myvote = sentinelVoteLeader(master,epoch,winner,&leader_epoch); + else + myvote = sentinelVoteLeader(master,epoch,server.runid,&leader_epoch); + + if (myvote && leader_epoch == epoch) { + uint64_t votes = sentinelLeaderIncr(counters,myvote); + + if (votes > max_votes) { + max_votes = votes; + winner = myvote; } - dictReleaseIterator(di); - if (winner && (max_votes < voters_quorum || max_votes < master->quorum)) - winner = NULL; } + voters++; /* Anyway, count me as one of the voters. */ + + voters_quorum = voters/2+1; + if (winner && (max_votes < voters_quorum || max_votes < master->quorum)) + winner = NULL; + winner = winner ? sdsnew(winner) : NULL; sdsfree(myvote); dictRelease(counters); From 812f76a85027769df9c294cbaa08bf0ac4b4cf39 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 16:50:04 +0100 Subject: [PATCH 0907/1752] Sentinel: distinguish between is-master-down-by-addr requests. Some are just to know if the master is down, and in this case the runid in the request is set to "*", others are actually in order to seek for a vote and get elected. In the latter case the runid is set to the runid of the instance seeking for the vote. --- src/redis.c | 2 ++ src/sentinel.c | 20 +++++++++++++------- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/redis.c b/src/redis.c index d4ff261ff50..60195ab8c6c 100644 --- a/src/redis.c +++ b/src/redis.c @@ -3011,6 +3011,8 @@ int main(int argc, char **argv) { redisLog(REDIS_NOTICE,"The server is now ready to accept connections on port %d", server.port); if (server.sofd > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); + } else { + redisLog(REDIS_WARNING,"Sentinel runid is %s", server.runid); } /* Warning the user about suspicious maxmemory setting. */ diff --git a/src/sentinel.c b/src/sentinel.c index d73114856c2..5a30db7be8d 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2361,8 +2361,9 @@ void sentinelCommand(redisClient *c) { (ri->flags & SRI_MASTER)) isdown = 1; - /* Vote for the master (or fetch the previous vote) */ - if (ri && ri->flags & SRI_MASTER) { + /* Vote for the master (or fetch the previous vote) if the request + * includes a runid, otherwise the sender is not seeking for a vote. */ + if (ri && ri->flags & SRI_MASTER && strcasecmp(c->argv[5]->ptr,"*")) { leader = sentinelVoteLeader(ri,(uint64_t)req_epoch, c->argv[5]->ptr, &leader_epoch); @@ -2372,7 +2373,7 @@ void sentinelCommand(redisClient *c) { * down state, leader, vote epoch. */ addReplyMultiBulkLen(c,3); addReply(c, isdown ? shared.cone : shared.czero); - addReplyBulkCString(c, leader ? leader : "?"); + addReplyBulkCString(c, leader ? leader : "*"); addReplyLongLong(c, (long long)leader_epoch); if (leader) sdsfree(leader); } else if (!strcasecmp(c->argv[1]->ptr,"reset")) { @@ -2607,9 +2608,13 @@ void sentinelReceiveIsMasterDownReply(redisAsyncContext *c, void *reply, void *p } else { ri->flags &= ~SRI_MASTER_DOWN; } - sdsfree(ri->leader); - ri->leader = sdsnew(r->element[1]->str); - ri->leader_epoch = r->element[2]->integer; + if (strcmp(r->element[1]->str,"*")) { + /* If the runid in the reply is not "*" the Sentinel actually + * replied with a vote. */ + sdsfree(ri->leader); + ri->leader = sdsnew(r->element[1]->str); + ri->leader_epoch = r->element[2]->integer; + } } } @@ -2662,7 +2667,8 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f "SENTINEL is-master-down-by-addr %s %s %llu %s", master->addr->ip, port, sentinel.current_epoch, - server.runid); + (master->failover_state > SENTINEL_FAILOVER_STATE_NONE) ? + server.runid : "*"); if (retval == REDIS_OK) ri->pending_commands++; } dictReleaseIterator(di); From c517f723432b43ee70b84119c343fa3e1d14b2fc Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 19 Nov 2013 17:58:11 +0100 Subject: [PATCH 0908/1752] CONFIG REWRITE: don't add the signature if it already exists. At the end of the file, CONFIG REWRITE adds a comment line that: # Generated by CONFIG REWRITE Followed by the additional config options required. However this was added again and again at every rewrite in praticular conditions (when a given set of options change in a given time during the time). Now if it was alrady encountered, it is not added a second time. This is especially important for Sentinel that rewrites the config at every state change. --- src/config.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index 29e1893b6f2..0d8376e5a16 100644 --- a/src/config.c +++ b/src/config.c @@ -1102,6 +1102,8 @@ void configGetCommand(redisClient *c) { * CONFIG REWRITE implementation *----------------------------------------------------------------------------*/ +#define REDIS_CONFIG_REWRITE_SIGNATURE "# Generated by CONFIG REWRITE" + /* We use the following dictionary type to store where a configuration * option is mentioned in the old configuration file, so it's * like "maxmemory" -> list of line numbers (first line is zero). */ @@ -1178,6 +1180,8 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { /* Handle comments and empty lines. */ if (line[0] == '#' || line[0] == '\0') { + if (!state->has_tail && !strcmp(line,REDIS_CONFIG_REWRITE_SIGNATURE)) + state->has_tail = 1; rewriteConfigAppendLine(state,line); continue; } @@ -1249,7 +1253,7 @@ void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sd /* Append a new line. */ if (!state->has_tail) { rewriteConfigAppendLine(state, - sdsnew("# Generated by CONFIG REWRITE")); + sdsnew(REDIS_CONFIG_REWRITE_SIGNATURE)); state->has_tail = 1; } rewriteConfigAppendLine(state,line); From eeb8cb33050093fb8ca252b83f9588bc252c8611 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 20 Nov 2013 15:52:44 +0100 Subject: [PATCH 0909/1752] Sentinel: take the replication offset in slaves state. --- src/sentinel.c | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 5a30db7be8d..60f6a436996 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -167,10 +167,11 @@ typedef struct sentinelRedisInstance { mstime_t master_link_down_time; /* Slave replication link down time. */ int slave_priority; /* Slave priority according to its INFO output. */ mstime_t slave_reconf_sent_time; /* Time at which we sent SLAVE OF */ - struct sentinelRedisInstance *master; /* Master instance if SRI_SLAVE is set. */ + struct sentinelRedisInstance *master; /* Master instance if it's slave. */ char *slave_master_host; /* Master host as reported by INFO */ int slave_master_port; /* Master port as reported by INFO */ int slave_master_link_status; /* Master link status as reported by INFO */ + unsigned long long slave_repl_offset; /* Slave replication offset. */ /* Failover */ char *leader; /* If this is a master instance, this is the runid of the Sentinel that should perform the failover. If @@ -902,6 +903,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * ri->slave_master_host = NULL; ri->slave_master_port = 0; ri->slave_master_link_status = SENTINEL_MASTER_LINK_STATUS_DOWN; + ri->slave_repl_offset = 0; ri->sentinels = dictCreate(&instancesDictType,NULL); ri->quorum = quorum; ri->parallel_syncs = SENTINEL_DEFAULT_PARALLEL_SYNCS; @@ -1740,6 +1742,10 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { /* slave_priority: */ if (sdslen(l) >= 15 && !memcmp(l,"slave_priority:",15)) ri->slave_priority = atoi(l+15); + + /* slave_repl_offset: */ + if (sdslen(l) >= 18 && !memcmp(l,"slave_repl_offset:",18)) + ri->slave_repl_offset = strtoull(l+18,NULL,10); } } ri->info_refresh = mstime(); @@ -2264,6 +2270,10 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { addReplyBulkCString(c,"slave-priority"); addReplyBulkLongLong(c,ri->slave_priority); fields++; + + addReplyBulkCString(c,"slave-repl-offset"); + addReplyBulkLongLong(c,ri->slave_repl_offset); + fields++; } /* Only sentinels */ From 9ecd819ecb02eaa0985902d7a7e1cafab6a99ec6 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 20 Nov 2013 16:05:36 +0100 Subject: [PATCH 0910/1752] Sentinel: select slave with best (greater) replication offset. --- src/sentinel.c | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 60f6a436996..4b1e3af71f7 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2905,6 +2905,9 @@ int sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { * NULL if no suitable slave was found. */ +/* Helper for sentinelSelectSlave(). This is used by qsort() in order to + * sort suitable slaves in a "better first" order, to take the first of + * the list. */ int compareSlavesForPromotion(const void *a, const void *b) { sentinelRedisInstance **sa = (sentinelRedisInstance **)a, **sb = (sentinelRedisInstance **)b; @@ -2913,8 +2916,16 @@ int compareSlavesForPromotion(const void *a, const void *b) { if ((*sa)->slave_priority != (*sb)->slave_priority) return (*sa)->slave_priority - (*sb)->slave_priority; - /* If priority is the same, select the slave with that has the - * lexicographically smaller runid. Note that we try to handle runid + /* If priority is the same, select the slave with greater replication + * offset (processed more data frmo the master). */ + if ((*sa)->slave_repl_offset > (*sb)->slave_repl_offset) { + return -1; /* a < b */ + } else if ((*sa)->slave_repl_offset < (*sb)->slave_repl_offset) { + return 1; /* b > a */ + } + + /* If the replication offset is the same select the slave with that has + * the lexicographically smaller runid. Note that we try to handle runid * == NULL as there are old Redis versions that don't publish runid in * INFO. A NULL runid is considered bigger than any other runid. */ sa_runid = (*sa)->runid; From 87098a39f01ad3cbcb1ea0397c7667f25f2b010f Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 11:31:06 +0100 Subject: [PATCH 0911/1752] Sentinel: Hello message sending code refactored. --- src/sentinel.c | 62 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 21 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 4b1e3af71f7..d3054b0c366 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2051,6 +2051,46 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd } } +/* Send an "Hello" message via Pub/Sub to the specified 'ri' Redis + * instance in order to broadcast the current configuraiton for this + * master, and to advertise the existence of this Sentinel at the same time. + * + * The message has the following format: + * + * sentinel_ip,sentinel_port,sentinel_runid,current_epoch, + * master_name,master_ip,master_port,master_config_epoch. + * + * Returns REDIS_OK if the PUBLISH was queued correctly, otherwise + * REDIS_ERR is returned. */ +int sentinelSendHello(sentinelRedisInstance *ri) { + char ip[REDIS_IP_STR_LEN]; + char payload[REDIS_IP_STR_LEN+1024]; + int retval; + sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? ri : ri->master; + sentinelAddr *master_addr = sentinelGetCurrentMasterAddress(master); + + /* Try to obtain our own IP address. */ + if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) == -1) return REDIS_ERR; + + /* Format and send the Hello message. */ + snprintf(payload,sizeof(payload), + "%s,%d,%s,%llu," /* Info about this sentinel. */ + "%s,%s,%d,%lld", /* Info about current master. */ + ip, server.port, server.runid, + (unsigned long long) sentinel.current_epoch, + /* --- */ + master->name,master_addr->ip,master_addr->port, + master->config_epoch); + retval = redisAsyncCommand(ri->cc, + sentinelPublishReplyCallback, NULL, "PUBLISH %s %s", + SENTINEL_HELLO_CHANNEL,payload); + if (retval != REDIS_OK) return REDIS_ERR; + ri->pending_commands++; + return REDIS_OK; +} + +/* Send periodic PING, INFO, and PUBLISH to the Hello channel to + * the specified master or slave instance. */ void sentinelPingInstance(sentinelRedisInstance *ri) { mstime_t now = mstime(); mstime_t info_period; @@ -2098,27 +2138,7 @@ void sentinelPingInstance(sentinelRedisInstance *ri) { (now - ri->last_pub_time) > SENTINEL_PUBLISH_PERIOD) { /* PUBLISH hello messages to masters and slaves. */ - char ip[REDIS_IP_STR_LEN]; - if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) != -1) { - char payload[REDIS_IP_STR_LEN+1024]; - sentinelRedisInstance *master = (ri->flags & SRI_MASTER) ? - ri : ri->master; - sentinelAddr *master_addr = sentinelGetCurrentMasterAddress(master); - - snprintf(payload,sizeof(payload), - "%s,%d,%s,%llu," /* Info about this sentinel. */ - "%s,%s,%d,%lld", /* Info about current master. */ - ip, server.port, server.runid, - (unsigned long long) sentinel.current_epoch, - /* --- */ - master->name,master_addr->ip,master_addr->port, - master->config_epoch); - retval = redisAsyncCommand(ri->cc, - sentinelPublishReplyCallback, NULL, "PUBLISH %s %s", - SENTINEL_HELLO_CHANNEL,payload); - if (retval != REDIS_OK) return; - ri->pending_commands++; - } + sentinelSendHello(ri); } } From 46e2f3468caa4dc71f136ddaea4d4fad10682c51 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 11:35:50 +0100 Subject: [PATCH 0912/1752] Sentinel: check for disconnected links in sentinelSendHello(). Does not fix any bug as the test is performed by the caller, but better to have the check. --- src/sentinel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentinel.c b/src/sentinel.c index d3054b0c366..d9ffe7cbfeb 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2071,6 +2071,7 @@ int sentinelSendHello(sentinelRedisInstance *ri) { /* Try to obtain our own IP address. */ if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) == -1) return REDIS_ERR; + if (ri->flags & SRI_DISCONNECTED) return; /* Format and send the Hello message. */ snprintf(payload,sizeof(payload), From 4bd1dd1c53042d40a56a050cc60ca0df01537555 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 12:27:14 +0100 Subject: [PATCH 0913/1752] Sentinel: test for writable config file. This commit introduces a funciton called when Sentinel is ready for normal operations to avoid putting Sentinel specific stuff in redis.c. --- src/redis.c | 2 +- src/redis.h | 1 + src/sentinel.c | 13 ++++++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index 60195ab8c6c..0778223e64b 100644 --- a/src/redis.c +++ b/src/redis.c @@ -3012,7 +3012,7 @@ int main(int argc, char **argv) { if (server.sofd > 0) redisLog(REDIS_NOTICE,"The server is now ready to accept connections at %s", server.unixsocket); } else { - redisLog(REDIS_WARNING,"Sentinel runid is %s", server.runid); + sentinelIsRunning(); } /* Warning the user about suspicious maxmemory setting. */ diff --git a/src/redis.h b/src/redis.h index eb9d9e0794b..7ef79cc6888 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1187,6 +1187,7 @@ void initSentinelConfig(void); void initSentinel(void); void sentinelTimer(void); char *sentinelHandleConfiguration(char **argv, int argc); +void sentinelIsRunning(void); /* Scripting */ void scriptingInit(void); diff --git a/src/sentinel.c b/src/sentinel.c index d9ffe7cbfeb..e6efa7ea721 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -415,6 +415,17 @@ void initSentinel(void) { sentinel.scripts_queue = listCreate(); } +/* This function gets called when the server is in Sentinel mode, started, + * loaded the configuration, and is ready for normal operations. */ +void sentinelIsRunning(void) { + redisLog(REDIS_WARNING,"Sentinel runid is %s", server.runid); + + if (server.configfile == NULL || access(server.configfile,W_OK) == -1) { + redisLog(REDIS_WARNING,"Sentinel started without a config file, or config file not writable. Exiting..."); + exit(1); + } +} + /* ============================== sentinelAddr ============================== */ /* Create a sentinelAddr object and return it on success. @@ -2071,7 +2082,7 @@ int sentinelSendHello(sentinelRedisInstance *ri) { /* Try to obtain our own IP address. */ if (anetSockName(ri->cc->c.fd,ip,sizeof(ip),NULL) == -1) return REDIS_ERR; - if (ri->flags & SRI_DISCONNECTED) return; + if (ri->flags & SRI_DISCONNECTED) return REDIS_ERR; /* Format and send the Hello message. */ snprintf(payload,sizeof(payload), From 750b007d7ce9b532a482e03e5e2cd65b1c26fdef Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 12:39:47 +0100 Subject: [PATCH 0914/1752] Sentinel: manual failover works again. --- src/sentinel.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index e6efa7ea721..e9562f3ad7f 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2454,6 +2454,8 @@ void sentinelCommand(redisClient *c) { addReplySds(c,sdsnew("-NOGOODSLAVE No suitable slave to promote\r\n")); return; } + redisLog(REDIS_WARNING,"Executing user requested FAILOVER of '%s'", + ri->name); sentinelStartFailover(ri); ri->flags |= SRI_FORCE_FAILOVER; addReply(c,shared.ok); @@ -3019,8 +3021,9 @@ void sentinelFailoverWaitStart(sentinelRedisInstance *ri) { isleader = leader && strcasecmp(leader,server.runid) == 0; sdsfree(leader); - /* If I'm not the leader, I can't continue with the failover. */ - if (!isleader) { + /* If I'm not the leader, and it is not a forced failover via + * SENTINEL FAILOVER, then I can't continue with the failover. */ + if (!isleader && !(ri->flags & SRI_FORCE_FAILOVER)) { int election_timeout = SENTINEL_ELECTION_TIMEOUT; /* The election timeout is the MIN between SENTINEL_ELECTION_TIMEOUT From 0b2639123beda2475e2c98e155946c3734fb11a6 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 15:43:48 +0100 Subject: [PATCH 0915/1752] Sentinel: removed mem leak and useless code. --- src/sentinel.c | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index e9562f3ad7f..85cec9908f1 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2671,14 +2671,6 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f dictIterator *di; dictEntry *de; - /* Vote for myself if I see the master is already in ODOWN state. */ - if (master->flags & SRI_O_DOWN) { - uint64_t leader_epoch; - - sentinelVoteLeader(master,sentinel.current_epoch,server.runid, - &leader_epoch); - } - di = dictGetIterator(master->sentinels); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *ri = dictGetVal(de); From c84275ee605fdaf5876508d48e69222a731b730e Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 16:05:41 +0100 Subject: [PATCH 0916/1752] Sentinel: cleanup around SENTINEL_INFO_VALIDITY_TIME. --- src/sentinel.c | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 85cec9908f1..7a4d6ae5995 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -58,7 +58,7 @@ typedef struct sentinelAddr { #define SRI_SENTINEL (1<<2) #define SRI_DISCONNECTED (1<<3) #define SRI_S_DOWN (1<<4) /* Subjectively down (no quorum). */ -#define SRI_O_DOWN (1<<5) /* Objectively down (quorum reached). */ +#define SRI_O_DOWN (1<<5) /* Objectively down (confirmed by others). */ #define SRI_MASTER_DOWN (1<<6) /* A Sentinel with this flag set thinks that its master is down. */ #define SRI_FAILOVER_IN_PROGRESS (1<<7) /* Failover is in progress for @@ -70,6 +70,7 @@ typedef struct sentinelAddr { #define SRI_FORCE_FAILOVER (1<<12) /* Force failover with master up. */ #define SRI_SCRIPT_KILL_SENT (1<<13) /* SCRIPT KILL already sent on -BUSY */ +/* Note: times are in milliseconds. */ #define SENTINEL_INFO_PERIOD 10000 #define SENTINEL_PING_PERIOD 1000 #define SENTINEL_ASK_PERIOD 1000 @@ -86,10 +87,6 @@ typedef struct sentinelAddr { #define SENTINEL_MAX_PENDING_COMMANDS 100 #define SENTINEL_ELECTION_TIMEOUT 10000 -/* How many milliseconds is an information valid? This applies for instance - * to the reply to SENTINEL IS-MASTER-DOWN-BY-ADDR replies. */ -#define SENTINEL_INFO_VALIDITY_TIME 5000 - /* Failover machine different states. */ #define SENTINEL_FAILOVER_STATE_NONE 0 /* No failover in progress. */ #define SENTINEL_FAILOVER_STATE_WAIT_START 1 /* Wait for failover_start_time*/ @@ -2679,7 +2676,7 @@ void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int f int retval; /* If the master state from other sentinel is too old, we clear it. */ - if (elapsed > SENTINEL_INFO_VALIDITY_TIME) { + if (elapsed > SENTINEL_ASK_PERIOD*5) { ri->flags &= ~SRI_MASTER_DOWN; sdsfree(ri->leader); ri->leader = NULL; @@ -2917,15 +2914,26 @@ int sentinelStartFailoverIfNeeded(sentinelRedisInstance *master) { * the following parameters: * * 1) None of the following conditions: S_DOWN, O_DOWN, DISCONNECTED. - * 2) last_avail_time more recent than SENTINEL_INFO_VALIDITY_TIME. - * 3) info_refresh more recent than SENTINEL_INFO_VALIDITY_TIME. + * 2) Last time the slave replied to ping no more than 5 times the PING period. + * 3) info_refresh not older than 3 times the INFO refresh period. * 4) master_link_down_time no more than: * (now - master->s_down_since_time) + (master->down_after_period * 10). + * Basically since the master is down from our POV, the slave reports + * to be disconnected no more than 10 times the configured down-after-period. + * This is pretty much black magic but the idea is, the master was not + * available so the slave may be lagging, but not over a certain time. + * Anyway we'll select the best slave according to replication offset. * 5) Slave priority can't be zero, otherwise the slave is discarded. * * Among all the slaves matching the above conditions we select the slave - * with lower slave_priority. If priority is the same we select the slave - * with lexicographically smaller runid. + * with, in order of sorting key: + * + * - lower slave_priority. + * - bigger processed replication offset. + * - lexicographically smaller runid. + * + * Basically if runid is the same, the slave that processed more commands + * from the master is selected. * * The function returns the pointer to the selected slave, otherwise * NULL if no suitable slave was found. @@ -2978,18 +2986,20 @@ sentinelRedisInstance *sentinelSelectSlave(sentinelRedisInstance *master) { di = dictGetIterator(master->slaves); while((de = dictNext(di)) != NULL) { sentinelRedisInstance *slave = dictGetVal(de); - mstime_t info_validity_time = mstime()-SENTINEL_INFO_VALIDITY_TIME; + mstime_t info_validity_time; if (slave->flags & (SRI_S_DOWN|SRI_O_DOWN|SRI_DISCONNECTED)) continue; - if (slave->last_avail_time < info_validity_time) continue; + if (mstime() - slave->last_avail_time > SENTINEL_PING_PERIOD*5) continue; if (slave->slave_priority == 0) continue; /* If the master is in SDOWN state we get INFO for slaves every second. * Otherwise we get it with the usual period so we need to account for * a larger delay. */ - if ((master->flags & SRI_S_DOWN) == 0) - info_validity_time -= SENTINEL_INFO_PERIOD; - if (slave->info_refresh < info_validity_time) continue; + if (master->flags & SRI_S_DOWN) + info_validity_time = SENTINEL_PING_PERIOD*5; + else + info_validity_time = SENTINEL_INFO_PERIOD*3; + if (mstime() - slave->info_refresh > info_validity_time) continue; if (slave->master_link_down_time > max_master_down_time) continue; instance[instances++] = slave; } From 312ca4dacc923c17314134fd504d4098b4e853b7 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 16:22:59 +0100 Subject: [PATCH 0917/1752] Sentinel: different comments updated to new implementation. --- src/sentinel.c | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 7a4d6ae5995..44c5fedcc09 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -995,8 +995,8 @@ const char *sentinelRedisInstanceTypeStr(sentinelRedisInstance *ri) { else return "unknown"; } -/* This function removes all the instances found in the dictionary of instances - * 'd', having either: +/* This function removes all the instances found in the dictionary of + * sentinels in the specified 'master', having either: * * 1) The same ip/port as specified. * 2) The same runid. @@ -1007,13 +1007,9 @@ const char *sentinelRedisInstanceTypeStr(sentinelRedisInstance *ri) { * * This function is useful because every time we add a new Sentinel into * a master's Sentinels dictionary, we want to be very sure about not - * having duplicated instances for any reason. This is so important because - * we use those other sentinels in order to run our quorum protocol to - * understand if it's time to proceed with the fail over. - * - * Making sure no duplication is possible we greatly improve the robustness - * of the quorum (otherwise we may end counting the same instance multiple - * times for some reason). + * having duplicated instances for any reason. This is important because + * other sentinels are needed to reach ODOWN quorum, and later to get + * voted for a given configuration epoch in order to perform the failover. * * The function returns the number of Sentinels removed. */ int removeMatchingSentinelsFromMaster(sentinelRedisInstance *master, char *ip, int port, char *runid) { @@ -1625,7 +1621,7 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) { * 1) It is actually a master in the current configuration. * 2) It reports itself as a master. * 3) It is not SDOWN or ODOWN. - * 4) We obtained last INFO no more than two times the INFO period of time ago. */ + * 4) We obtained last INFO no more than two times the INFO period time ago. */ int sentinelMasterLooksSane(sentinelRedisInstance *master) { return master->flags & SRI_MASTER && @@ -2562,8 +2558,8 @@ void sentinelCheckSubjectivelyDown(sentinelRedisInstance *ri) { sentinelKillLink(ri,ri->pc); } - /* Update the subjectively down flag. We believe the instance is in SDOWN - * state if: + /* Update the SDOWN flag. We believe the instance is SDOWN if: + * * 1) It is not replying. * 2) We believe it is a master, it reports to be a slave for enough time * to meet the down_after_period, plus enough time to get two times @@ -2589,7 +2585,12 @@ void sentinelCheckSubjectivelyDown(sentinelRedisInstance *ri) { } } -/* Is this instance down accordingly to the configured quorum? */ +/* Is this instance down according to the configured quorum? + * + * Note that ODOWN is a weak quorum, it only means that enough Sentinels + * reported in a given time range that the instance was not reachable. + * However messages can be delayed so there are no strong guarantees about + * N instances agreeing at the same time about the down state. */ void sentinelCheckObjectivelyDown(sentinelRedisInstance *master) { dictIterator *di; dictEntry *de; @@ -2659,10 +2660,10 @@ void sentinelReceiveIsMasterDownReply(redisAsyncContext *c, void *reply, void *p } } -/* If we think (subjectively) the master is down, we start sending +/* If we think the master is down, we start sending * SENTINEL IS-MASTER-DOWN-BY-ADDR requests to other sentinels - * in order to get the replies that allow to reach the quorum and - * possibly also mark the master as objectively down. */ + * in order to get the replies that allow to reach the quorum + * needed to mark the master in ODOWN state and trigger a failover. */ #define SENTINEL_ASK_FORCED (1<<0) void sentinelAskMasterStateToOtherSentinels(sentinelRedisInstance *master, int flags) { dictIterator *di; @@ -3224,11 +3225,7 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) { /* This function is called when the slave is in * SENTINEL_FAILOVER_STATE_UPDATE_CONFIG state. In this state we need - * to remove it from the master table and add the promoted slave instead. - * - * If there are no promoted slaves as this instance is unique, we remove - * and re-add it with the same address to trigger a complete state - * refresh. */ + * to remove it from the master table and add the promoted slave instead. */ void sentinelFailoverSwitchToPromotedSlave(sentinelRedisInstance *master) { sentinelRedisInstance *ref = master->promoted_slave ? master->promoted_slave : master; From f75fccafe7857c0e3d9b43f4874dab77ea8453e5 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 21 Nov 2013 17:07:00 +0100 Subject: [PATCH 0918/1752] Sentinel: example sentinel.conf updated. --- sentinel.conf | 58 ++++++++++++++++++++++++--------------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/sentinel.conf b/sentinel.conf index ac687b5358c..248e76c06de 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -9,6 +9,10 @@ port 26379 # Tells Sentinel to monitor this slave, and to consider it in O_DOWN # (Objectively Down) state only if at least sentinels agree. # +# Note that whatever is the ODOWN quorum, a Sentinel will require to +# be elected by the majority of the known Sentinels in order to +# start a failover, so no failover can be performed in minority. +# # Note: master name should not include special characters or spaces. # The valid charset is A-z 0-9 and the three characters ".-_". sentinel monitor mymaster 127.0.0.1 6379 2 @@ -42,11 +46,6 @@ sentinel monitor mymaster 127.0.0.1 6379 2 # Default is 30 seconds. sentinel down-after-milliseconds mymaster 30000 -# sentinel can-failover -# -# Specify if this Sentinel can start the failover for this master. -sentinel can-failover mymaster yes - # sentinel parallel-syncs # # How many slaves we can reconfigure to point to the new slave simultaneously @@ -57,19 +56,28 @@ sentinel parallel-syncs mymaster 1 # sentinel failover-timeout # -# Specifies the failover timeout in milliseconds. When this time has elapsed -# without any progress in the failover process, it is considered concluded by -# the sentinel even if not all the attached slaves were correctly configured -# to replicate with the new master (however a "best effort" SLAVEOF command -# is sent to all the slaves before). +# Specifies the failover timeout in milliseconds. It is used in many ways: +# +# - The time needed to re-start a failover after a previous failover was +# already tried against the same master by a given Sentinel, is two +# times the failover timeout. +# +# - The time needed for a slave replicating to a wrong master according +# to a Sentinel currnet configuration, to be forced to replicate +# with the right master, is exactly the failover timeout (counting since +# the moment a Sentinel detected the misconfiguration). # -# Also when 25% of this time has elapsed without any advancement, and there -# is a leader switch (the sentinel did not started the failover but is now -# elected as leader), the sentinel will continue the failover doing a -# "takeover". +# - The time needed to cancel a failover that is already in progress but +# did not produced any configuration change (SLAVEOF NO ONE yet not +# acknowledged by the promoted slave). # -# Default is 15 minutes. -sentinel failover-timeout mymaster 900000 +# - The maximum time a failover in progress waits for all the slaves to be +# reconfigured as slaves of the new master. However even after this time +# the slaves will be reconfigured by the Sentinels anyway, but not with +# the exact parallel-syncs progression as specified. +# +# Default is 3 minutes. +sentinel failover-timeout mymaster 180000 # SCRIPTS EXECUTION # @@ -114,32 +122,20 @@ sentinel failover-timeout mymaster 900000 # # sentinel client-reconfig-script # -# When the failover starts, ends, or is aborted, a script can be called in +# When the master changed because of a failover a script can be called in # order to perform application-specific tasks to notify the clients that the # configuration has changed and the master is at a different address. # -# The script is called in the following cases: -# -# Failover started (a slave is already promoted) -# Failover finished (all the additional slaves already reconfigured) -# Failover aborted (in that case the script was previously called when the -# failover started, and now gets called again with swapped -# addresses). -# # The following arguments are passed to the script: # # # -# is "start", "end" or "abort" +# is currently always "failover" # is either "leader" or "observer" # # The arguments from-ip, from-port, to-ip, to-port are used to communicate # the old address of the master and the new address of the elected slave -# (now a master) in the case state is "start" or "end". -# -# For abort instead the "from" is the address of the promoted slave and -# "to" is the address of the original master address, since the failover -# was aborted. +# (now a master). # # This script should be resistant to multiple invocations. # From 4e6d34bd2ccd8998335baefb6be08e1704a839d2 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 22 Nov 2013 10:25:24 +0100 Subject: [PATCH 0919/1752] Redis 2.8.0. --- 00-RELEASENOTES | 9 +++++++++ src/version.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 1336e7c49a2..17cdcd0d950 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,15 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8.0 ] Release date: 22 Nov 2013 + +# UPGRADE URGENCY: LOW, unless you want to upgrade to new Sentinel code. + +* [FIX] Fixed an error in rdbWriteRaw() that should have no practical impact. +* [NEW] Log the new master when SLAVEOF command is used. +* [NEW] Sentinel code synchronized with the unstable branch, the new Sentinel + is a reimplementation that uses more reliable algorithms. + --[ Redis 2.8 Release Candidate 6 (2.7.106) ] Release date: 6 Nov 2013 This is the 6th release candidate of Redis 2.8 (official version is 2.7.106). diff --git a/src/version.h b/src/version.h index a3fb3c93ae6..b7b6b0081b6 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.7.106" +#define REDIS_VERSION "2.8.0" From e83741746c0ac669231e0d2fabe32a8e4c68d803 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 25 Nov 2013 10:21:18 +0100 Subject: [PATCH 0920/1752] Fix false positive in memory efficiency test. Fixes issue #1298. --- tests/unit/memefficiency.tcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/memefficiency.tcl b/tests/unit/memefficiency.tcl index 3612f06e56f..14e135ced1c 100644 --- a/tests/unit/memefficiency.tcl +++ b/tests/unit/memefficiency.tcl @@ -22,7 +22,7 @@ start_server {tags {"memefficiency"}} { 64 0.25 128 0.35 1024 0.75 - 16384 0.90 + 16384 0.82 } { test "Memory efficiency with values in range $size_range" { set efficiency [test_memory_efficiency $size_range] From d2402022610e25fd6c330e1d9ecb81c5667b4f76 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 25 Nov 2013 10:24:34 +0100 Subject: [PATCH 0921/1752] Sentinel: fix type specifier for Hello msg generation. This fixes issue #1395. --- src/sentinel.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 44c5fedcc09..9ae76859778 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2080,12 +2080,12 @@ int sentinelSendHello(sentinelRedisInstance *ri) { /* Format and send the Hello message. */ snprintf(payload,sizeof(payload), "%s,%d,%s,%llu," /* Info about this sentinel. */ - "%s,%s,%d,%lld", /* Info about current master. */ + "%s,%s,%d,%llu", /* Info about current master. */ ip, server.port, server.runid, (unsigned long long) sentinel.current_epoch, /* --- */ master->name,master_addr->ip,master_addr->port, - master->config_epoch); + (unsigned long long) master->config_epoch); retval = redisAsyncCommand(ri->cc, sentinelPublishReplyCallback, NULL, "PUBLISH %s %s", SENTINEL_HELLO_CHANNEL,payload); From d13635b2a9a4b93fdd5606f7a2aac0956434c6f6 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 25 Nov 2013 10:57:20 +0100 Subject: [PATCH 0922/1752] Sentinel: fixes inverted strcmp() test preventing config updates. The result of this one-char bug was pretty serious, if the new master had the same port of the previous master, but just a different IP address, non-leader Sentinels would not be able to recognize the configuration change. This commit fixes issue #1394. Many thanks to @shanemadden that reported the bug and helped investigating it. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 9ae76859778..7318832d07a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2027,7 +2027,7 @@ void sentinelReceiveHelloMessages(redisAsyncContext *c, void *reply, void *privd if (msgmaster->config_epoch < master_config_epoch) { msgmaster->config_epoch = master_config_epoch; if (master_port != msgmaster->addr->port || - !strcmp(msgmaster->addr->ip, token[5])) + strcmp(msgmaster->addr->ip, token[5])) { sentinelAddr *old_addr; From dd239c37db09c77f65e735a9917d1c5a2ab822ca Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 25 Nov 2013 11:21:49 +0100 Subject: [PATCH 0923/1752] Redis 2.8.1. --- 00-RELEASENOTES | 9 +++++++++ src/version.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 17cdcd0d950..9fcb663d8b0 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,15 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8.1 ] Release date: 25 Nov 2013 + +# UPGRADE URGENCY: LOW for Redis, CRITICAL for Senitnel. You don't need to + upgrade your Redis instances but it is highly recommended + to upgrade and restart all the Sentinel processes. + +* [FIX] Fixed a bug in "new Sentinel" config propagation. +* [FIX] Fixed a false positive in Redis tests. + --[ Redis 2.8.0 ] Release date: 22 Nov 2013 # UPGRADE URGENCY: LOW, unless you want to upgrade to new Sentinel code. diff --git a/src/version.h b/src/version.h index b7b6b0081b6..7afbdcb3cde 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.8.0" +#define REDIS_VERSION "2.8.1" From e5c577e679ad156e15d2e56a755472079d13ad6d Mon Sep 17 00:00:00 2001 From: huangz1990 Date: Tue, 26 Nov 2013 19:55:51 +0800 Subject: [PATCH 0924/1752] fix a bug in sentinel.c about pub/sub link --- src/sentinel.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 7318832d07a..1ca003c8b35 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1609,9 +1609,8 @@ void sentinelReconnectInstance(sentinelRedisInstance *ri) { } } /* Clear the DISCONNECTED flags only if we have both the connections - * (or just the commands connection if this is a slave or a - * sentinel instance). */ - if (ri->cc && (ri->flags & (SRI_SLAVE|SRI_SENTINEL) || ri->pc)) + * (or just the commands connection if this is a sentinel instance). */ + if (ri->cc && (ri->flags & SRI_SENTINEL || ri->pc)) ri->flags &= ~SRI_DISCONNECTED; } From a46f841df3c4ef5a60117f2db1f7e4c6fbacbceb Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Nov 2013 15:23:46 +0100 Subject: [PATCH 0925/1752] Sentinel: log vote received from other Sentinels. --- src/sentinel.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 1ca003c8b35..4a6dbeada0b 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2653,6 +2653,11 @@ void sentinelReceiveIsMasterDownReply(redisAsyncContext *c, void *reply, void *p /* If the runid in the reply is not "*" the Sentinel actually * replied with a vote. */ sdsfree(ri->leader); + if (ri->leader_epoch != r->element[2]->integer) + redisLog(REDIS_WARNING, + "%s voted for %s %llu", ri->name, + r->element[1]->str, + (unsigned long long) r->element[2]->integer); ri->leader = sdsnew(r->element[1]->str); ri->leader_epoch = r->element[2]->integer; } From 2eb8a46061037e9e7daae6d82e2c556b493a522c Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Nov 2013 16:16:58 +0100 Subject: [PATCH 0926/1752] Reply to PING with error when there is a MISCONF state. --- src/redis.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 0778223e64b..dd2a0df0f88 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1922,7 +1922,8 @@ int processCommand(redisClient *c) { if (server.stop_writes_on_bgsave_err && server.saveparamslen > 0 && server.lastbgsave_status == REDIS_ERR && - c->cmd->flags & REDIS_CMD_WRITE) + (c->cmd->flags & REDIS_CMD_WRITE || + c->cmd->proc == pingCommand)) { flagTransaction(c); addReply(c, shared.bgsaveerr); From 50d140e90befee67067c9c39bb7b650aac430c24 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 4 Mar 2013 13:22:21 +0100 Subject: [PATCH 0927/1752] SLAVEOF command refactored into a proper API. We now have replicationSetMaster() and replicationUnsetMaster() that can be called in other contexts (for instance Redis Cluster). --- src/redis.h | 2 ++ src/replication.c | 41 ++++++++++++++++++++++++++--------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/src/redis.h b/src/redis.h index 7ef79cc6888..68b91c51a29 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1025,6 +1025,8 @@ void replicationScriptCacheInit(void); void replicationScriptCacheFlush(void); void replicationScriptCacheAdd(sds sha1); int replicationScriptCacheExists(sds sha1); +void replicationSetMaster(char *ip, int port); +void replicationUnsetMaster(void); /* Generic persistence functions */ void startLoading(FILE *fp); diff --git a/src/replication.c b/src/replication.c index 2c99a6624d0..9a96d80aa8d 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1205,16 +1205,35 @@ int cancelReplicationHandshake(void) { return 1; } +/* Set replication to the specified master address and port. */ +void replicationSetMaster(char *ip, int port) { + sdsfree(server.masterhost); + server.masterhost = sdsdup(ip); + server.masterport = port; + if (server.master) freeClient(server.master); + disconnectSlaves(); /* Force our slaves to resync with us as well. */ + replicationDiscardCachedMaster(); /* Don't try a PSYNC. */ + freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */ + cancelReplicationHandshake(); + server.repl_state = REDIS_REPL_CONNECT; +} + +/* Cancel replication, setting the instance as a master itself. */ +void replicationUnsetMaster(void) { + if (server.masterhost == NULL) return; /* Nothing to do. */ + sdsfree(server.masterhost); + server.masterhost = NULL; + if (server.master) freeClient(server.master); + replicationDiscardCachedMaster(); + cancelReplicationHandshake(); + server.repl_state = REDIS_REPL_NONE; +} + void slaveofCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"no") && !strcasecmp(c->argv[2]->ptr,"one")) { if (server.masterhost) { - sdsfree(server.masterhost); - server.masterhost = NULL; - if (server.master) freeClient(server.master); - replicationDiscardCachedMaster(); - cancelReplicationHandshake(); - server.repl_state = REDIS_REPL_NONE; + replicationUnsetMaster(); redisLog(REDIS_NOTICE,"MASTER MODE enabled (user request)"); } } else { @@ -1232,15 +1251,7 @@ void slaveofCommand(redisClient *c) { } /* There was no previous master or the user specified a different one, * we can continue. */ - sdsfree(server.masterhost); - server.masterhost = sdsdup(c->argv[1]->ptr); - server.masterport = port; - if (server.master) freeClient(server.master); - disconnectSlaves(); /* Force our slaves to resync with us as well. */ - replicationDiscardCachedMaster(); /* Don't try a PSYNC. */ - freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */ - cancelReplicationHandshake(); - server.repl_state = REDIS_REPL_CONNECT; + replicationSetMaster(c->argv[1]->ptr, port); redisLog(REDIS_NOTICE,"SLAVE OF %s:%d enabled (user request)", server.masterhost, server.masterport); } From 83333b08d0d423f7caf5c2155cc3464e652645d9 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 28 Nov 2013 16:25:49 +0100 Subject: [PATCH 0928/1752] Stop writes on MISCONF only if instance is a master. From the point of view of the slave not accepting writes from the master can only create a bigger consistency issue. --- src/redis.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/redis.c b/src/redis.c index dd2a0df0f88..42f957b35c4 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1918,10 +1918,12 @@ int processCommand(redisClient *c) { } } - /* Don't accept write commands if there are problems persisting on disk. */ + /* Don't accept write commands if there are problems persisting on disk + * and if this is a master instance. */ if (server.stop_writes_on_bgsave_err && server.saveparamslen > 0 && server.lastbgsave_status == REDIS_ERR && + server.masterhost != NULL && (c->cmd->flags & REDIS_CMD_WRITE || c->cmd->proc == pingCommand)) { @@ -1931,7 +1933,7 @@ int processCommand(redisClient *c) { } /* Don't accept write commands if there are not enough good slaves and - * used configured the min-slaves-to-write option. */ + * user configured the min-slaves-to-write option. */ if (server.repl_min_slaves_to_write && server.repl_min_slaves_max_lag && c->cmd->flags & REDIS_CMD_WRITE && From 75347ada7f431925b97b037b56b5e3801e3fd16d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Dec 2013 12:22:17 +0100 Subject: [PATCH 0929/1752] Sentinel: better time desynchronization. Sentinels are now desynchronized in a better way changing the time handler frequency between 10 and 20 HZ. This way on average a desynchronization of 25 milliesconds is produced that should be larger enough compared to network latency, avoiding most split-brain condition during the vote. Now that the clocks are desynchronized, to have larger random delays when performing operations can be easily achieved in the following way. Take as example the function that starts the failover, that is called with a frequency between 10 and 20 HZ and will start the failover every time there are the conditions. By just adding as an additional condition something like rand()%4 == 0, we can amplify the desynchronization between Sentinel instances easily. See issue #1419. --- src/sentinel.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 4a6dbeada0b..77786d71548 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2735,12 +2735,9 @@ char *sentinelVoteLeader(sentinelRedisInstance *master, uint64_t req_epoch, char master->leader, (unsigned long long) master->leader_epoch); /* If we did not voted for ourselves, set the master failover start * time to now, in order to force a delay before we can start a - * failover for the same master. - * - * The random addition is useful to desynchronize a bit the slaves - * and reduce the chance that no slave gets majority. */ + * failover for the same master. */ if (strcasecmp(master->leader,server.runid)) - master->failover_start_time = mstime() + rand() % 2000; + master->failover_start_time = mstime(); } *leader_epoch = master->leader_epoch; @@ -3391,5 +3388,13 @@ void sentinelTimer(void) { sentinelRunPendingScripts(); sentinelCollectTerminatedScripts(); sentinelKillTimedoutScripts(); + + /* We continuously change the frequency of the Redis "timer interrupt" + * in order to desynchronize every Sentinel from every other. + * This non-determinism avoids that Sentinels started at the same time + * exactly continue to stay synchronized asking to be voted at the + * same time again and again (resulting in nobody likely winning the + * election because of split brain voting). */ + server.hz = REDIS_DEFAULT_HZ + rand() % REDIS_DEFAULT_HZ; } From dd0ac4ac72828cf3738632d2d53ed1948562bf9d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Dec 2013 15:55:19 +0100 Subject: [PATCH 0930/1752] Sentinel: don't write HZ when flushing config. See issue #1419. --- src/sentinel.c | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 77786d71548..4405e88da59 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1474,15 +1474,19 @@ void rewriteConfigSentinelOption(struct rewriteConfigState *state) { * On failure the function logs a warning on the Redis log. */ void sentinelFlushConfig(void) { int fd; - - if (rewriteConfig(server.configfile) == -1) { + int saved_hz = server.hz; + + server.hz = REDIS_DEFAULT_HZ; + if (rewriteConfig(server.configfile) != -1) { + /* Rewrite succeded, fsync it. */ + if ((fd = open(server.configfile,O_RDONLY)) != -1) { + fsync(fd); + close(fd); + } + } else { redisLog(REDIS_WARNING,"WARNING: Senitnel was not able to save the new configuration on disk!!!: %s", strerror(errno)); - return; - } - if ((fd = open(server.configfile,O_RDONLY)) != -1) { - fsync(fd); - close(fd); } + server.hz = saved_hz; return; } From 4d650a3baa614364043db8947f73161ebf11eafb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 2 Dec 2013 16:07:46 +0100 Subject: [PATCH 0931/1752] Redis 2.8.2. --- 00-RELEASENOTES | 9 +++++++++ src/version.h | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index 9fcb663d8b0..f50cda76545 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,15 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8.2 ] Release date: 2 Dec 2013 + +# UPGRADE URGENCY: MODERATE for both Redis and Sentinel. + +* [FIX] Sentinel better desynchronization to avoid split-brain elections + where no Sentinel managed to get elected. +* [FIX] Stop accepting writes on "MISCONF" error only if master, not slave. +* [FIX] Reply to PING with an error on "MISCONF" errors. + --[ Redis 2.8.1 ] Release date: 25 Nov 2013 # UPGRADE URGENCY: LOW for Redis, CRITICAL for Senitnel. You don't need to diff --git a/src/version.h b/src/version.h index 7afbdcb3cde..d678aa8d836 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.8.1" +#define REDIS_VERSION "2.8.2" From 333453646c3b6e3b20af0223a68c04914a479f39 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Dec 2013 13:40:41 +0100 Subject: [PATCH 0932/1752] Grammar fix in freeClient(). --- src/networking.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 4233dd306be..8be62b54a4d 100644 --- a/src/networking.c +++ b/src/networking.c @@ -685,7 +685,7 @@ void freeClient(redisClient *c) { listDelNode(server.clients,ln); } /* When client was just unblocked because of a blocking operation, - * remove it from the list with unblocked clients. */ + * remove it from the list of unblocked clients. */ if (c->flags & REDIS_UNBLOCKED) { ln = listSearchKey(server.unblocked_clients,c); redisAssert(ln != NULL); From 3d7263aa41d204a4c1cdeac73a672e26a909372a Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 3 Dec 2013 13:54:06 +0100 Subject: [PATCH 0933/1752] Removed old comments and dead code from freeClient(). --- src/networking.c | 26 ++++++++++++++------------ src/redis.h | 2 -- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/networking.c b/src/networking.c index 8be62b54a4d..5ee0b288d00 100644 --- a/src/networking.c +++ b/src/networking.c @@ -100,9 +100,7 @@ redisClient *createClient(int fd) { c->bpop.keys = dictCreate(&setDictType,NULL); c->bpop.timeout = 0; c->bpop.target = NULL; - c->io_keys = listCreate(); c->watched_keys = listCreate(); - listSetFreeMethod(c->io_keys,decrRefCountVoid); c->pubsub_channels = dictCreate(&setDictType,NULL); c->pubsub_patterns = listCreate(); listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); @@ -650,13 +648,11 @@ void freeClient(redisClient *c) { return; } - /* Note that if the client we are freeing is blocked into a blocking - * call, we have to set querybuf to NULL *before* to call - * unblockClientWaitingData() to avoid processInputBuffer() will get - * called. Also it is important to remove the file events after - * this, because this call adds the READABLE event. */ + /* Free the query buffer */ sdsfree(c->querybuf); c->querybuf = NULL; + + /* Deallocate structures used to block on blocking ops. */ if (c->flags & REDIS_BLOCKED) unblockClientWaitingData(c); dictRelease(c->bpop.keys); @@ -664,11 +660,13 @@ void freeClient(redisClient *c) { /* UNWATCH all the keys */ unwatchAllKeys(c); listRelease(c->watched_keys); + /* Unsubscribe from all the pubsub channels */ pubsubUnsubscribeAllChannels(c,0); pubsubUnsubscribeAllPatterns(c,0); dictRelease(c->pubsub_channels); listRelease(c->pubsub_patterns); + /* Close socket, unregister events, and remove list of replies and * accumulated arguments. */ if (c->fd != -1) { @@ -678,12 +676,14 @@ void freeClient(redisClient *c) { } listRelease(c->reply); freeClientArgv(c); + /* Remove from the list of clients */ if (c->fd != -1) { ln = listSearchKey(server.clients,c); redisAssert(ln != NULL); listDelNode(server.clients,ln); } + /* When client was just unblocked because of a blocking operation, * remove it from the list of unblocked clients. */ if (c->flags & REDIS_UNBLOCKED) { @@ -691,9 +691,9 @@ void freeClient(redisClient *c) { redisAssert(ln != NULL); listDelNode(server.unblocked_clients,ln); } - listRelease(c->io_keys); - /* Master/slave cleanup. - * Case 1: we lost the connection with a slave. */ + + /* Master/slave cleanup Case 1: + * we lost the connection with a slave. */ if (c->flags & REDIS_SLAVE) { if (c->replstate == REDIS_REPL_SEND_BULK && c->repldbfd != -1) close(c->repldbfd); @@ -709,7 +709,8 @@ void freeClient(redisClient *c) { refreshGoodSlavesCount(); } - /* Case 2: we lost the connection with the master. */ + /* Master/slave cleanup Case 2: + * we lost the connection with the master. */ if (c->flags & REDIS_MASTER) replicationHandleMasterDisconnection(); /* If this client was scheduled for async freeing we need to remove it @@ -720,7 +721,8 @@ void freeClient(redisClient *c) { listDelNode(server.clients_to_close,ln); } - /* Release memory */ + /* Release other dynamically allocated client structure fields, + * and finally release the client structure itself. */ if (c->name) decrRefCount(c->name); zfree(c->argv); freeClientMultiState(c); diff --git a/src/redis.h b/src/redis.h index 68b91c51a29..35714590200 100644 --- a/src/redis.h +++ b/src/redis.h @@ -475,8 +475,6 @@ typedef struct redisClient { int slave_listening_port; /* As configured with: SLAVECONF listening-port */ multiState mstate; /* MULTI/EXEC state */ blockingState bpop; /* blocking state */ - list *io_keys; /* Keys this client is waiting to be loaded from the - * swap file in order to continue. */ list *watched_keys; /* Keys WATCHED for MULTI/EXEC CAS */ dict *pubsub_channels; /* channels a client is interested in (SUBSCRIBE) */ list *pubsub_patterns; /* patterns a client is interested in (SUBSCRIBE) */ From edae78999c0439eedacb2b3a89bfb8fefa977760 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Dec 2013 16:28:35 +0100 Subject: [PATCH 0934/1752] Fixed typos in redis.conf file. --- redis.conf | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/redis.conf b/redis.conf index 2e3239c8027..f556cc375fd 100644 --- a/redis.conf +++ b/redis.conf @@ -34,7 +34,7 @@ port 6379 # bind 192.168.1.100 10.0.0.1 # bind 127.0.0.1 -# Specify the path for the unix socket that will be used to listen for +# Specify the path for the Unix socket that will be used to listen for # incoming connections. There is no default, so Redis will not listen # on a unix socket when not specified. # @@ -68,7 +68,7 @@ tcp-keepalive 0 # warning (only very important / critical messages are logged) loglevel notice -# Specify the log file name. Also the emptry string can be used to force +# Specify the log file name. Also the empty string can be used to force # Redis to log on the standard output. Note that if you use standard # output for logging but daemonize, logs will be sent to /dev/null logfile "" @@ -116,9 +116,9 @@ save 60 10000 # By default Redis will stop accepting writes if RDB snapshots are enabled # (at least one save point) and the latest background save failed. -# This will make the user aware (in an hard way) that data is not persisting +# This will make the user aware (in a hard way) that data is not persisting # on disk properly, otherwise chances are that no one will notice and some -# distater will happen. +# disaster will happen. # # If the background saving process will start working again Redis will # automatically allow writes again. @@ -263,7 +263,7 @@ repl-disable-tcp-nodelay no # # A slave with a low priority number is considered better for promotion, so # for instance if there are three slaves with priority 10, 100, 25 Sentinel will -# pick the one wtih priority 10, that is the lowest. +# pick the one with priority 10, that is the lowest. # # However a special priority of 0 marks the slave as not able to perform the # role of master, so a slave with priority of 0 will never be selected by @@ -351,7 +351,7 @@ slave-priority 100 # to reply to read-only commands like GET. # # This option is usually useful when using Redis as an LRU cache, or to set -# an hard memory limit for an instance (using the 'noeviction' policy). +# a hard memory limit for an instance (using the 'noeviction' policy). # # WARNING: If you have slaves attached to an instance with maxmemory on, # the size of the output buffers needed to feed the slaves are subtracted @@ -607,7 +607,7 @@ zset-max-ziplist-value 64 # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in # order to help rehashing the main Redis hash table (the one mapping top-level # keys to values). The hash table implementation Redis uses (see dict.c) -# performs a lazy rehashing: the more operation you run into an hash table +# performs a lazy rehashing: the more operation you run into a hash table # that is rehashing, the more rehashing "steps" are performed, so if the # server is idle the rehashing is never complete and some more memory is used # by the hash table. @@ -662,7 +662,7 @@ client-output-buffer-limit slave 256mb 64mb 60 client-output-buffer-limit pubsub 32mb 8mb 60 # Redis calls an internal function to perform many background tasks, like -# closing connections of clients in timeot, purging expired keys that are +# closing connections of clients in timeout, purging expired keys that are # never requested, and so forth. # # Not all tasks are performed with the same frequency, but Redis checks for From 7f6743a5812cd8c70e95fe17e0f531de6afac2fe Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 5 Dec 2013 16:35:32 +0100 Subject: [PATCH 0935/1752] Fixed grammar: before H the article is a, not an. --- src/ae_epoll.c | 2 +- src/aof.c | 2 +- src/db.c | 10 +++++----- src/dict.c | 4 ++-- src/networking.c | 2 +- src/object.c | 2 +- src/rdb.c | 2 +- src/redis-check-dump.c | 2 +- src/redis.c | 2 +- src/replication.c | 2 +- src/scripting.c | 4 ++-- src/sds.c | 2 +- src/sentinel.c | 6 +++--- src/sort.c | 4 ++-- src/t_list.c | 4 ++-- src/t_zset.c | 2 +- src/zipmap.c | 2 +- src/zmalloc.c | 2 +- 18 files changed, 28 insertions(+), 28 deletions(-) diff --git a/src/ae_epoll.c b/src/ae_epoll.c index 41af3e8749f..da9c7b90672 100644 --- a/src/ae_epoll.c +++ b/src/ae_epoll.c @@ -45,7 +45,7 @@ static int aeApiCreate(aeEventLoop *eventLoop) { zfree(state); return -1; } - state->epfd = epoll_create(1024); /* 1024 is just an hint for the kernel */ + state->epfd = epoll_create(1024); /* 1024 is just a hint for the kernel */ if (state->epfd == -1) { zfree(state->events); zfree(state); diff --git a/src/aof.c b/src/aof.c index 6fb969fa823..5bec0ac147f 100644 --- a/src/aof.c +++ b/src/aof.c @@ -771,7 +771,7 @@ int rewriteSortedSetObject(rio *r, robj *key, robj *o) { return 1; } -/* Write either the key or the value of the currently selected item of an hash. +/* Write either the key or the value of the currently selected item of a hash. * The 'hi' argument passes a valid Redis hash iterator. * The 'what' filed specifies if to write a key or a value and can be * either REDIS_HASH_KEY or REDIS_HASH_VALUE. diff --git a/src/db.c b/src/db.c index 117ebb7bce6..b91e4f86c67 100644 --- a/src/db.c +++ b/src/db.c @@ -360,7 +360,7 @@ int parseScanCursorOrReply(redisClient *c, robj *o, unsigned long *cursor) { } /* This command implements SCAN, HSCAN and SSCAN commands. - * If object 'o' is passed, then it must be an Hash or Set object, otherwise + * If object 'o' is passed, then it must be a Hash or Set object, otherwise * if 'o' is NULL the command will operate on the dictionary associated with * the current database. * @@ -368,7 +368,7 @@ int parseScanCursorOrReply(redisClient *c, robj *o, unsigned long *cursor) { * the client arguments vector is a key so it skips it before iterating * in order to parse options. * - * In the case of an Hash object the function returns both the field and value + * In the case of a Hash object the function returns both the field and value * of every element on the Hash. */ void scanGenericCommand(redisClient *c, robj *o, unsigned long cursor) { int rv; @@ -423,12 +423,12 @@ void scanGenericCommand(redisClient *c, robj *o, unsigned long cursor) { /* Step 2: Iterate the collection. * * Note that if the object is encoded with a ziplist, intset, or any other - * representation that is not an hash table, we are sure that it is also + * representation that is not a hash table, we are sure that it is also * composed of a small number of elements. So to avoid taking state we * just return everything inside the object in a single call, setting the * cursor to zero to signal the end of the iteration. */ - /* Handle the case of an hash table. */ + /* Handle the case of a hash table. */ ht = NULL; if (o == NULL) { ht = c->db->dict; @@ -510,7 +510,7 @@ void scanGenericCommand(redisClient *c, robj *o, unsigned long cursor) { listDelNode(keys, node); } - /* If this is an hash or a sorted set, we have a flat list of + /* If this is a hash or a sorted set, we have a flat list of * key-value elements, so if this element was filtered, remove the * value, or skip it if it was not filtered: we only match keys. */ if (o && (o->type == REDIS_ZSET || o->type == REDIS_HASH)) { diff --git a/src/dict.c b/src/dict.c index 54f87773a7c..4d251479966 100644 --- a/src/dict.c +++ b/src/dict.c @@ -53,7 +53,7 @@ * around when there is a child performing saving operations. * * Note that even when dict_can_resize is set to 0, not all resizes are - * prevented: an hash table is still allowed to grow if the ratio between + * prevented: a hash table is still allowed to grow if the ratio between * the number of elements and the buckets > dict_force_resize_ratio. */ static int dict_can_resize = 1; static unsigned int dict_force_resize_ratio = 5; @@ -853,7 +853,7 @@ static unsigned long _dictNextPower(unsigned long size) } /* Returns the index of a free slot that can be populated with - * an hash entry for the given 'key'. + * a hash entry for the given 'key'. * If the key already exists, -1 is returned. * * Note that if we are in the process of rehashing the hash table, the diff --git a/src/networking.c b/src/networking.c index 5ee0b288d00..c77ff5dbd41 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1150,7 +1150,7 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, *biggest_input_buffer = bib; } -/* This is an helper function for getClientPeerId(). +/* This is a helper function for getClientPeerId(). * It writes the specified ip/port to "peerid" as a null termiated string * in the form ip:port if ip does not contain ":" itself, otherwise * [ip]:port format is used (for IPv6 addresses basically). */ diff --git a/src/object.c b/src/object.c index 6095a11540b..6f6dcaf83de 100644 --- a/src/object.c +++ b/src/object.c @@ -587,7 +587,7 @@ unsigned long estimateObjectIdleTime(robj *o) { } } -/* This is an helper function for the DEBUG command. We need to lookup keys +/* This is a helper function for the DEBUG command. We need to lookup keys * without any modification of LRU or other parameters. */ robj *objectCommandLookup(redisClient *c, robj *key) { dictEntry *de; diff --git a/src/rdb.c b/src/rdb.c index f7dcc171945..d8716f26042 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -892,7 +892,7 @@ robj *rdbLoadObject(int rdbtype, rio *rdb) { o = createHashObject(); - /* Too many entries? Use an hash table. */ + /* Too many entries? Use a hash table. */ if (len > server.hash_max_ziplist_entries) hashTypeConvert(o, REDIS_ENCODING_HT); diff --git a/src/redis-check-dump.c b/src/redis-check-dump.c index d0952778156..4b0c1300fda 100644 --- a/src/redis-check-dump.c +++ b/src/redis-check-dump.c @@ -60,7 +60,7 @@ #define REDIS_ENCODING_RAW 0 /* Raw representation */ #define REDIS_ENCODING_INT 1 /* Encoded as integer */ #define REDIS_ENCODING_ZIPMAP 2 /* Encoded as zipmap */ -#define REDIS_ENCODING_HT 3 /* Encoded as an hash table */ +#define REDIS_ENCODING_HT 3 /* Encoded as a hash table */ /* Object types only used for dumping to disk */ #define REDIS_EXPIRETIME_MS 252 diff --git a/src/redis.c b/src/redis.c index 42f957b35c4..a8ece8197d7 100644 --- a/src/redis.c +++ b/src/redis.c @@ -370,7 +370,7 @@ void exitFromChild(int retcode) { /*====================== Hash table type implementation ==================== */ -/* This is an hash table type that uses the SDS dynamic strings library as +/* This is a hash table type that uses the SDS dynamic strings library as * keys and radis objects as values (objects can hold SDS strings, * lists, sets). */ diff --git a/src/replication.c b/src/replication.c index 9a96d80aa8d..8ee1ce7c2c3 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1399,7 +1399,7 @@ void refreshGoodSlavesCount(void) { * connected slave, in order to be able to replicate EVALSHA as it is without * translating it to EVAL every time it is possible. * - * We use a capped collection implemented by an hash table for fast lookup + * We use a capped collection implemented by a hash table for fast lookup * of scripts we can send as EVALSHA, plus a linked list that is used for * eviction of the oldest entry when the max number of items is reached. * diff --git a/src/scripting.c b/src/scripting.c index ac1a913f035..e2ce607652a 100644 --- a/src/scripting.c +++ b/src/scripting.c @@ -895,7 +895,7 @@ void evalGenericCommand(redisClient *c, int evalsha) { /* Select the right DB in the context of the Lua client */ selectDb(server.lua_client,c->db->id); - /* Set an hook in order to be able to stop the script execution if it + /* Set a hook in order to be able to stop the script execution if it * is running for too much time. * We set the hook only if the time limit is enabled as the hook will * make the Lua script execution slower. */ @@ -1059,7 +1059,7 @@ void scriptCommand(redisClient *c) { if (server.lua_caller == NULL) { addReplySds(c,sdsnew("-NOTBUSY No scripts in execution right now.\r\n")); } else if (server.lua_write_dirty) { - addReplySds(c,sdsnew("-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in an hard way using the SHUTDOWN NOSAVE command.\r\n")); + addReplySds(c,sdsnew("-UNKILLABLE Sorry the script already executed write commands against the dataset. You can either wait the script termination or kill the server in a hard way using the SHUTDOWN NOSAVE command.\r\n")); } else { server.lua_kill = 1; addReply(c,shared.ok); diff --git a/src/sds.c b/src/sds.c index d66c1d7301e..338b4262a3e 100644 --- a/src/sds.c +++ b/src/sds.c @@ -582,7 +582,7 @@ int is_hex_digit(char c) { (c >= 'A' && c <= 'F'); } -/* Helper function for sdssplitargs() that converts an hex digit into an +/* Helper function for sdssplitargs() that converts a hex digit into an * integer from 0 to 15 */ int hex_digit_to_int(char c) { switch(c) { diff --git a/src/sentinel.c b/src/sentinel.c index 4405e88da59..95048f427f3 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -135,7 +135,7 @@ typedef struct sentinelRedisInstance { if the link is idle and must be reconnected. */ mstime_t last_pub_time; /* Last time we sent hello via Pub/Sub. */ mstime_t last_hello_time; /* Only used if SRI_SENTINEL is set. Last time - we received an hello from this Sentinel + we received a hello from this Sentinel via Pub/Sub. */ mstime_t last_master_down_reply_time; /* Time of last reply to SENTINEL is-master-down command. */ @@ -1492,7 +1492,7 @@ void sentinelFlushConfig(void) { /* ====================== hiredis connection handling ======================= */ -/* Completely disconnect an hiredis link from an instance. */ +/* Completely disconnect a hiredis link from an instance. */ void sentinelKillLink(sentinelRedisInstance *ri, redisAsyncContext *c) { if (ri->cc == c) { ri->cc = NULL; @@ -1504,7 +1504,7 @@ void sentinelKillLink(sentinelRedisInstance *ri, redisAsyncContext *c) { redisAsyncFree(c); } -/* This function takes an hiredis context that is in an error condition +/* This function takes a hiredis context that is in an error condition * and make sure to mark the instance as disconnected performing the * cleanup needed. * diff --git a/src/sort.c b/src/sort.c index a4b062645fa..3e195a9352b 100644 --- a/src/sort.c +++ b/src/sort.c @@ -48,8 +48,8 @@ redisSortOperation *createSortOperation(int type, robj *pattern) { * 1) The first occurrence of '*' in 'pattern' is substituted with 'subst'. * * 2) If 'pattern' matches the "->" string, everything on the left of - * the arrow is treated as the name of an hash field, and the part on the - * left as the key name containing an hash. The value of the specified + * the arrow is treated as the name of a hash field, and the part on the + * left as the key name containing a hash. The value of the specified * field is returned. * * 3) If 'pattern' equals "#", the function simply returns 'subst' itself so diff --git a/src/t_list.c b/src/t_list.c index 0413dc69b4f..3790628d3f1 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -850,7 +850,7 @@ void unblockClientWaitingData(redisClient *c) { /* If the specified key has clients blocked waiting for list pushes, this * function will put the key reference into the server.ready_keys list. - * Note that db->ready_keys is an hash table that allows us to avoid putting + * Note that db->ready_keys is a hash table that allows us to avoid putting * the same key again and again in the list in case of multiple pushes * made by a script or in the context of MULTI/EXEC. * @@ -878,7 +878,7 @@ void signalListAsReady(redisClient *c, robj *key) { redisAssert(dictAdd(c->db->ready_keys,key,NULL) == DICT_OK); } -/* This is an helper function for handleClientsBlockedOnLists(). It's work +/* This is a helper function for handleClientsBlockedOnLists(). It's work * is to serve a specific client (receiver) that is blocked on 'key' * in the context of the specified 'db', doing the following: * diff --git a/src/t_zset.c b/src/t_zset.c index cbc4abd9bda..d653c110124 100644 --- a/src/t_zset.c +++ b/src/t_zset.c @@ -36,7 +36,7 @@ * in order to get O(log(N)) INSERT and REMOVE operations into a sorted * data structure. * - * The elements are added to an hash table mapping Redis objects to scores. + * The elements are added to a hash table mapping Redis objects to scores. * At the same time the elements are added to a skip list mapping scores * to Redis objects (so objects are sorted by scores in this "view"). */ diff --git a/src/zipmap.c b/src/zipmap.c index 3cd56e5ffb0..140126a7108 100644 --- a/src/zipmap.c +++ b/src/zipmap.c @@ -4,7 +4,7 @@ * efficient. * * The Redis Hash type uses this data structure for hashes composed of a small - * number of elements, to switch to an hash table once a given number of + * number of elements, to switch to a hash table once a given number of * elements is reached. * * Given that many times Redis Hashes are used to represent objects composed diff --git a/src/zmalloc.c b/src/zmalloc.c index 21042582874..e7e97aa67ca 100644 --- a/src/zmalloc.c +++ b/src/zmalloc.c @@ -176,7 +176,7 @@ void *zrealloc(void *ptr, size_t size) { } /* Provide zmalloc_size() for systems where this function is not provided by - * malloc itself, given that in that case we store an header with this + * malloc itself, given that in that case we store a header with this * information as the first bytes of every allocation. */ #ifndef HAVE_MALLOC_SIZE size_t zmalloc_size(void *ptr) { From 4c53178c6c68ade4bcd5cc42d939408ab7246e45 Mon Sep 17 00:00:00 2001 From: Anurag Ramdasan Date: Thu, 5 Dec 2013 21:09:31 +0530 Subject: [PATCH 0936/1752] Fixed grammar: 'usually' to 'usual' --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index f556cc375fd..874861af42a 100644 --- a/redis.conf +++ b/redis.conf @@ -125,7 +125,7 @@ save 60 10000 # # However if you have setup your proper monitoring of the Redis server # and persistence, you may want to disable this feature so that Redis will -# continue to work as usually even if there are problems with disk, +# continue to work as usual even if there are problems with disk, # permissions, and so forth. stop-writes-on-bgsave-error yes From b67f39da0911d29931b76ad716aee26096818d20 Mon Sep 17 00:00:00 2001 From: Anurag Ramdasan Date: Thu, 5 Dec 2013 21:47:17 +0530 Subject: [PATCH 0937/1752] fixed typo --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 874861af42a..ec9473cd4a9 100644 --- a/redis.conf +++ b/redis.conf @@ -197,7 +197,7 @@ slave-serve-stale-data yes # Note: read only slaves are not designed to be exposed to untrusted clients # on the internet. It's just a protection layer against misuse of the instance. # Still a read only slave exports by default all the administrative commands -# such as CONFIG, DEBUG, and so forth. To a limited extend you can improve +# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve # security of read only slaves using 'rename-command' to shadow all the # administrative / dangerous commands. slave-read-only yes From 4f9d30b33bd2e986ea9368aba6e202894a93aca6 Mon Sep 17 00:00:00 2001 From: Anurag Ramdasan Date: Thu, 5 Dec 2013 23:15:47 +0530 Subject: [PATCH 0938/1752] Grammar fix. --- CONTRIBUTING | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING b/CONTRIBUTING index 06e1e9660c0..f7b6836f701 100644 --- a/CONTRIBUTING +++ b/CONTRIBUTING @@ -22,7 +22,7 @@ each source file that you contribute. 1. Drop a message to the Redis Google Group with a proposal of semantics/API. -2. If in steps 1 you get an acknowledge from the project leaders, use the +2. If in step 1 you get an acknowledge from the project leaders, use the following procedure to submit a patch: a. Fork Redis on github ( http://help.github.com/fork-a-repo/ ) From bf307cfb4d55d38225cc88c06a2b7cc69b34f4cc Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Dec 2013 10:48:40 +0100 Subject: [PATCH 0939/1752] Fixed typo in redis.conf. --- redis.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index ec9473cd4a9..6786e8dfa58 100644 --- a/redis.conf +++ b/redis.conf @@ -343,7 +343,7 @@ slave-priority 100 # Don't use more memory than the specified amount of bytes. # When the memory limit is reached Redis will try to remove keys -# accordingly to the eviction policy selected (see maxmemmory-policy). +# according to the eviction policy selected (see maxmemory-policy). # # If Redis can't remove keys according to the policy, or if the policy is # set to 'noeviction', Redis will start to reply with errors to commands From dceaca1f699ef33a05905ec9c545cbc059143463 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Dec 2013 11:37:46 +0100 Subject: [PATCH 0940/1752] Sentinel: fix reported role fields when master is reset. When there is a master address switch, the reported role must be set to master so that we have a chance to re-sample the INFO output to check if the new address is reporting the right role. Otherwise if the role was wrong, it will be sensed as wrong even after the address switch, and for enough time according to the role change time, for Sentinel consider the master SDOWN. This fixes isue #1446, that describes the effects of this bug in practice. --- src/sentinel.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 95048f427f3..63df3fb6daa 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1133,6 +1133,8 @@ void sentinelResetMaster(sentinelRedisInstance *ri, int flags) { ri->slave_master_host = NULL; ri->last_avail_time = mstime(); ri->last_pong_time = mstime(); + ri->role_reported_time = mstime(); + ri->role_reported = SRI_MASTER; if (flags & SENTINEL_GENERATE_EVENT) sentinelEvent(REDIS_WARNING,"+reset-master",ri,"%@"); } From fba0b23e724e86ef6b98f07594cec3b45b8a9ab9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 6 Dec 2013 12:46:56 +0100 Subject: [PATCH 0941/1752] Sentinel: fix reported role info sampling. The way the role change was recoded was not sane and too much convoluted, causing the role information to be not always updated. This commit fixes issue #1445. --- src/sentinel.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 63df3fb6daa..909f7e8ead5 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1764,22 +1764,22 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { * Some things will not happen if sentinel.tilt is true, but some will * still be processed. */ + /* Remember when the role changed. */ + if (role != ri->role_reported) { + ri->role_reported_time = mstime(); + ri->role_reported = role; + if (role == SRI_SLAVE) ri->slave_conf_change_time = mstime(); + } + /* Handle master -> slave role switch. */ if ((ri->flags & SRI_MASTER) && role == SRI_SLAVE) { - if (ri->role_reported != SRI_SLAVE) { - ri->role_reported_time = mstime(); - ri->role_reported = SRI_SLAVE; - ri->slave_conf_change_time = mstime(); - } + /* Nothing to do, but masters claiming to be slaves are + * considered to be unreachable by Sentinel, so eventually + * a failover will be triggered. */ } /* Handle slave -> master role switch. */ if ((ri->flags & SRI_SLAVE) && role == SRI_MASTER) { - if (ri->role_reported != SRI_MASTER) { - ri->role_reported_time = mstime(); - ri->role_reported = SRI_MASTER; - } - /* If this is a promoted slave we can change state to the * failover state machine. */ if (!sentinel.tilt && From 8d0083ba250fead6a425a08246744e4b796d49fb Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Dec 2013 13:28:39 +0100 Subject: [PATCH 0942/1752] Handle inline requested terminated with just \n. --- src/networking.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index c77ff5dbd41..ac3c2f0b9d7 100644 --- a/src/networking.c +++ b/src/networking.c @@ -842,11 +842,14 @@ void resetClient(redisClient *c) { } int processInlineBuffer(redisClient *c) { - char *newline = strstr(c->querybuf,"\r\n"); + char *newline; int argc, j; sds *argv, aux; size_t querylen; + /* Search for end of line */ + newline = strchr(c->querybuf,'\n'); + /* Nothing to do without a \r\n */ if (newline == NULL) { if (sdslen(c->querybuf) > REDIS_INLINE_MAX_SIZE) { @@ -856,6 +859,10 @@ int processInlineBuffer(redisClient *c) { return REDIS_ERR; } + /* Handle the \r\n case. */ + if (newline && newline != c->querybuf && *(newline-1) == '\r') + newline--; + /* Split the input buffer up to the \r\n */ querylen = newline-(c->querybuf); aux = sdsnewlen(c->querybuf,querylen); From 75bf5a4a4a0d06eeaacb1acb79cd6b03e51926fd Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 9 Dec 2013 13:32:44 +0100 Subject: [PATCH 0943/1752] Slaves heartbeat while loading RDB files. Starting with Redis 2.8 masters are able to detect timed out slaves, while before 2.8 only slaves were able to detect a timed out master. Now that timeout detection is bi-directional the following problem happens as described "in the field" by issue #1449: 1) Master and slave setup with big dataset. 2) Slave performs the first synchronization, or a full sync after a failed partial resync. 3) Master sends the RDB payload to the slave. 4) Slave loads this payload. 5) Master detects the slave as timed out since does not receive back the REPLCONF ACK acknowledges. Here the problem is that the master has no way to know how much the slave will take to load the RDB file in memory. The obvious solution is to use a greater replication timeout setting, but this is a shame since for the 0.1% of operation time we are forced to use a timeout that is not what is suited for 99.9% of operation time. This commit tries to fix this problem with a solution that is a bit of an hack, but that modifies little of the replication internals, in order to be back ported to 2.8 safely. During the RDB loading time, we send the master newlines to avoid being sensed as timed out. This is the same that the master already does while saving the RDB file to still signal its presence to the slave. The single newline is used because: 1) It can't desync the protocol, as it is only transmitted all or nothing. 2) It can be safely sent while we don't have a client structure for the master or in similar situations just with write(2). --- src/networking.c | 6 ++++++ src/rdb.c | 13 ++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index ac3c2f0b9d7..476af38f939 100644 --- a/src/networking.c +++ b/src/networking.c @@ -869,6 +869,12 @@ int processInlineBuffer(redisClient *c) { argv = sdssplitargs(aux,&argc); sdsfree(aux); + /* Newline from slaves can be used to refresh the last ACK time. + * This is useful for a slave to ping back while loading a big + * RDB file. */ + if (querylen == 0 && c->flags & REDIS_SLAVE) + c->repl_ack_time = server.unixtime; + /* Leave data after the first line of the query in the buffer */ sdsrange(c->querybuf,querylen+2,-1); diff --git a/src/rdb.c b/src/rdb.c index d8716f26042..ff0ddd58d54 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1064,7 +1064,18 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { if (server.rdb_checksum) rioGenericUpdateChecksum(r, buf, len); if (server.loading_process_events_interval_bytes && - (r->processed_bytes + len)/server.loading_process_events_interval_bytes > r->processed_bytes/server.loading_process_events_interval_bytes) { + (r->processed_bytes + len)/server.loading_process_events_interval_bytes > r->processed_bytes/server.loading_process_events_interval_bytes) + { + if (server.masterhost && server.repl_state == REDIS_REPL_TRANSFER) { + /* Avoid the master to detect the slave is timing out while + * loading the RDB file in initial synchronization. We send + * a single newline character that is valid protocol but is + * guaranteed to either be sent entierly or not, since the byte + * is indivisible. */ + if (write(server.repl_transfer_s,"\n",1) == -1) { + /* Pinging back in this stage is best-effort. */ + } + } loadingProgress(r->processed_bytes); aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); } From 303cc97ff90a1aaabd6d08e3eb8df83769927a49 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Dec 2013 17:49:45 +0100 Subject: [PATCH 0944/1752] Don't send more than 1 newline/sec while loading RDB. --- src/rdb.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index ff0ddd58d54..72b3b4bb1d7 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1067,13 +1067,17 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { (r->processed_bytes + len)/server.loading_process_events_interval_bytes > r->processed_bytes/server.loading_process_events_interval_bytes) { if (server.masterhost && server.repl_state == REDIS_REPL_TRANSFER) { + static time_t newline_sent; /* Avoid the master to detect the slave is timing out while * loading the RDB file in initial synchronization. We send * a single newline character that is valid protocol but is * guaranteed to either be sent entierly or not, since the byte * is indivisible. */ - if (write(server.repl_transfer_s,"\n",1) == -1) { - /* Pinging back in this stage is best-effort. */ + if (time(NULL) != newline_sent) { + newline_sent = time(NULL); + if (write(server.repl_transfer_s,"\n",1) == -1) { + /* Pinging back in this stage is best-effort. */ + } } } loadingProgress(r->processed_bytes); From 26cf5c8ac6386667ed8ab46c7017e249babfc063 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Dec 2013 17:51:14 +0100 Subject: [PATCH 0945/1752] Log empty DB + Loading data into two separated messages. --- src/replication.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 8ee1ce7c2c3..b5941be1484 100644 --- a/src/replication.c +++ b/src/replication.c @@ -780,7 +780,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { replicationAbortSyncTransfer(); return; } - redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Loading DB in memory"); + redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Flushing old data"); signalFlushedDb(-1); emptyDb(); /* Before loading the DB into memory we need to delete the readable @@ -788,6 +788,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { * rdbLoad() will call the event loop to process events from time to * time for non blocking loading. */ aeDeleteFileEvent(server.el,server.repl_transfer_s,AE_READABLE); + redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Loading DB in memory"); if (rdbLoad(server.rdb_filename) != REDIS_OK) { redisLog(REDIS_WARNING,"Failed trying to load the MASTER synchronization DB from disk"); replicationAbortSyncTransfer(); From b6610a569d80e13d5927a661d2cfa784081ff2bf Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Dec 2013 18:18:24 +0100 Subject: [PATCH 0946/1752] dict.c: added optional callback to dictEmpty(). Redis hash table implementation has many non-blocking features like incremental rehashing, however while deleting a large hash table there was no way to have a callback called to do some incremental work. This commit adds this support, as an optiona callback argument to dictEmpty() that is currently called at a fixed interval (one time every 65k deletions). --- src/db.c | 12 ++++++------ src/debug.c | 4 ++-- src/dict.c | 15 ++++++++------- src/dict.h | 2 +- src/redis.h | 2 +- src/replication.c | 4 ++-- src/sentinel.c | 2 +- src/t_list.c | 2 +- 8 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/db.c b/src/db.c index b91e4f86c67..b625fea5351 100644 --- a/src/db.c +++ b/src/db.c @@ -166,14 +166,14 @@ int dbDelete(redisDb *db, robj *key) { } } -long long emptyDb() { +long long emptyDb(void(callback)(void*)) { int j; long long removed = 0; for (j = 0; j < server.dbnum; j++) { removed += dictSize(server.db[j].dict); - dictEmpty(server.db[j].dict); - dictEmpty(server.db[j].expires); + dictEmpty(server.db[j].dict,callback); + dictEmpty(server.db[j].expires,callback); } return removed; } @@ -209,14 +209,14 @@ void signalFlushedDb(int dbid) { void flushdbCommand(redisClient *c) { server.dirty += dictSize(c->db->dict); signalFlushedDb(c->db->id); - dictEmpty(c->db->dict); - dictEmpty(c->db->expires); + dictEmpty(c->db->dict,NULL); + dictEmpty(c->db->expires,NULL); addReply(c,shared.ok); } void flushallCommand(redisClient *c) { signalFlushedDb(-1); - server.dirty += emptyDb(); + server.dirty += emptyDb(NULL); addReply(c,shared.ok); if (server.rdb_child_pid != -1) { kill(server.rdb_child_pid,SIGUSR1); diff --git a/src/debug.c b/src/debug.c index 7c1a277f769..032653fb2c4 100644 --- a/src/debug.c +++ b/src/debug.c @@ -261,7 +261,7 @@ void debugCommand(redisClient *c) { addReply(c,shared.err); return; } - emptyDb(); + emptyDb(NULL); if (rdbLoad(server.rdb_filename) != REDIS_OK) { addReplyError(c,"Error trying to load the RDB dump"); return; @@ -269,7 +269,7 @@ void debugCommand(redisClient *c) { redisLog(REDIS_WARNING,"DB reloaded by DEBUG RELOAD"); addReply(c,shared.ok); } else if (!strcasecmp(c->argv[1]->ptr,"loadaof")) { - emptyDb(); + emptyDb(NULL); if (loadAppendOnlyFile(server.aof_filename) != REDIS_OK) { addReply(c,shared.err); return; diff --git a/src/dict.c b/src/dict.c index 4d251479966..d0607f56a56 100644 --- a/src/dict.c +++ b/src/dict.c @@ -444,14 +444,15 @@ int dictDeleteNoFree(dict *ht, const void *key) { } /* Destroy an entire dictionary */ -int _dictClear(dict *d, dictht *ht) -{ +int _dictClear(dict *d, dictht *ht, void(callback)(void *)) { unsigned long i; /* Free all the elements */ for (i = 0; i < ht->size && ht->used > 0; i++) { dictEntry *he, *nextHe; + if (callback && (i & 65535) == 0) callback(d->privdata); + if ((he = ht->table[i]) == NULL) continue; while(he) { nextHe = he->next; @@ -472,8 +473,8 @@ int _dictClear(dict *d, dictht *ht) /* Clear & Release the hash table */ void dictRelease(dict *d) { - _dictClear(d,&d->ht[0]); - _dictClear(d,&d->ht[1]); + _dictClear(d,&d->ht[0],NULL); + _dictClear(d,&d->ht[1],NULL); zfree(d); } @@ -882,9 +883,9 @@ static int _dictKeyIndex(dict *d, const void *key) return idx; } -void dictEmpty(dict *d) { - _dictClear(d,&d->ht[0]); - _dictClear(d,&d->ht[1]); +void dictEmpty(dict *d, void(callback)(void*)) { + _dictClear(d,&d->ht[0],callback); + _dictClear(d,&d->ht[1],callback); d->rehashidx = -1; d->iterators = 0; } diff --git a/src/dict.h b/src/dict.h index 11e1b97eedf..3385e9f061c 100644 --- a/src/dict.h +++ b/src/dict.h @@ -160,7 +160,7 @@ dictEntry *dictGetRandomKey(dict *d); void dictPrintStats(dict *d); unsigned int dictGenHashFunction(const void *key, int len); unsigned int dictGenCaseHashFunction(const unsigned char *buf, int len); -void dictEmpty(dict *d); +void dictEmpty(dict *d, void(callback)(void*)); void dictEnableResize(void); void dictDisableResize(void); int dictRehash(dict *d, int n); diff --git a/src/redis.h b/src/redis.h index 35714590200..d23dc69fab7 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1165,7 +1165,7 @@ void setKey(redisDb *db, robj *key, robj *val); int dbExists(redisDb *db, robj *key); robj *dbRandomKey(redisDb *db); int dbDelete(redisDb *db, robj *key); -long long emptyDb(); +long long emptyDb(void(callback)(void*)); int selectDb(redisClient *c, int id); void signalModifiedKey(redisDb *db, robj *key); void signalFlushedDb(int dbid); diff --git a/src/replication.c b/src/replication.c index b5941be1484..ffcad39f2b0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -782,7 +782,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { } redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Flushing old data"); signalFlushedDb(-1); - emptyDb(); + emptyDb(NULL); /* Before loading the DB into memory we need to delete the readable * handler, otherwise it will get called recursively since * rdbLoad() will call the event loop to process events from time to @@ -1445,7 +1445,7 @@ void replicationScriptCacheInit(void) { * to reclaim otherwise unused memory. */ void replicationScriptCacheFlush(void) { - dictEmpty(server.repl_scriptcache_dict); + dictEmpty(server.repl_scriptcache_dict,NULL); listRelease(server.repl_scriptcache_fifo); server.repl_scriptcache_fifo = listCreate(); } diff --git a/src/sentinel.c b/src/sentinel.c index 909f7e8ead5..4c5ebfd7a51 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -393,7 +393,7 @@ void initSentinel(void) { /* Remove usual Redis commands from the command table, then just add * the SENTINEL command. */ - dictEmpty(server.commands); + dictEmpty(server.commands,NULL); for (j = 0; j < sizeof(sentinelcmds)/sizeof(sentinelcmds[0]); j++) { int retval; struct redisCommand *cmd = sentinelcmds+j; diff --git a/src/t_list.c b/src/t_list.c index 3790628d3f1..24febe51a6d 100644 --- a/src/t_list.c +++ b/src/t_list.c @@ -837,7 +837,7 @@ void unblockClientWaitingData(redisClient *c) { dictReleaseIterator(di); /* Cleanup the client structure */ - dictEmpty(c->bpop.keys); + dictEmpty(c->bpop.keys,NULL); if (c->bpop.target) { decrRefCount(c->bpop.target); c->bpop.target = NULL; From 563d6b3f98acd5a922a9ec8e44248058999ba690 Mon Sep 17 00:00:00 2001 From: antirez Date: Tue, 10 Dec 2013 18:38:26 +0100 Subject: [PATCH 0947/1752] Slaves heartbeats during sync improved. The previous fix for false positive timeout detected by master was not complete. There is another blocking stage while loading data for the first synchronization with the master, that is, flushing away the current data from the DB memory. This commit uses the newly introduced dict.c callback in order to make some incremental work (to send "\n" heartbeats to the master) while flushing the old data from memory. It is hard to write a regression test for this issue unfortunately. More support for debugging in the Redis core would be needed in terms of functionalities to simulate a slow DB loading / deletion. --- src/rdb.c | 16 ++-------------- src/redis.h | 1 + src/replication.c | 27 ++++++++++++++++++++++++++- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/src/rdb.c b/src/rdb.c index 72b3b4bb1d7..7d853301d59 100644 --- a/src/rdb.c +++ b/src/rdb.c @@ -1066,20 +1066,8 @@ void rdbLoadProgressCallback(rio *r, const void *buf, size_t len) { if (server.loading_process_events_interval_bytes && (r->processed_bytes + len)/server.loading_process_events_interval_bytes > r->processed_bytes/server.loading_process_events_interval_bytes) { - if (server.masterhost && server.repl_state == REDIS_REPL_TRANSFER) { - static time_t newline_sent; - /* Avoid the master to detect the slave is timing out while - * loading the RDB file in initial synchronization. We send - * a single newline character that is valid protocol but is - * guaranteed to either be sent entierly or not, since the byte - * is indivisible. */ - if (time(NULL) != newline_sent) { - newline_sent = time(NULL); - if (write(server.repl_transfer_s,"\n",1) == -1) { - /* Pinging back in this stage is best-effort. */ - } - } - } + if (server.masterhost && server.repl_state == REDIS_REPL_TRANSFER) + replicationSendNewlineToMaster(); loadingProgress(r->processed_bytes); aeProcessEvents(server.el, AE_FILE_EVENTS|AE_DONT_WAIT); } diff --git a/src/redis.h b/src/redis.h index d23dc69fab7..036596d7c22 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1025,6 +1025,7 @@ void replicationScriptCacheAdd(sds sha1); int replicationScriptCacheExists(sds sha1); void replicationSetMaster(char *ip, int port); void replicationUnsetMaster(void); +void replicationSendNewlineToMaster(void); /* Generic persistence functions */ void startLoading(FILE *fp); diff --git a/src/replication.c b/src/replication.c index ffcad39f2b0..b4bd9fe69de 100644 --- a/src/replication.c +++ b/src/replication.c @@ -701,6 +701,31 @@ void replicationAbortSyncTransfer(void) { server.repl_state = REDIS_REPL_CONNECT; } +/* Avoid the master to detect the slave is timing out while loading the + * RDB file in initial synchronization. We send a single newline character + * that is valid protocol but is guaranteed to either be sent entierly or + * not, since the byte is indivisible. + * + * The function is called in two contexts: while we flush the current + * data with emptyDb(), and while we load the new data received as an + * RDB file from the master. */ +void replicationSendNewlineToMaster(void) { + static time_t newline_sent; + if (time(NULL) != newline_sent) { + newline_sent = time(NULL); + if (write(server.repl_transfer_s,"\n",1) == -1) { + /* Pinging back in this stage is best-effort. */ + } + } +} + +/* Callback used by emptyDb() while flushing away old data to load + * the new dataset received by the master. */ +void replicationEmptyDbCallback(void *privdata) { + REDIS_NOTUSED(privdata); + replicationSendNewlineToMaster(); +} + /* Asynchronously read the SYNC payload we receive from a master */ #define REPL_MAX_WRITTEN_BEFORE_FSYNC (1024*1024*8) /* 8 MB */ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { @@ -782,7 +807,7 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { } redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Flushing old data"); signalFlushedDb(-1); - emptyDb(NULL); + emptyDb(replicationEmptyDbCallback); /* Before loading the DB into memory we need to delete the readable * handler, otherwise it will get called recursively since * rdbLoad() will call the event loop to process events from time to From 25ba2e9607d96bf59540e1a902ac8a84770830b5 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Mon, 24 Dec 2012 23:10:41 +0200 Subject: [PATCH 0948/1752] Fix wrong repldboff type which causes dropped replication in rare cases. --- src/redis.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/redis.h b/src/redis.h index 036596d7c22..483d95bfce1 100644 --- a/src/redis.h +++ b/src/redis.h @@ -466,7 +466,7 @@ typedef struct redisClient { int authenticated; /* when requirepass is non-NULL */ int replstate; /* replication state if this is a slave */ int repldbfd; /* replication DB file descriptor */ - long repldboff; /* replication DB file offset */ + off_t repldboff; /* replication DB file offset */ off_t repldbsize; /* replication DB file size */ long long reploff; /* replication offset if this is our master */ long long repl_ack_off; /* replication ack offset, if this is a slave */ From 0ff078d8d07c98f7c38af8858b4ec29c5623e7b2 Mon Sep 17 00:00:00 2001 From: Yossi Gottlieb Date: Sun, 8 Dec 2013 12:57:03 +0200 Subject: [PATCH 0949/1752] Return proper error on requests with an unbalanced number of quotes. --- src/networking.c | 5 +++++ tests/unit/protocol.tcl | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/src/networking.c b/src/networking.c index 476af38f939..5beb933fa34 100644 --- a/src/networking.c +++ b/src/networking.c @@ -868,6 +868,11 @@ int processInlineBuffer(redisClient *c) { aux = sdsnewlen(c->querybuf,querylen); argv = sdssplitargs(aux,&argc); sdsfree(aux); + if (argv == NULL) { + addReplyError(c,"Protocol error: unbalanced quotes in request"); + setProtocolError(c,0); + return REDIS_ERR; + } /* Newline from slaves can be used to refresh the last ACK time. * This is useful for a slave to ping back while loading a big diff --git a/tests/unit/protocol.tcl b/tests/unit/protocol.tcl index 1700e489249..ac99c3abb47 100644 --- a/tests/unit/protocol.tcl +++ b/tests/unit/protocol.tcl @@ -60,6 +60,14 @@ start_server {tags {"protocol"}} { assert_error "*wrong*arguments*ping*" {r ping x y z} } + test "Unbalanced number of quotes" { + reconnect + r write "set \"\"\"test-key\"\"\" test-value\r\n" + r write "ping\r\n" + r flush + assert_error "*unbalanced*" {r read} + } + set c 0 foreach seq [list "\x00" "*\x00" "$\x00"] { incr c From 9a8ae5a553ddf1ca3866104f12ba428c9fd0431c Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Dec 2013 15:23:10 +0100 Subject: [PATCH 0950/1752] Replication: publish the slave_repl_offset when disconnected from master. When a slave was disconnected from its master the replication offset was reported as -1. Now it is reported as the replication offset of the previous master, so that failover can be performed using this value in order to try to select a slave with more processed data from a set of slaves of the old master. --- src/redis.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index a8ece8197d7..b8e0f8f5c04 100644 --- a/src/redis.c +++ b/src/redis.c @@ -2416,6 +2416,13 @@ sds genRedisInfoString(char *section) { "role:%s\r\n", server.masterhost == NULL ? "master" : "slave"); if (server.masterhost) { + long long slave_repl_offset = 1; + + if (server.master) + slave_repl_offset = server.master->reploff; + else if (server.cached_master) + slave_repl_offset = server.cached_master->reploff; + info = sdscatprintf(info, "master_host:%s\r\n" "master_port:%d\r\n" @@ -2430,7 +2437,7 @@ sds genRedisInfoString(char *section) { server.master ? ((int)(server.unixtime-server.master->lastinteraction)) : -1, server.repl_state == REDIS_REPL_TRANSFER, - server.master ? server.master->reploff : -1 + slave_repl_offset ); if (server.repl_state == REDIS_REPL_TRANSFER) { From 8995634014e0f7bfd93b7c018f7b3979796a6d84 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 11 Dec 2013 15:31:57 +0100 Subject: [PATCH 0951/1752] Redis 2.8.3. --- 00-RELEASENOTES | 18 ++++++++++++++++++ src/version.h | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/00-RELEASENOTES b/00-RELEASENOTES index f50cda76545..3c5a8604610 100644 --- a/00-RELEASENOTES +++ b/00-RELEASENOTES @@ -14,6 +14,24 @@ HIGH: There is a critical bug that may affect a subset of users. Upgrade! CRITICAL: There is a critical bug affecting MOST USERS. Upgrade ASAP. -------------------------------------------------------------------------------- +--[ Redis 2.8.3 ] Release date: 11 Dec 2013 + +# UPGRADE URGENCY: MODERATE for Redis, HIGH for Sentinel. + +* [FIX] Sentinel instance role sampling fixed, the system is now more + reliable during failover and when reconfiguring instances with + non matching configuration. +* [FIX] Inline requests are now handled even when terminated with just LF. +* [FIX] Replication timeout handling greatly improved, now the slave is able + to ping the master while removing the old data from memory, and while + loading the new RDB file. This avoid false timeouts sensed by + masters. +* [FIX] Fixed a replication bug involving 32 bit instances and big datasets + hard to compress that resulted into more than 2GB of RDB file sent. +* [FIX] Return error for inline requests with unbalanced quotes. +* [FIX] Publish the slave replication offset even when disconnected from the + master if there is still a cached master instance. + --[ Redis 2.8.2 ] Release date: 2 Dec 2013 # UPGRADE URGENCY: MODERATE for both Redis and Sentinel. diff --git a/src/version.h b/src/version.h index d678aa8d836..fd199c41a78 100644 --- a/src/version.h +++ b/src/version.h @@ -1 +1 @@ -#define REDIS_VERSION "2.8.2" +#define REDIS_VERSION "2.8.3" From 5e75e681fe42c9986ddc4360a602669aaebb578e Mon Sep 17 00:00:00 2001 From: codeeply Date: Thu, 12 Dec 2013 16:33:29 +0800 Subject: [PATCH 0952/1752] comment mistake fixed --- src/sds.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sds.c b/src/sds.c index 338b4262a3e..64c9a9615a0 100644 --- a/src/sds.c +++ b/src/sds.c @@ -383,7 +383,7 @@ sds sdstrim(sds s, const char *cset) { * Example: * * s = sdsnew("Hello World"); - * sdstrim(s,1,-1); => "ello Worl" + * sdsrange(s,1,-1); => "ello World" */ void sdsrange(sds s, int start, int end) { struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); From 45c60a903ab87e56106f84ad7ebeb7007e29bb8b Mon Sep 17 00:00:00 2001 From: Yubao Liu Date: Sat, 30 Nov 2013 14:14:28 +0800 Subject: [PATCH 0953/1752] fix typo in redis.conf and sentinel.conf --- redis.conf | 2 +- sentinel.conf | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/redis.conf b/redis.conf index 6786e8dfa58..7000580a90e 100644 --- a/redis.conf +++ b/redis.conf @@ -633,7 +633,7 @@ activerehashing yes # # normal -> normal clients # slave -> slave clients and MONITOR clients -# pubsub -> clients subcribed to at least one pubsub channel or pattern +# pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: # diff --git a/sentinel.conf b/sentinel.conf index 248e76c06de..e4434222135 100644 --- a/sentinel.conf +++ b/sentinel.conf @@ -6,7 +6,7 @@ port 26379 # sentinel monitor # -# Tells Sentinel to monitor this slave, and to consider it in O_DOWN +# Tells Sentinel to monitor this master, and to consider it in O_DOWN # (Objectively Down) state only if at least sentinels agree. # # Note that whatever is the ODOWN quorum, a Sentinel will require to @@ -63,7 +63,7 @@ sentinel parallel-syncs mymaster 1 # times the failover timeout. # # - The time needed for a slave replicating to a wrong master according -# to a Sentinel currnet configuration, to be forced to replicate +# to a Sentinel current configuration, to be forced to replicate # with the right master, is exactly the failover timeout (counting since # the moment a Sentinel detected the misconfiguration). # @@ -102,7 +102,7 @@ sentinel failover-timeout mymaster 180000 # # sentinel notification-script # -# Call the specified notification script for any sentienl event that is +# Call the specified notification script for any sentinel event that is # generated in the WARNING level (for instance -sdown, -odown, and so forth). # This script should notify the system administrator via email, SMS, or any # other messaging system, that there is something wrong with the monitored From f9e9448a5b74fd11b52165b49765c3c632d24587 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 12 Dec 2013 15:19:08 +0100 Subject: [PATCH 0954/1752] Makefile: remove odd syntax not compatible with some make versions. See issue #1448. --- src/Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index c37549dfe6d..a8631e02077 100644 --- a/src/Makefile +++ b/src/Makefile @@ -55,15 +55,19 @@ FINAL_LIBS=-lm DEBUG=-g -ggdb ifeq ($(uname_S),SunOS) + # SunOS INSTALL=cp -pf FINAL_CFLAGS+= -D__EXTENSIONS__ -D_XPG6 FINAL_LIBS+= -ldl -lnsl -lsocket -lpthread -else ifeq ($(uname_S),Darwin) - else +ifeq ($(uname_S),Darwin) + # Darwin (nothing to do) +else + # All the other OSes (notably Linux) FINAL_LDFLAGS+= -rdynamic FINAL_LIBS+= -pthread endif +endif # Include paths to dependencies FINAL_CFLAGS+= -I../deps/hiredis -I../deps/linenoise -I../deps/lua/src From 2507e366b2684e58ce195765df84af2f2af23564 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Dec 2013 11:01:13 +0100 Subject: [PATCH 0955/1752] Sentinel: dead code removed. --- src/sentinel.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/sentinel.c b/src/sentinel.c index 4c5ebfd7a51..30188e3e55a 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -1640,8 +1640,6 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { sds *lines; int numlines, j; int role = 0; - int runid_changed = 0; /* true if runid changed. */ - int first_runid = 0; /* true if this is the first runid we receive. */ /* The following fields must be reset to a given value in the case they * are not found at all in the INFO output. */ @@ -1657,10 +1655,8 @@ void sentinelRefreshInstanceInfo(sentinelRedisInstance *ri, const char *info) { if (sdslen(l) >= 47 && !memcmp(l,"run_id:",7)) { if (ri->runid == NULL) { ri->runid = sdsnewlen(l+7,40); - first_runid = 1; } else { if (strncmp(ri->runid,l+7,40) != 0) { - runid_changed = 1; sentinelEvent(REDIS_NOTICE,"+reboot",ri,"%@"); sdsfree(ri->runid); ri->runid = sdsnewlen(l+7,40); @@ -3236,17 +3232,12 @@ void sentinelFailoverReconfNextSlave(sentinelRedisInstance *master) { void sentinelFailoverSwitchToPromotedSlave(sentinelRedisInstance *master) { sentinelRedisInstance *ref = master->promoted_slave ? master->promoted_slave : master; - sds old_master_ip; - int old_master_port; sentinelEvent(REDIS_WARNING,"+switch-master",master,"%s %s %d %s %d", master->name, master->addr->ip, master->addr->port, ref->addr->ip, ref->addr->port); - old_master_ip = sdsdup(master->addr->ip); - old_master_port = master->addr->port; sentinelResetMasterAndChangeAddress(master,ref->addr->ip,ref->addr->port); - sdsfree(old_master_ip); } void sentinelFailoverStateMachine(sentinelRedisInstance *ri) { From 993e0ede762184bdb8620389017e07be621d8836 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Dec 2013 11:29:59 +0100 Subject: [PATCH 0956/1752] SDIFF iterator misuse fixed in diff algorithm #1. The bug could be easily triggered by: SADD foo a b c 1 2 3 4 5 6 SDIFF foo foo When the key was the same in two sets, an unsafe iterator was used to check existence of elements in the same set we were iterating. Usually this would just result into a wrong output, however with the dict.c API misuse protection we have in place, the result was actually an assertion failed that was triggered by the CI test, while creating random datasets for the "MASTER and SLAVE consistency" test. --- src/t_set.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/t_set.c b/src/t_set.c index 7b586f6ddc0..64a0252ebb5 100644 --- a/src/t_set.c +++ b/src/t_set.c @@ -825,6 +825,7 @@ void sunionDiffGenericCommand(redisClient *c, robj **setkeys, int setnum, robj * while((ele = setTypeNextObject(si)) != NULL) { for (j = 1; j < setnum; j++) { if (!sets[j]) continue; /* no key is an empty set. */ + if (sets[j] == sets[0]) break; /* same set! */ if (setTypeIsMember(sets[j],ele)) break; } if (j == setnum) { From 8eb1cb3b525c110d52fe76f52bf3a21c1eab056d Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Dec 2013 11:37:03 +0100 Subject: [PATCH 0957/1752] SDIFF iterator misuse bug regression test added. See commit c00453d for more info about the bug. --- tests/unit/type/set.tcl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/unit/type/set.tcl b/tests/unit/type/set.tcl index a77759e5d7e..162de0af770 100644 --- a/tests/unit/type/set.tcl +++ b/tests/unit/type/set.tcl @@ -214,6 +214,12 @@ start_server { r sdiff set1 set2 set3 } {} + test "SDIFF with same set two times" { + r del set1 + r sadd set1 a b c 1 2 3 4 5 6 + r sdiff set1 set1 + } {} + test "SDIFF fuzzing" { for {set j 0} {$j < 100} {incr j} { unset -nocomplain s From 33f6f35f3489722ee36d26e828f6838785fde98d Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 13 Dec 2013 13:13:54 +0100 Subject: [PATCH 0958/1752] Makefile.dep updated. --- src/Makefile.dep | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Makefile.dep b/src/Makefile.dep index 784b272af38..fafb3d29a93 100644 --- a/src/Makefile.dep +++ b/src/Makefile.dep @@ -1,5 +1,5 @@ adlist.o: adlist.c adlist.h zmalloc.h -ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c +ae.o: ae.c ae.h zmalloc.h config.h ae_kqueue.c ae_select.c ae_evport.c ae_epoll.c ae_epoll.o: ae_epoll.c ae_evport.o: ae_evport.c ae_kqueue.o: ae_kqueue.c @@ -24,7 +24,7 @@ db.o: db.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ debug.o: debug.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h crc64.h bio.h -dict.o: dict.c fmacros.h dict.h zmalloc.h +dict.o: dict.c fmacros.h dict.h zmalloc.h redisassert.h endianconv.o: endianconv.c intset.o: intset.c intset.h zmalloc.h endianconv.h config.h lzf_c.o: lzf_c.c lzfP.h @@ -70,18 +70,18 @@ replication.o: replication.c redis.h fmacros.h config.h \ ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h sds.h dict.h \ adlist.h zmalloc.h anet.h ziplist.h intset.h version.h util.h rdb.h \ rio.h -rio.o: rio.c fmacros.h rio.h sds.h util.h crc64.h +rio.o: rio.c fmacros.h rio.h sds.h util.h crc64.h config.h redis.h \ + ../deps/lua/src/lua.h ../deps/lua/src/luaconf.h ae.h dict.h adlist.h \ + zmalloc.h anet.h ziplist.h intset.h version.h rdb.h scripting.o: scripting.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h sha1.h rand.h \ - ../deps/lua/src/lauxlib.h ../deps/lua/src/lua.h \ - ../deps/lua/src/lualib.h + ../deps/lua/src/lauxlib.h ../deps/lua/src/lualib.h sds.o: sds.c sds.h zmalloc.h sentinel.o: sentinel.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h \ - ../deps/hiredis/hiredis.h ../deps/hiredis/async.h \ - ../deps/hiredis/hiredis.h + ../deps/hiredis/hiredis.h ../deps/hiredis/async.h setproctitle.o: setproctitle.c sha1.o: sha1.c sha1.h config.h slowlog.o: slowlog.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ @@ -108,7 +108,8 @@ t_string.o: t_string.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ t_zset.o: t_zset.c redis.h fmacros.h config.h ../deps/lua/src/lua.h \ ../deps/lua/src/luaconf.h ae.h sds.h dict.h adlist.h zmalloc.h anet.h \ ziplist.h intset.h version.h util.h rdb.h rio.h -util.o: util.c fmacros.h util.h -ziplist.o: ziplist.c zmalloc.h util.h ziplist.h endianconv.h config.h +util.o: util.c fmacros.h util.h sds.h +ziplist.o: ziplist.c zmalloc.h util.h sds.h ziplist.h endianconv.h \ + config.h redisassert.h zipmap.o: zipmap.c zmalloc.h endianconv.h config.h zmalloc.o: zmalloc.c config.h zmalloc.h From 673c42bb4e2f8ab68b70857f48619c6b5b068068 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Dec 2013 10:18:45 +0100 Subject: [PATCH 0959/1752] Example redis.conf formatted to better show appendfilename option. --- redis.conf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redis.conf b/redis.conf index 7000580a90e..460987a8eba 100644 --- a/redis.conf +++ b/redis.conf @@ -420,7 +420,8 @@ slave-priority 100 appendonly no # The name of the append only file (default: "appendonly.aof") -# appendfilename appendonly.aof + +appendfilename "appendonly.aof" # The fsync() call tells the Operating System to actually write data on disk # instead to wait for more data in the output buffer. Some OS will really flush @@ -467,6 +468,7 @@ appendfsync everysec # # If you have latency problems turn this to "yes". Otherwise leave it as # "no" that is the safest pick from the point of view of durability. + no-appendfsync-on-rewrite no # Automatic rewrite of the append only file. From ddd529bc3bbec3f4b7746482483a736ab65b2b54 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Dec 2013 15:25:45 +0100 Subject: [PATCH 0960/1752] CONFIG REWRITE: don't wipe unknown options. With this commit options not explicitly rewritten by CONFIG REWRITE are not touched at all. These include new options that may not have support for REWRITE, and other special cases like rename-command and include. --- src/config.c | 59 ++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 9 deletions(-) diff --git a/src/config.c b/src/config.c index 0d8376e5a16..453b8988e21 100644 --- a/src/config.c +++ b/src/config.c @@ -1117,17 +1117,27 @@ void dictListDestructor(void *privdata, void *val); void rewriteConfigSentinelOption(struct rewriteConfigState *state); dictType optionToLineDictType = { - dictSdsHash, /* hash function */ + dictSdsCaseHash, /* hash function */ NULL, /* key dup */ NULL, /* val dup */ - dictSdsKeyCompare, /* key compare */ + dictSdsKeyCaseCompare, /* key compare */ dictSdsDestructor, /* key destructor */ dictListDestructor /* val destructor */ }; +dictType optionSetDictType = { + dictSdsCaseHash, /* hash function */ + NULL, /* key dup */ + NULL, /* val dup */ + dictSdsKeyCaseCompare, /* key compare */ + dictSdsDestructor, /* key destructor */ + NULL /* val destructor */ +}; + /* The config rewrite state. */ struct rewriteConfigState { dict *option_to_line; /* Option -> list of config file lines map */ + dict *rewritten; /* Dictionary of already processed options */ int numlines; /* Number of lines in current config */ sds *lines; /* Current lines as an array of sds strings */ int has_tail; /* True if we already added directives that were @@ -1151,6 +1161,16 @@ void rewriteConfigAddLineNumberToOption(struct rewriteConfigState *state, sds op listAddNodeTail(l,(void*)(long)linenum); } +/* Add the specified option to the set of processed options. + * This is useful as only unused lines of processed options will be blanked + * in the config file, while options the rewrite process does not understand + * remain untouched. */ +void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, char *option) { + sds opt = sdsnew(option); + + if (dictAdd(state->rewritten,opt) != DICT_OK) sdsfree(opt); +} + /* Read the old file, split it into lines to populate a newly created * config rewrite state, and return it to the caller. * @@ -1165,6 +1185,7 @@ struct rewriteConfigState *rewriteConfigReadOldFile(char *path) { if (fp == NULL && errno != ENOENT) return NULL; state->option_to_line = dictCreate(&optionToLineDictType,NULL); + state->rewritten = dictCreate(&optionSetDictType,NULL); state->numlines = 0; state->lines = NULL; state->has_tail = 0; @@ -1232,6 +1253,8 @@ void rewriteConfigRewriteLine(struct rewriteConfigState *state, char *option, sd sds o = sdsnew(option); list *l = dictFetchValue(state->option_to_line,o); + rewriteConfigMarkAsProcessed(state,option); + if (!l && !force) { /* Option not used previously, and we are not forced to use it. */ sdsfree(line); @@ -1288,7 +1311,6 @@ void rewriteConfigBytesOption(struct rewriteConfigState *state, char *option, lo rewriteConfigFormatMemory(buf,sizeof(buf),value); line = sdscatprintf(sdsempty(),"%s %s",option,buf); rewriteConfigRewriteLine(state,option,line,force); - } /* Rewrite a yes/no option. */ @@ -1307,7 +1329,10 @@ void rewriteConfigStringOption(struct rewriteConfigState *state, char *option, c /* String options set to NULL need to be not present at all in the * configuration file to be set to NULL again at the next reboot. */ - if (value == NULL) return; + if (value == NULL) { + rewriteConfigMarkAsProcessed(state,option); + return; + } /* Compare the strings as sds strings to have a binary safe comparison. */ if (defvalue && strcmp(value,defvalue) == 0) force = 0; @@ -1397,7 +1422,10 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { void rewriteConfigDirOption(struct rewriteConfigState *state) { char cwd[1024]; - if (getcwd(cwd,sizeof(cwd)) == NULL) return; /* no rewrite on error. */ + if (getcwd(cwd,sizeof(cwd)) == NULL) { + rewriteConfigMarkAsProcessed(state,"dir"); + return; /* no rewrite on error. */ + } rewriteConfigStringOption(state,"dir",cwd,NULL); } @@ -1408,7 +1436,10 @@ void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { /* If this is a master, we want all the slaveof config options * in the file to be removed. */ - if (server.masterhost == NULL) return; + if (server.masterhost == NULL) { + rewriteConfigMarkAsProcessed(state,"slaveof"); + return; + } line = sdscatprintf(sdsempty(),"%s %s %d", option, server.masterhost, server.masterport); rewriteConfigRewriteLine(state,option,line,1); @@ -1473,7 +1504,10 @@ void rewriteConfigBindOption(struct rewriteConfigState *state) { char *option = "bind"; /* Nothing to rewrite if we don't have bind addresses. */ - if (server.bindaddr_count == 0) return; + if (server.bindaddr_count == 0) { + rewriteConfigMarkAsProcessed(state,option); + return; + } /* Rewrite as bind ... */ addresses = sdsjoin(server.bindaddr,server.bindaddr_count," "); @@ -1509,6 +1543,7 @@ sds rewriteConfigGetContentFromState(struct rewriteConfigState *state) { void rewriteConfigReleaseState(struct rewriteConfigState *state) { sdsfreesplitres(state->lines,state->numlines); dictRelease(state->option_to_line); + dictRelease(state->rewritten); zfree(state); } @@ -1526,6 +1561,14 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { while((de = dictNext(di)) != NULL) { list *l = dictGetVal(de); + sds option = dictGetKey(de); + + /* Don't blank lines about options the rewrite process + * don't understand. */ + if (dictFetch(state->rewritten,option) == NULL) { + redisLog(REDIS_DEBUG,"Not rewritten option: %s", option); + continue; + } while(listLength(l)) { listNode *ln = listFirst(l); @@ -1615,8 +1658,6 @@ int rewriteConfig(char *path) { /* Step 2: rewrite every single option, replacing or appending it inside * the rewrite state. */ - /* TODO: Turn every default into a define, use it also in - * initServerConfig(). */ rewriteConfigYesNoOption(state,"daemonize",server.daemonize,0); rewriteConfigStringOption(state,"pidfile",server.pidfile,REDIS_DEFAULT_PID_FILE); rewriteConfigNumericalOption(state,"port",server.port,REDIS_SERVERPORT); From 090bcc946f631f584479a0062823d3a892eed709 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Dec 2013 15:30:06 +0100 Subject: [PATCH 0961/1752] CONFIG REWRITE: old development comments removed. --- src/config.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/config.c b/src/config.c index 453b8988e21..25b87663e70 100644 --- a/src/config.c +++ b/src/config.c @@ -1107,8 +1107,8 @@ void configGetCommand(redisClient *c) { /* We use the following dictionary type to store where a configuration * option is mentioned in the old configuration file, so it's * like "maxmemory" -> list of line numbers (first line is zero). */ -unsigned int dictSdsHash(const void *key); -int dictSdsKeyCompare(void *privdata, const void *key1, const void *key2); +unsigned int dictSdsCaseHash(const void *key); +int dictSdsKeyCaseCompare(void *privdata, const void *key1, const void *key2); void dictSdsDestructor(void *privdata, void *val); void dictListDestructor(void *privdata, void *val); @@ -1168,7 +1168,7 @@ void rewriteConfigAddLineNumberToOption(struct rewriteConfigState *state, sds op void rewriteConfigMarkAsProcessed(struct rewriteConfigState *state, char *option) { sds opt = sdsnew(option); - if (dictAdd(state->rewritten,opt) != DICT_OK) sdsfree(opt); + if (dictAdd(state->rewritten,opt,NULL) != DICT_OK) sdsfree(opt); } /* Read the old file, split it into lines to populate a newly created @@ -1565,7 +1565,7 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { /* Don't blank lines about options the rewrite process * don't understand. */ - if (dictFetch(state->rewritten,option) == NULL) { + if (dictFetchValue(state->rewritten,option) == NULL) { redisLog(REDIS_DEBUG,"Not rewritten option: %s", option); continue; } From d62bbfda1654a5e0c0c9f489f30e7944dbc0c01c Mon Sep 17 00:00:00 2001 From: Yubao Liu Date: Sat, 30 Nov 2013 23:57:48 +0800 Subject: [PATCH 0962/1752] CONFIG REWRITE: don't throw some options on config rewrite Those options will be thrown without this patch: include, rename-command, min-slaves-to-write, min-slaves-max-lag, appendfilename. --- redis.conf | 27 +++++++++++++++++---------- src/config.c | 29 ++++++++++++++++------------- src/redis.c | 2 +- src/redis.h | 1 + 4 files changed, 35 insertions(+), 24 deletions(-) diff --git a/redis.conf b/redis.conf index 460987a8eba..0224b60c952 100644 --- a/redis.conf +++ b/redis.conf @@ -12,6 +12,22 @@ # # units are case insensitive so 1GB 1Gb 1gB are all the same. +################################## INCLUDES ################################### + +# Include one or more other config files here. This is useful if you +# have a standard template that goes to all Redis server but also need +# to customize a few per-server settings. Include files can include +# other files, so use this wisely. +# +# Notice option "include" won't be rewritten by command "CONFIG REWRITE" +# from admin or Redis sentinel, you'd better put this option at the +# beginning of this file to avoid overwriting config change at runtime. +# +# include /path/to/local.conf +# include /path/to/other.conf + +################################ GENERAL ##################################### + # By default Redis does not run as a daemon. Use 'yes' if you need it. # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. daemonize no @@ -88,7 +104,7 @@ logfile "" # dbid is a number between 0 and 'databases'-1 databases 16 -################################ SNAPSHOTTING ################################# +################################ SNAPSHOTTING ################################ # # Save the DB on disk: # @@ -686,12 +702,3 @@ hz 10 # big latency spikes. aof-rewrite-incremental-fsync yes -################################## INCLUDES ################################### - -# Include one or more other config files here. This is useful if you -# have a standard template that goes to all Redis server but also need -# to customize a few per-server settings. Include files can include -# other files, so use this wisely. -# -# include /path/to/local.conf -# include /path/to/other.conf diff --git a/src/config.c b/src/config.c index 25b87663e70..83082c539eb 100644 --- a/src/config.c +++ b/src/config.c @@ -1445,17 +1445,6 @@ void rewriteConfigSlaveofOption(struct rewriteConfigState *state) { rewriteConfigRewriteLine(state,option,line,1); } -/* Rewrite the appendonly option. */ -void rewriteConfigAppendonlyOption(struct rewriteConfigState *state) { - int force = server.aof_state != REDIS_AOF_OFF; - char *option = "appendonly"; - sds line; - - line = sdscatprintf(sdsempty(),"%s %s", option, - (server.aof_state == REDIS_AOF_OFF) ? "no" : "yes"); - rewriteConfigRewriteLine(state,option,line,force); -} - /* Rewrite the notify-keyspace-events option. */ void rewriteConfigNotifykeyspaceeventsOption(struct rewriteConfigState *state) { int force = server.notify_keyspace_events != 0; @@ -1554,12 +1543,23 @@ void rewriteConfigReleaseState(struct rewriteConfigState *state) { * should be replaced by empty lines. * * This function does just this, iterating all the option names and - * blanking all the lines still associated. */ + * blanking all the lines still associated. + * + * Two options "include" and "rename-command" are special, they are + * just kept because struct RedisServer doesn't record them. Notice + * this also means the included config file isn't rewritten, you'd + * better put "include" at the beginning of Redis main config file + * so that runtime config change won't be canceled by conflicted + * options in the included config file. */ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { dictIterator *di = dictGetIterator(state->option_to_line); dictEntry *de; while((de = dictNext(di)) != NULL) { + sds option = dictGetKey(de); + if (!strcmp(option, "include") || !strcmp(option, "rename-command")) + continue; + list *l = dictGetVal(de); sds option = dictGetKey(de); @@ -1693,6 +1693,8 @@ int rewriteConfig(char *path) { rewriteConfigBytesOption(state,"repl-backlog-ttl",server.repl_backlog_time_limit,REDIS_DEFAULT_REPL_BACKLOG_TIME_LIMIT); rewriteConfigYesNoOption(state,"repl-disable-tcp-nodelay",server.repl_disable_tcp_nodelay,REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY); rewriteConfigNumericalOption(state,"slave-priority",server.slave_priority,REDIS_DEFAULT_SLAVE_PRIORITY); + rewriteConfigNumericalOption(state,"min-slaves-to-write",server.repl_min_slaves_to_write,REDIS_DEFAULT_MIN_SLAVES_TO_WRITE); + rewriteConfigNumericalOption(state,"min-slaves-max-lag",server.repl_min_slaves_max_lag,REDIS_DEFAULT_MIN_SLAVES_MAX_LAG); rewriteConfigStringOption(state,"requirepass",server.requirepass,NULL); rewriteConfigNumericalOption(state,"maxclients",server.maxclients,REDIS_MAX_CLIENTS); rewriteConfigBytesOption(state,"maxmemory",server.maxmemory,REDIS_DEFAULT_MAXMEMORY); @@ -1705,7 +1707,8 @@ int rewriteConfig(char *path) { "noeviction", REDIS_MAXMEMORY_NO_EVICTION, NULL, REDIS_DEFAULT_MAXMEMORY_POLICY); rewriteConfigNumericalOption(state,"maxmemory-samples",server.maxmemory_samples,REDIS_DEFAULT_MAXMEMORY_SAMPLES); - rewriteConfigAppendonlyOption(state); + rewriteConfigYesNoOption(state,"appendonly",server.aof_state != REDIS_AOF_OFF,0); + rewriteConfigStringOption(state,"appendfilename",server.aof_filename,REDIS_DEFAULT_AOF_FILENAME); rewriteConfigEnumOption(state,"appendfsync",server.aof_fsync, "everysec", AOF_FSYNC_EVERYSEC, "always", AOF_FSYNC_ALWAYS, diff --git a/src/redis.c b/src/redis.c index b8e0f8f5c04..5fb75cd63af 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1302,7 +1302,7 @@ void initServerConfig() { server.aof_rewrite_incremental_fsync = REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC; server.pidfile = zstrdup(REDIS_DEFAULT_PID_FILE); server.rdb_filename = zstrdup(REDIS_DEFAULT_RDB_FILENAME); - server.aof_filename = zstrdup("appendonly.aof"); + server.aof_filename = zstrdup(REDIS_DEFAULT_AOF_FILENAME); server.requirepass = NULL; server.rdb_compression = REDIS_DEFAULT_RDB_COMPRESSION; server.rdb_checksum = REDIS_DEFAULT_RDB_CHECKSUM; diff --git a/src/redis.h b/src/redis.h index 483d95bfce1..7cba7ad304e 100644 --- a/src/redis.h +++ b/src/redis.h @@ -113,6 +113,7 @@ #define REDIS_DEFAULT_REPL_DISABLE_TCP_NODELAY 0 #define REDIS_DEFAULT_MAXMEMORY 0 #define REDIS_DEFAULT_MAXMEMORY_SAMPLES 3 +#define REDIS_DEFAULT_AOF_FILENAME "appendonly.aof" #define REDIS_DEFAULT_AOF_NO_FSYNC_ON_REWRITE 0 #define REDIS_DEFAULT_ACTIVE_REHASHING 1 #define REDIS_DEFAULT_AOF_REWRITE_INCREMENTAL_FSYNC 1 From fb8a480f542b411ee14a998fd575e150dbc317ee Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Dec 2013 15:55:25 +0100 Subject: [PATCH 0963/1752] CONFIG REWRITE: no special handling or include and rename-command. CONFIG REWRITE is now wiser and does not touch what it does not understand inside redis.conf. --- src/config.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/config.c b/src/config.c index 83082c539eb..b4f09440184 100644 --- a/src/config.c +++ b/src/config.c @@ -1543,23 +1543,12 @@ void rewriteConfigReleaseState(struct rewriteConfigState *state) { * should be replaced by empty lines. * * This function does just this, iterating all the option names and - * blanking all the lines still associated. - * - * Two options "include" and "rename-command" are special, they are - * just kept because struct RedisServer doesn't record them. Notice - * this also means the included config file isn't rewritten, you'd - * better put "include" at the beginning of Redis main config file - * so that runtime config change won't be canceled by conflicted - * options in the included config file. */ + * blanking all the lines still associated. */ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { dictIterator *di = dictGetIterator(state->option_to_line); dictEntry *de; while((de = dictNext(di)) != NULL) { - sds option = dictGetKey(de); - if (!strcmp(option, "include") || !strcmp(option, "rename-command")) - continue; - list *l = dictGetVal(de); sds option = dictGetKey(de); From d33f3689d7c42bab19ff84eac4a095648c107317 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 19 Dec 2013 16:02:24 +0100 Subject: [PATCH 0964/1752] Clarify include directive behavior in example redis.conf. --- redis.conf | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/redis.conf b/redis.conf index 0224b60c952..6306b8a0315 100644 --- a/redis.conf +++ b/redis.conf @@ -20,8 +20,12 @@ # other files, so use this wisely. # # Notice option "include" won't be rewritten by command "CONFIG REWRITE" -# from admin or Redis sentinel, you'd better put this option at the -# beginning of this file to avoid overwriting config change at runtime. +# from admin or Redis Sentinel. Since Redis always uses the last processed +# line as value of a configuration directive, you'd better put includes +# at the beginning of this file to avoid overwriting config change at runtime. +# +# If instead you are interested in using includes to override configuration +# options, it is better to use include as the last line. # # include /path/to/local.conf # include /path/to/other.conf From 9ba9d78ada6db6822ec463c47ab58868473cc3f4 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 21 Dec 2013 00:23:37 +0100 Subject: [PATCH 0965/1752] Log when a slave lose the connection with its master. --- src/networking.c | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/networking.c b/src/networking.c index 5beb933fa34..98239531e38 100644 --- a/src/networking.c +++ b/src/networking.c @@ -637,15 +637,16 @@ void freeClient(redisClient *c) { * * Note that before doing this we make sure that the client is not in * some unexpected state, by checking its flags. */ - if (server.master && - (c->flags & REDIS_MASTER) && - !(c->flags & (REDIS_CLOSE_AFTER_REPLY| - REDIS_CLOSE_ASAP| - REDIS_BLOCKED| - REDIS_UNBLOCKED))) - { - replicationCacheMaster(c); - return; + if (server.master && c->flags & REDIS_MASTER) { + redisLog(REDIS_WARNING,"Connection with master lost."); + if (!(c->flags & (REDIS_CLOSE_AFTER_REPLY| + REDIS_CLOSE_ASAP| + REDIS_BLOCKED| + REDIS_UNBLOCKED))) + { + replicationCacheMaster(c); + return; + } } /* Free the query buffer */ From 5fa937d956786fc43c223adac505aacb93476117 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 22 Dec 2013 10:15:35 +0100 Subject: [PATCH 0966/1752] Slave disconnection is an event worth logging. --- src/networking.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/networking.c b/src/networking.c index 98239531e38..5a692c79bcd 100644 --- a/src/networking.c +++ b/src/networking.c @@ -649,6 +649,16 @@ void freeClient(redisClient *c) { } } + /* Log link disconnection with slave */ + if (c->flags & REDIS_SLAVE) { + char ip[REDIS_IP_STR_LEN]; + + if (anetPeerToString(c->fd,ip,sizeof(ip),NULL) == -1) + strncpy(ip,"?",REDIS_IP_STR_LEN); + redisLog(REDIS_WARNING,"Connection with slave %s:%d lost.", + ip, c->slave_listening_port); + } + /* Free the query buffer */ sdsfree(c->querybuf); c->querybuf = NULL; From 4456ee11734872fcab676ff439784ce94070d5a5 Mon Sep 17 00:00:00 2001 From: antirez Date: Sun, 22 Dec 2013 11:43:25 +0100 Subject: [PATCH 0967/1752] Make new masters inherit replication offsets. Currently replication offsets could be used into a limited way in order to understand, out of a set of slaves, what is the one with the most updated data. For example this comparison is possible of N slaves were replicating all with the same master. However the replication offset was not transferred from master to slaves (that are later promoted as masters) in any way, so for instance if there were three instances A, B, C, with A master and B and C replication from A, the following could happen: C disconnects from A. B is turned into master. A is switched to master of B. B receives some write. In this context there was no way to compare the offset of A and C, because B would use its own local master replication offset as replication offset to initialize the replication with A. With this commit what happens is that when B is turned into master it inherits the replication offset from A, making A and C comparable. In the above case assuming no inconsistencies are created during the disconnection and failover process, A will show to have a replication offset greater than C. Note that this does not mean offsets are always comparable to understand what is, in a set of instances, since in more complex examples the replica with the higher replication offset could be partitioned away when picking the instance to elect as new master. However this in general improves the ability of a system to try to pick a good replica to promote to master. --- src/replication.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index b4bd9fe69de..b1d08c7749d 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1242,6 +1242,7 @@ void replicationSetMaster(char *ip, int port) { freeReplicationBacklog(); /* Don't allow our chained slaves to PSYNC. */ cancelReplicationHandshake(); server.repl_state = REDIS_REPL_CONNECT; + server.master_repl_offset = 0; } /* Cancel replication, setting the instance as a master itself. */ @@ -1249,7 +1250,17 @@ void replicationUnsetMaster(void) { if (server.masterhost == NULL) return; /* Nothing to do. */ sdsfree(server.masterhost); server.masterhost = NULL; - if (server.master) freeClient(server.master); + if (server.master) { + if (listLength(server.slaves) == 0) { + /* If this instance is turned into a master and there are no + * slaves, it inherits the replication offset from the master. + * Under certain conditions this makes replicas comparable by + * replication offset to understand what is the most updated. */ + server.master_repl_offset = server.master->reploff; + freeReplicationBacklog(); + } + freeClient(server.master); + } replicationDiscardCachedMaster(); cancelReplicationHandshake(); server.repl_state = REDIS_REPL_NONE; From c6db326d1d96c29d828f5b40cfd76896e6548775 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Dec 2013 11:31:35 +0100 Subject: [PATCH 0968/1752] Configuring port to 0 disables IP socket as specified. This was no longer the case with 2.8 becuase of a bug introduced with the IPv6 support. Now it is fixed. This fixes issue #1287 and #1477. --- src/redis.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/redis.c b/src/redis.c index 5fb75cd63af..0ad08e8c8e4 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1523,7 +1523,8 @@ void initServer() { server.db = zmalloc(sizeof(redisDb)*server.dbnum); /* Open the TCP listening socket for the user commands. */ - if (listenToPort(server.port,server.ipfd,&server.ipfd_count) == REDIS_ERR) + if (server.port != 0 && + listenToPort(server.port,server.ipfd,&server.ipfd_count) == REDIS_ERR) exit(1); /* Open the listening Unix domain socket. */ From 4ad219adc835ca39aa2bd7a618fb8b5bbc01086c Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 23 Dec 2013 12:48:39 +0100 Subject: [PATCH 0969/1752] Fix CONFIG REWRITE handling of unknown options. There were two problems with the implementation. 1) "save" was not correctly processed when no save point was configured, as reported in issue #1416. 2) The way the code checked if an option existed in the "processed" dictionary was wrong, as we add the element with as a key associated with a NULL value, so dictFetchValue() can't be used to check for existance, but dictFind() must be used, that returns NULL only if the entry does not exist at all. --- src/config.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/config.c b/src/config.c index b4f09440184..46da567584d 100644 --- a/src/config.c +++ b/src/config.c @@ -1416,6 +1416,8 @@ void rewriteConfigSaveOption(struct rewriteConfigState *state) { server.saveparams[j].seconds, server.saveparams[j].changes); rewriteConfigRewriteLine(state,"save",line,1); } + /* Mark "save" as processed in case server.saveparamslen is zero. */ + rewriteConfigMarkAsProcessed(state,"save"); } /* Rewrite the dir option, always using absolute paths.*/ @@ -1554,7 +1556,7 @@ void rewriteConfigRemoveOrphaned(struct rewriteConfigState *state) { /* Don't blank lines about options the rewrite process * don't understand. */ - if (dictFetchValue(state->rewritten,option) == NULL) { + if (dictFind(state->rewritten,option) == NULL) { redisLog(REDIS_DEBUG,"Not rewritten option: %s", option); continue; } From 9d1997706c6735c37afd2e67a2d315da54843c35 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Dec 2013 18:13:27 +0100 Subject: [PATCH 0970/1752] anetTcpGenericConnect() code improved + 1 bug fix. Now the socket is closed if anetNonBlock() fails, and in general the code structure makes it harder to introduce this kind of bugs in the future. Reference: pull request #1059. --- src/anet.c | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/anet.c b/src/anet.c index 257b491e915..45b31e78510 100644 --- a/src/anet.c +++ b/src/anet.c @@ -219,11 +219,11 @@ static int anetCreateSocket(char *err, int domain) { #define ANET_CONNECT_NONBLOCK 1 static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) { - int s, rv; - char _port[6]; /* strlen("65535"); */ + int s = ANET_ERR, rv; + char _port[6]; /* strlen("65535") + 1; */ struct addrinfo hints, *servinfo, *p; - snprintf(_port,6,"%d",port); + snprintf(_port,sizeof(port),"%d",port); memset(&hints,0,sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; @@ -233,20 +233,26 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) return ANET_ERR; } for (p = servinfo; p != NULL; p = p->ai_next) { + /* Try to create the socket and to connect it. + * If we fail in the socket() call, or on connect(), we retry with + * the next entry in servinfo. */ if ((s = socket(p->ai_family,p->ai_socktype,p->ai_protocol)) == -1) continue; - - /* if we set err then goto cleanup, otherwise next */ if (anetSetReuseAddr(err,s) == ANET_ERR) goto error; if (flags & ANET_CONNECT_NONBLOCK && anetNonBlock(err,s) != ANET_OK) goto error; if (connect(s,p->ai_addr,p->ai_addrlen) == -1) { - if (errno == EINPROGRESS && flags & ANET_CONNECT_NONBLOCK) goto end; + /* If the socket is non-blocking, it is ok for connect() to + * return an EINPROGRESS error here. */ + if (errno == EINPROGRESS && flags & ANET_CONNECT_NONBLOCK) + goto end; close(s); + s = ANET_ERR; continue; } - /* break with the socket */ + /* If we ended an iteration of the for loop without errors, we + * have a connected socket. Let's return to the caller. */ goto end; } if (p == NULL) { @@ -255,7 +261,10 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) } error: - s = ANET_ERR; + if (s != ANET_ERR) { + close(s); + s = ANET_ERR; + } end: freeaddrinfo(servinfo); return s; From d07d4a876c1e2803e90b29ff1a9f4f53022772fc Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Dec 2013 18:24:04 +0100 Subject: [PATCH 0971/1752] Remove useless goto from anetTcpGenericConnect(). --- src/anet.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/anet.c b/src/anet.c index 45b31e78510..75bebf7b1e2 100644 --- a/src/anet.c +++ b/src/anet.c @@ -255,10 +255,8 @@ static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) * have a connected socket. Let's return to the caller. */ goto end; } - if (p == NULL) { + if (p == NULL) anetSetError(err, "creating socket: %s", strerror(errno)); - goto error; - } error: if (s != ANET_ERR) { From 5b7c16137dda97ba9b8c6a88a9ca94fa40a54711 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Dec 2013 18:28:30 +0100 Subject: [PATCH 0972/1752] anetTcpGenericConnect() bug introduced in 9d19977 fixed. Durign a refactoring I mispelled _port for port. This is one of the reasons I never used _varname myself. --- src/anet.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/anet.c b/src/anet.c index 75bebf7b1e2..e37ae8064f0 100644 --- a/src/anet.c +++ b/src/anet.c @@ -220,15 +220,15 @@ static int anetCreateSocket(char *err, int domain) { static int anetTcpGenericConnect(char *err, char *addr, int port, int flags) { int s = ANET_ERR, rv; - char _port[6]; /* strlen("65535") + 1; */ + char portstr[6]; /* strlen("65535") + 1; */ struct addrinfo hints, *servinfo, *p; - snprintf(_port,sizeof(port),"%d",port); + snprintf(portstr,sizeof(portstr),"%d",port); memset(&hints,0,sizeof(hints)); hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; - if ((rv = getaddrinfo(addr,_port,&hints,&servinfo)) != 0) { + if ((rv = getaddrinfo(addr,portstr,&hints,&servinfo)) != 0) { anetSetError(err, "%s", gai_strerror(rv)); return ANET_ERR; } From 27d06111db5b33705d30170b88e5e1880445b87e Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Dec 2013 18:39:49 +0100 Subject: [PATCH 0973/1752] anetPeerToString / SockName: port can be NULL on errors too. --- src/anet.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/anet.c b/src/anet.c index e37ae8064f0..ef52d955d62 100644 --- a/src/anet.c +++ b/src/anet.c @@ -488,7 +488,7 @@ int anetPeerToString(int fd, char *ip, size_t ip_len, int *port) { socklen_t salen = sizeof(sa); if (getpeername(fd,(struct sockaddr*)&sa,&salen) == -1) { - *port = 0; + if (port) *port = 0; ip[0] = '?'; ip[1] = '\0'; return -1; @@ -510,7 +510,7 @@ int anetSockName(int fd, char *ip, size_t ip_len, int *port) { socklen_t salen = sizeof(sa); if (getsockname(fd,(struct sockaddr*)&sa,&salen) == -1) { - *port = 0; + if (port) *port = 0; ip[0] = '?'; ip[1] = '\0'; return -1; From 0a1a236e3e7773b76ba8aaccfd21eb77e7accce1 Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 25 Dec 2013 18:41:10 +0100 Subject: [PATCH 0974/1752] Log disconnection with slave only when ip:port is available. --- src/networking.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/networking.c b/src/networking.c index 5a692c79bcd..6bcbb66fb53 100644 --- a/src/networking.c +++ b/src/networking.c @@ -653,10 +653,10 @@ void freeClient(redisClient *c) { if (c->flags & REDIS_SLAVE) { char ip[REDIS_IP_STR_LEN]; - if (anetPeerToString(c->fd,ip,sizeof(ip),NULL) == -1) - strncpy(ip,"?",REDIS_IP_STR_LEN); - redisLog(REDIS_WARNING,"Connection with slave %s:%d lost.", - ip, c->slave_listening_port); + if (anetPeerToString(c->fd,ip,sizeof(ip),NULL) != -1) { + redisLog(REDIS_WARNING,"Connection with slave %s:%d lost.", + ip, c->slave_listening_port); + } } /* Free the query buffer */ From 418d3d358a1a09a59eec74bcc92649162585f8ec Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Jan 2014 14:11:02 +0100 Subject: [PATCH 0975/1752] Clarify a comment in slaveTryPartialResynchronization(). --- src/replication.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index b1d08c7749d..c480c9e82d0 100644 --- a/src/replication.c +++ b/src/replication.c @@ -991,7 +991,7 @@ int slaveTryPartialResynchronization(int fd) { /* If we reach this point we receied either an error since the master does * not understand PSYNC, or an unexpected reply from the master. - * Reply with PSYNC_NOT_SUPPORTED in both cases. */ + * Return PSYNC_NOT_SUPPORTED to the caller in both cases. */ if (strncmp(reply,"-ERR",4)) { /* If it's not an error, log the unexpected event. */ From 2a1a31ca9d41032af705c0de673ac984a73cb73a Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Jan 2014 14:25:06 +0100 Subject: [PATCH 0976/1752] Don't send REPLCONF ACK to old masters. Masters not understanding REPLCONF ACK will reply with errors to our requests causing a number of possible issues. This commit detects a global replication offest set to -1 at the end of the replication, and marks the client representing the master with the REDIS_PRE_PSYNC flag. Note that this flag was called REDIS_PRE_PSYNC_SLAVE but now it is just REDIS_PRE_PSYNC as it is used for both slaves and masters starting with this commit. This commit fixes issue #1488. --- src/redis.h | 2 +- src/replication.c | 15 +++++++++++---- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/redis.h b/src/redis.h index 7cba7ad304e..39ee45a0ffc 100644 --- a/src/redis.h +++ b/src/redis.h @@ -229,7 +229,7 @@ #define REDIS_MASTER_FORCE_REPLY (1<<13) /* Queue replies even if is master */ #define REDIS_FORCE_AOF (1<<14) /* Force AOF propagation of current cmd. */ #define REDIS_FORCE_REPL (1<<15) /* Force replication of current cmd. */ -#define REDIS_PRE_PSYNC_SLAVE (1<<16) /* Slave don't understand PSYNC. */ +#define REDIS_PRE_PSYNC (1<<16) /* Instance don't understand PSYNC. */ /* Client request types */ #define REDIS_REQ_INLINE 1 diff --git a/src/replication.c b/src/replication.c index c480c9e82d0..d585ff1f92d 100644 --- a/src/replication.c +++ b/src/replication.c @@ -458,7 +458,7 @@ void syncCommand(redisClient *c) { /* If a slave uses SYNC, we are dealing with an old implementation * of the replication protocol (like redis-cli --slave). Flag the client * so that we don't expect to receive REPLCONF ACK feedbacks. */ - c->flags |= REDIS_PRE_PSYNC_SLAVE; + c->flags |= REDIS_PRE_PSYNC; } /* Full resynchronization. */ @@ -829,6 +829,10 @@ void readSyncBulkPayload(aeEventLoop *el, int fd, void *privdata, int mask) { server.master->reploff = server.repl_master_initial_offset; memcpy(server.master->replrunid, server.repl_master_runid, sizeof(server.repl_master_runid)); + /* If master offset is set to -1, this master is old and is not + * PSYNC capable, so we flag it accordingly. */ + if (server.master->reploff == -1) + server.master->flags |= REDIS_PRE_PSYNC; redisLog(REDIS_NOTICE, "MASTER <-> SLAVE sync: Finished with success"); /* Restart the AOF subsystem now that we finished the sync. This * will trigger an AOF rewrite, and when done will start appending @@ -1554,8 +1558,11 @@ void replicationCron(void) { } } - /* Send ACK to master from time to time. */ - if (server.masterhost && server.master) + /* Send ACK to master from time to time. + * Note that we do not send periodic acks to masters that don't + * support PSYNC and replication offsets. */ + if (server.masterhost && server.master && + !(server.master->flags & REDIS_PRE_PSYNC)) replicationSendAck(); /* If we have attached slaves, PING them from time to time. @@ -1599,7 +1606,7 @@ void replicationCron(void) { redisClient *slave = ln->value; if (slave->replstate != REDIS_REPL_ONLINE) continue; - if (slave->flags & REDIS_PRE_PSYNC_SLAVE) continue; + if (slave->flags & REDIS_PRE_PSYNC) continue; if ((server.unixtime - slave->repl_ack_time) > server.repl_timeout) { char ip[REDIS_IP_STR_LEN]; From 05d219a6fa8619402b1dbb07980b1bdb708b557f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Jan 2014 17:16:04 +0100 Subject: [PATCH 0977/1752] Test: stress events flags to/from string conversion. --- tests/unit/pubsub.tcl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/pubsub.tcl b/tests/unit/pubsub.tcl index 90166885100..e2420830e58 100644 --- a/tests/unit/pubsub.tcl +++ b/tests/unit/pubsub.tcl @@ -356,4 +356,15 @@ start_server {tags {"pubsub"}} { r config set maxmemory 0 $rd1 close } + + test "Keyspace notifications: test CONFIG GET/SET of event flags" { + r config set notify-keyspace-events gKE + assert_equal {gKE} [lindex [r config get notify-keyspace-events] 1] + r config set notify-keyspace-events {$lshzxeKE} + assert_equal {$lshzxeKE} [lindex [r config get notify-keyspace-events] 1] + r config set notify-keyspace-events KA + assert_equal {AK} [lindex [r config get notify-keyspace-events] 1] + r config set notify-keyspace-events EA + assert_equal {AE} [lindex [r config get notify-keyspace-events] 1] + } } From 51f71e2d8ee96b7da8e710e16ab01cf69e4ec95f Mon Sep 17 00:00:00 2001 From: antirez Date: Wed, 8 Jan 2014 17:18:00 +0100 Subject: [PATCH 0978/1752] Fix keyspace events flags-to-string conversion. Fixes issue #1491 on Github. --- src/notify.c | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/notify.c b/src/notify.c index bb598c4f21c..f77239ecfca 100644 --- a/src/notify.c +++ b/src/notify.c @@ -67,17 +67,19 @@ int keyspaceEventsStringToFlags(char *classes) { sds keyspaceEventsFlagsToString(int flags) { sds res; - if ((flags & REDIS_NOTIFY_ALL) == REDIS_NOTIFY_ALL) - return sdsnew("A"); res = sdsempty(); - if (flags & REDIS_NOTIFY_GENERIC) res = sdscatlen(res,"g",1); - if (flags & REDIS_NOTIFY_STRING) res = sdscatlen(res,"$",1); - if (flags & REDIS_NOTIFY_LIST) res = sdscatlen(res,"l",1); - if (flags & REDIS_NOTIFY_SET) res = sdscatlen(res,"s",1); - if (flags & REDIS_NOTIFY_HASH) res = sdscatlen(res,"h",1); - if (flags & REDIS_NOTIFY_ZSET) res = sdscatlen(res,"z",1); - if (flags & REDIS_NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); - if (flags & REDIS_NOTIFY_EVICTED) res = sdscatlen(res,"e",1); + if ((flags & REDIS_NOTIFY_ALL) == REDIS_NOTIFY_ALL) { + res = sdscatlen(res,"A",1); + } else { + if (flags & REDIS_NOTIFY_GENERIC) res = sdscatlen(res,"g",1); + if (flags & REDIS_NOTIFY_STRING) res = sdscatlen(res,"$",1); + if (flags & REDIS_NOTIFY_LIST) res = sdscatlen(res,"l",1); + if (flags & REDIS_NOTIFY_SET) res = sdscatlen(res,"s",1); + if (flags & REDIS_NOTIFY_HASH) res = sdscatlen(res,"h",1); + if (flags & REDIS_NOTIFY_ZSET) res = sdscatlen(res,"z",1); + if (flags & REDIS_NOTIFY_EXPIRED) res = sdscatlen(res,"x",1); + if (flags & REDIS_NOTIFY_EVICTED) res = sdscatlen(res,"e",1); + } if (flags & REDIS_NOTIFY_KEYSPACE) res = sdscatlen(res,"K",1); if (flags & REDIS_NOTIFY_KEYEVENT) res = sdscatlen(res,"E",1); return res; From e4c51759f5d7dedd84a539e4cf5340e0043e11d2 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Jan 2014 11:09:23 +0100 Subject: [PATCH 0979/1752] Fix RESTORE ttl handling in 32 bit archs. long was used instead of long long in order to handle a 64 bit resolution millisecond timestamp. This fixes issue #1483. --- src/migrate.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/migrate.c b/src/migrate.c index 94aa1e40db6..3cc482fa6e6 100644 --- a/src/migrate.c +++ b/src/migrate.c @@ -84,7 +84,7 @@ void dumpCommand(redisClient *c) { /* RESTORE key ttl serialized-value */ void restoreCommand(redisClient *c) { - long ttl; + long long ttl; rio payload; int type; robj *obj; @@ -96,7 +96,7 @@ void restoreCommand(redisClient *c) { } /* Check if the TTL value makes sense */ - if (getLongFromObjectOrReply(c,c->argv[2],&ttl,NULL) != REDIS_OK) { + if (getLongLongFromObjectOrReply(c,c->argv[2],&ttl,NULL) != REDIS_OK) { return; } else if (ttl < 0) { addReplyError(c,"Invalid TTL value, must be >= 0"); @@ -104,7 +104,8 @@ void restoreCommand(redisClient *c) { } /* Verify RDB version and data checksum. */ - if (verifyDumpPayload(c->argv[3]->ptr,sdslen(c->argv[3]->ptr)) == REDIS_ERR) { + if (verifyDumpPayload(c->argv[3]->ptr,sdslen(c->argv[3]->ptr)) == REDIS_ERR) + { addReplyError(c,"DUMP payload version or checksum are wrong"); return; } From bca65866ff1e59d510c9a7bbd1947f659605d302 Mon Sep 17 00:00:00 2001 From: antirez Date: Thu, 9 Jan 2014 11:19:03 +0100 Subject: [PATCH 0980/1752] Test: regression for issues #1483. --- tests/unit/dump.tcl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/unit/dump.tcl b/tests/unit/dump.tcl index be891a96f95..b79c3ba9d0b 100644 --- a/tests/unit/dump.tcl +++ b/tests/unit/dump.tcl @@ -16,6 +16,16 @@ start_server {tags {"dump"}} { r get foo } {bar} + test {RESTORE can set an expire that overflows a 32 bit integer} { + r set foo bar + set encoded [r dump foo] + r del foo + r restore foo 2569591501 $encoded + set ttl [r pttl foo] + assert {$ttl >= (2569591501-3000) && $ttl <= 2569591501} + r get foo + } {bar} + test {RESTORE returns an error of the key already exists} { r set foo bar set e {} From ea2bffa03012bc95f6312761fe1a0bed8edd49f9 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 14:13:04 +0100 Subject: [PATCH 0981/1752] Trip comment to 80 cols in SentinelCommand(). --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 30188e3e55a..6edeacb77e8 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2375,7 +2375,7 @@ void sentinelCommand(redisClient *c) { return; addReplyDictOfRedisInstances(c,ri->sentinels); } else if (!strcasecmp(c->argv[1]->ptr,"is-master-down-by-addr")) { - /* SENTINEL IS-MASTER-DOWN-BY-ADDR */ + /* SENTINEL IS-MASTER-DOWN-BY-ADDR */ sentinelRedisInstance *ri; long long req_epoch; uint64_t leader_epoch = 0; From 6bcc370c9d0dcabc605ff03ccae6abccff7aceff Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 14:31:41 +0100 Subject: [PATCH 0982/1752] Add all the configurable fields to addReplySentinelRedisInstance(). Note: the auth password with the master is voluntarily not exposed. --- src/sentinel.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 6edeacb77e8..584ebf09dcb 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2270,6 +2270,30 @@ void addReplySentinelRedisInstance(redisClient *c, sentinelRedisInstance *ri) { addReplyBulkCString(c,"quorum"); addReplyBulkLongLong(c,ri->quorum); fields++; + + addReplyBulkCString(c,"down-after-milliseconds"); + addReplyBulkLongLong(c,ri->down_after_period); + fields++; + + addReplyBulkCString(c,"failover-timeout"); + addReplyBulkLongLong(c,ri->failover_timeout); + fields++; + + addReplyBulkCString(c,"parallel-syncs"); + addReplyBulkLongLong(c,ri->parallel_syncs); + fields++; + + if (ri->notification_script) { + addReplyBulkCString(c,"notification-script"); + addReplyBulkCString(c,ri->notification_script); + fields++; + } + + if (ri->client_reconfig_script) { + addReplyBulkCString(c,"client-reconfig-script"); + addReplyBulkCString(c,ri->client_reconfig_script); + fields++; + } } /* Only slaves */ From 52cf0975c780a02b720233c306b9c7e08d169b47 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 14:41:44 +0100 Subject: [PATCH 0983/1752] Sentinel: added SENTINEL MASTER command. With SENTINEL MASTERS it was already possible to list all the configured masters, but not a specific one. --- src/sentinel.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 584ebf09dcb..13d949e5c9b 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2380,8 +2380,14 @@ void sentinelCommand(redisClient *c) { if (!strcasecmp(c->argv[1]->ptr,"masters")) { /* SENTINEL MASTERS */ if (c->argc != 2) goto numargserr; - addReplyDictOfRedisInstances(c,sentinel.masters); + } else if (!strcasecmp(c->argv[1]->ptr,"master")) { + /* SENTINEL MASTER */ + sentinelRedisInstance *ri; + + if ((ri = sentinelGetMasterByNameOrReplyError(c,c->argv[2])) + == NULL) return; + addReplySentinelRedisInstance(c,ri); } else if (!strcasecmp(c->argv[1]->ptr,"slaves")) { /* SENTINEL SLAVES */ sentinelRedisInstance *ri; From 2e0ba7bb5b13af2c9d4bde40ae11d83f727fd9fe Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 15:02:39 +0100 Subject: [PATCH 0984/1752] anetResolveIP() added to anet.c. The new function is used when we want to normalize an IP address without performing a DNS lookup if the string to resolve is not a valid IP. This is useful every time only IPs are valid inputs or when we want to skip DNS resolution that is slow during runtime operations if we are required to block. --- src/anet.c | 19 ++++++++++++++++++- src/anet.h | 4 ++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/anet.c b/src/anet.c index ef52d955d62..fc585c8d129 100644 --- a/src/anet.c +++ b/src/anet.c @@ -163,12 +163,21 @@ int anetTcpKeepAlive(char *err, int fd) return ANET_OK; } -int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) +/* anetGenericResolve() is called by anetResolve() and anetResolveIP() to + * do the actual work. It resolves the hostname "host" and set the string + * representation of the IP address into the buffer pointed by "ipbuf". + * + * If flags is set to ANET_IP_ONLY the function only resolves hostnames + * that are actually already IPv4 or IPv6 addresses. This turns the function + * into a validating / normalizing function. */ +int anetGenericResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len, + int flags) { struct addrinfo hints, *info; int rv; memset(&hints,0,sizeof(hints)); + if (flags & ANET_IP_ONLY) hints.ai_flags = AI_NUMERICHOST; hints.ai_family = AF_UNSPEC; hints.ai_socktype = SOCK_STREAM; /* specify socktype to avoid dups */ @@ -188,6 +197,14 @@ int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) return ANET_OK; } +int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len) { + return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_NONE); +} + +int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len) { + return anetGenericResolve(err,host,ipbuf,ipbuf_len,ANET_IP_ONLY); +} + static int anetSetReuseAddr(char *err, int fd) { int yes = 1; /* Make sure connection-intensive things like the redis benckmark diff --git a/src/anet.h b/src/anet.h index b23411cbbe3..f0ab63ab73c 100644 --- a/src/anet.h +++ b/src/anet.h @@ -35,6 +35,10 @@ #define ANET_ERR -1 #define ANET_ERR_LEN 256 +/* Flags used with certain functions. */ +#define ANET_NONE 0 +#define ANET_IP_ONLY (1<<0) + #if defined(__sun) #define AF_LOCAL AF_UNIX #endif From b7dc3204a66dc3e3f272c5ebfd00669aa2bdaf88 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 15:16:55 +0100 Subject: [PATCH 0985/1752] Sentinel: SENTINEL MONITOR command implemented. It allows to add new masters to monitor at runtime. --- src/sentinel.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 13d949e5c9b..22f895966e3 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2488,6 +2488,40 @@ void sentinelCommand(redisClient *c) { if (c->argc != 2) goto numargserr; sentinelPendingScriptsCommand(c); + } else if (!strcasecmp(c->argv[1]->ptr,"monitor")) { + /* SENTINEL MONITOR */ + long quorum, port; + char buf[32]; + + if (c->argc != 6) goto numargserr; + if (getLongFromObjectOrReply(c,c->argv[5],&quorum,"Invalid quorum") + != REDIS_OK) return; + if (getLongFromObjectOrReply(c,c->argv[4],&port,"Invalid port") + != REDIS_OK) return; + /* Make sure the IP field is actually a valid IP before passing it + * to createSentinelRedisInstance(), otherwise we may trigger a + * DNS lookup at runtime. */ + if (anetResolveIP(NULL,c->argv[3]->ptr,buf,sizeof(buf)) == ANET_ERR) { + addReplyError(c,"Invalid IP address specified"); + return; + } + if (createSentinelRedisInstance(c->argv[2]->ptr,SRI_MASTER, + c->argv[3]->ptr,port,quorum,NULL) == NULL) + { + switch(errno) { + case EBUSY: + addReplyError(c,"Duplicated master name"); + break; + case EINVAL: + addReplyError(c,"Invalid port number"); + break; + default: + addReplyError(c,"Unspecified error adding the instance"); + break; + } + } else { + addReply(c,shared.ok); + } } else { addReplyErrorFormat(c,"Unknown sentinel subcommand '%s'", (char*)c->argv[1]->ptr); From bb207007cad63752e6e2b367fcaea17db3e8d4ae Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 15:18:41 +0100 Subject: [PATCH 0986/1752] anetResolveIP() prototype added to anet.h. --- src/anet.h | 1 + 1 file changed, 1 insertion(+) diff --git a/src/anet.h b/src/anet.h index f0ab63ab73c..2ab9398adc4 100644 --- a/src/anet.h +++ b/src/anet.h @@ -49,6 +49,7 @@ int anetUnixConnect(char *err, char *path); int anetUnixNonBlockConnect(char *err, char *path); int anetRead(int fd, char *buf, int count); int anetResolve(char *err, char *host, char *ipbuf, size_t ipbuf_len); +int anetResolveIP(char *err, char *host, char *ipbuf, size_t ipbuf_len); int anetTcpServer(char *err, int port, char *bindaddr); int anetTcp6Server(char *err, int port, char *bindaddr); int anetUnixServer(char *err, char *path, mode_t perm); From 1f9580c125e1a332c19a381bf496b4b63ed4dd2b Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 15:22:06 +0100 Subject: [PATCH 0987/1752] Sentinel: flush config on disk when new master is added. --- src/sentinel.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sentinel.c b/src/sentinel.c index 22f895966e3..611c31f3599 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2520,6 +2520,7 @@ void sentinelCommand(redisClient *c) { break; } } else { + sentinelFlushConfig(); addReply(c,shared.ok); } } else { From 0ca750d94add42009ba1d1b645fa23c9dcf3796d Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 15:33:42 +0100 Subject: [PATCH 0988/1752] Sentinel: releaseSentinelRedisInstance() top comment fixed. The claim about unlinking the instance from the connected hash tables was the opposite of the reality. Also the current actual behavior is safer in most cases, so it is better to manually unlink when needed. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index 611c31f3599..3d1477c97e4 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -942,7 +942,7 @@ sentinelRedisInstance *createSentinelRedisInstance(char *name, int flags, char * } /* Release this instance and all its slaves, sentinels, hiredis connections. - * This function also takes care of unlinking the instance from the main + * This function does not take care of unlinking the instance from the main * masters table (if it is a master) or from its master sentinels/slaves table * if it is a slave or sentinel. */ void releaseSentinelRedisInstance(sentinelRedisInstance *ri) { From 560e548dc6b13413c4b6bacbcf42feae221b09c8 Mon Sep 17 00:00:00 2001 From: antirez Date: Fri, 10 Jan 2014 15:39:10 +0100 Subject: [PATCH 0989/1752] Sentinel: SENTINEL REMOVE command added. The command totally removes a monitored master. --- src/sentinel.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 3d1477c97e4..ee6ec17c251 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2523,6 +2523,15 @@ void sentinelCommand(redisClient *c) { sentinelFlushConfig(); addReply(c,shared.ok); } + } else if (!strcasecmp(c->argv[1]->ptr,"remove")) { + /* SENTINEL REMOVE */ + sentinelRedisInstance *ri; + + if ((ri = sentinelGetMasterByNameOrReplyError(c,c->argv[2])) + == NULL) return; + dictDelete(sentinel.masters,c->argv[2]->ptr); + sentinelFlushConfig(); + addReply(c,shared.ok); } else { addReplyErrorFormat(c,"Unknown sentinel subcommand '%s'", (char*)c->argv[1]->ptr); From 8ed0b49525b8e0c790317b076330a8ead24f90c4 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2014 11:05:13 +0100 Subject: [PATCH 0990/1752] Sentinel: fix wrong arity error message. --- src/sentinel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sentinel.c b/src/sentinel.c index ee6ec17c251..940cc0b11f3 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -2539,7 +2539,7 @@ void sentinelCommand(redisClient *c) { return; numargserr: - addReplyErrorFormat(c,"Wrong number of commands for 'sentinel %s'", + addReplyErrorFormat(c,"Wrong number of arguments for 'sentinel %s'", (char*)c->argv[1]->ptr); } From b8db8a0c4812b5db9573b19a9b05b4182dc94561 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 13 Jan 2014 11:50:38 +0100 Subject: [PATCH 0991/1752] SENTINEL SET implemented. The new command allows to change master-specific configurations at runtime. All the settable parameters can be retrivied via the SENTINEL MASTER command, so there is no equivalent "GET" command. --- src/sentinel.c | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/sentinel.c b/src/sentinel.c index 940cc0b11f3..3658c616284 100644 --- a/src/sentinel.c +++ b/src/sentinel.c @@ -370,6 +370,7 @@ dictType leaderVotesDictType = { void sentinelCommand(redisClient *c); void sentinelInfoCommand(redisClient *c); +void sentinelSetCommand(redisClient *c); struct redisCommand sentinelcmds[] = { {"ping",pingCommand,1,"",0,NULL,0,0,0,0,0}, @@ -2532,6 +2533,9 @@ void sentinelCommand(redisClient *c) { dictDelete(sentinel.masters,c->argv[2]->ptr); sentinelFlushConfig(); addReply(c,shared.ok); + } else if (!strcasecmp(c->argv[1]->ptr,"set")) { + if (c->argc < 3 || c->argc % 2 == 0) goto numargserr; + sentinelSetCommand(c); } else { addReplyErrorFormat(c,"Unknown sentinel subcommand '%s'", (char*)c->argv[1]->ptr); @@ -2543,6 +2547,7 @@ void sentinelCommand(redisClient *c) { (char*)c->argv[1]->ptr); } +/* SENTINEL INFO [section] */ void sentinelInfoCommand(redisClient *c) { char *section = c->argc == 2 ? c->argv[1]->ptr : "default"; sds info = sdsempty(); @@ -2602,6 +2607,79 @@ void sentinelInfoCommand(redisClient *c) { addReply(c,shared.crlf); } +/* SENTINEL SET [

DEBUGGING MALLOC PROBLEMS

When debugging, it is a good idea to configure/build jemalloc with the --enable-debug and --enable-fill options, and recompile the program with suitable options and symbols for debugger support. When so configured, jemalloc incorporates a wide variety @@ -1308,7 +1390,7 @@ it detects, because the performance impact for storing such information would be prohibitive. However, jemalloc does integrate with the most excellent Valgrind tool if the - --enable-valgrind configuration option is enabled.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an + --enable-valgrind configuration option is enabled.

DIAGNOSTIC MESSAGES

If any of the memory allocation/deallocation functions detect an error or warning condition, a message will be printed to file descriptor STDERR_FILENO. Errors will result in the process dumping core. If the @@ -1324,7 +1406,7 @@ malloc_stats_print(), followed by a string pointer. Please note that doing anything which tries to allocate memory in this function is likely to result in a crash or deadlock.

All messages are prefixed by - “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and + “<jemalloc>: ”.

RETURN VALUES

Standard API

The malloc() and calloc() functions return a pointer to the allocated memory if successful; otherwise a NULL pointer is returned and errno is set to @@ -1332,7 +1414,7 @@ returns the value 0 if successful; otherwise it returns an error value. The posix_memalign() function will fail if: -

EINVAL

The alignment parameter is +

EINVAL

The alignment parameter is not a power of 2 at least as large as sizeof(void *).

ENOMEM

Memory allocation error.

@@ -1341,7 +1423,7 @@ NULL pointer is returned and errno is set. The aligned_alloc() function will fail if: -

EINVAL

The alignment parameter is +

EINVAL

The alignment parameter is not a power of 2.

ENOMEM

Memory allocation error.

The realloc() function returns a @@ -1352,26 +1434,38 @@ allocation failure. The realloc() function always leaves the original buffer intact when an error occurs.

The free() function returns no - value.

Non-standard API

The malloc_usable_size() function - returns the usable size of the allocation pointed to by - ptr.

The mallctl(), + value.

Non-standard API

The mallocx() and + rallocx() functions return a pointer to + the allocated memory if successful; otherwise a NULL + pointer is returned to indicate insufficient contiguous memory was + available to service the allocation request.

The xallocx() function returns the + real size of the resulting resized allocation pointed to by + ptr, which is a value less than + size if the allocation could not be adequately + grown in place.

The sallocx() function returns the + real size of the allocation pointed to by ptr. +

The nallocx() returns the real size + that would result from a successful equivalent + mallocx() function call, or zero if + insufficient memory is available to perform the size computation.

The mallctl(), mallctlnametomib(), and mallctlbymib() functions return 0 on success; otherwise they return an error value. The functions will fail if: -

EINVAL

newp is not +

EINVAL

newp is not NULL, and newlen is too large or too small. Alternatively, *oldlenp is too large or too small; in this case as much data as possible - are read despite the error.

ENOMEM

*oldlenp is too short to - hold the requested value.

ENOENT

name or + are read despite the error.

ENOENT

name or mib specifies an unknown/invalid value.

EPERM

Attempt to read or write void value, or attempt to write read-only value.

EAGAIN

A memory allocation failure occurred.

EFAULT

An interface with side effects failed in some way not directly related to mallctl*() read/write processing.

-

Experimental API

The allocm(), +

The malloc_usable_size() function + returns the usable size of the allocation pointed to by + ptr.

Experimental API

The allocm(), rallocm(), sallocm(), dallocm(), and @@ -1380,7 +1474,7 @@ error value. The allocm(), rallocm(), and nallocm() functions will fail if: -

ALLOCM_ERR_OOM

Out of memory. Insufficient contiguous memory was +

ALLOCM_ERR_OOM

Out of memory. Insufficient contiguous memory was available to service the allocation request. The allocm() function additionally sets *ptr to NULL, whereas @@ -1388,25 +1482,25 @@ *ptr unmodified.

The rallocm() function will also fail if: -

ALLOCM_ERR_NOT_MOVED

ALLOCM_NO_MOVE was specified, +

ALLOCM_ERR_NOT_MOVED

ALLOCM_NO_MOVE was specified, but the reallocation request could not be serviced without moving the object.

-

ENVIRONMENT

The following environment variable affects the execution of the +

ENVIRONMENT

The following environment variable affects the execution of the allocation functions: -

MALLOC_CONF

If the environment variable +

MALLOC_CONF

If the environment variable MALLOC_CONF is set, the characters it contains will be interpreted as options.

-

EXAMPLES

To dump core whenever a problem occurs: +

EXAMPLES

To dump core whenever a problem occurs: 

ln -s 'abort:true' /etc/malloc.conf

To specify in the source a chunk size that is 16 MiB: 

-malloc_conf = "lg_chunk:24";

SEE ALSO

madvise(2), +malloc_conf = "lg_chunk:24";

SEE ALSO

madvise(2), mmap(2), sbrk(2), utrace(2), alloca(3), atexit(3), - getpagesize(3)

STANDARDS

The malloc(), + getpagesize(3)

STANDARDS

The malloc(), calloc(), realloc(), and free() functions conform to ISO/IEC diff --git a/deps/jemalloc/doc/jemalloc.xml.in b/deps/jemalloc/doc/jemalloc.xml.in index 54b87474c84..d8e2e711f2c 100644 --- a/deps/jemalloc/doc/jemalloc.xml.in +++ b/deps/jemalloc/doc/jemalloc.xml.in @@ -33,11 +33,17 @@ aligned_alloc realloc free - malloc_usable_size - malloc_stats_print + mallocx + rallocx + xallocx + sallocx + dallocx + nallocx mallctl mallctlnametomib mallctlbymib + malloc_stats_print + malloc_usable_size allocm rallocm sallocm @@ -92,16 +98,37 @@ Non-standard API - size_t malloc_usable_size - const void *ptr + void *mallocx + size_t size + int flags - void malloc_stats_print - void (*write_cb) - void *, const char * - - void *cbopaque - const char *opts + void *rallocx + void *ptr + size_t size + int flags + + + size_t xallocx + void *ptr + size_t size + size_t extra + int flags + + + size_t sallocx + void *ptr + int flags + + + void dallocx + void *ptr + int flags + + + size_t nallocx + size_t size + int flags int mallctl @@ -126,6 +153,18 @@ void *newp size_t newlen + + void malloc_stats_print + void (*write_cb) + void *, const char * + + void *cbopaque + const char *opts + + + size_t malloc_usable_size + const void *ptr + void (*malloc_message) void *cbopaque @@ -225,42 +264,103 @@ Non-standard API + The mallocx, + rallocx, + xallocx, + sallocx, + dallocx, and + nallocx functions all have a + flags argument that can be used to specify + options. The functions only check the options that are contextually + relevant. Use bitwise or (|) operations to + specify one or more of the following: + + + MALLOCX_LG_ALIGN(la) + - The malloc_usable_size function - returns the usable size of the allocation pointed to by - ptr. The return value may be larger than the size - that was requested during allocation. The - malloc_usable_size function is not a - mechanism for in-place realloc; rather - it is provided solely as a tool for introspection purposes. Any - discrepancy between the requested allocation size and the size reported - by malloc_usable_size should not be - depended on, since such behavior is entirely implementation-dependent. - + Align the memory allocation to start at an address + that is a multiple of (1 << + la). This macro does not validate + that la is within the valid + range. + + + MALLOCX_ALIGN(a) + - The malloc_stats_print function - writes human-readable summary statistics via the - write_cb callback function pointer and - cbopaque data passed to - write_cb, or - malloc_message if - write_cb is NULL. This - function can be called repeatedly. General information that never - changes during execution can be omitted by specifying "g" as a character - within the opts string. Note that - malloc_message uses the - mallctl* functions internally, so - inconsistent statistics can be reported if multiple threads use these - functions simultaneously. If is - specified during configuration, “m” and “a” can - be specified to omit merged arena and per arena statistics, respectively; - “b” and “l” can be specified to omit per size - class statistics for bins and large objects, respectively. Unrecognized - characters are silently ignored. Note that thread caching may prevent - some statistics from being completely up to date, since extra locking - would be required to merge counters that track thread cache operations. + Align the memory allocation to start at an address + that is a multiple of a, where + a is a power of two. This macro does not + validate that a is a power of 2. + + + + MALLOCX_ZERO + + Initialize newly allocated memory to contain zero + bytes. In the growing reallocation case, the real size prior to + reallocation defines the boundary between untouched bytes and those + that are initialized to contain zero bytes. If this macro is + absent, newly allocated memory is uninitialized. + + + MALLOCX_ARENA(a) + + + Use the arena specified by the index + a (and by necessity bypass the thread + cache). This macro has no effect for huge regions, nor for regions + that were allocated via an arena other than the one specified. + This macro does not validate that a + specifies an arena index in the valid range. + + + The mallocx function allocates at + least size bytes of memory, and returns a pointer + to the base address of the allocation. Behavior is undefined if + size is 0, or if request size + overflows due to size class and/or alignment constraints. + + The rallocx function resizes the + allocation at ptr to be at least + size bytes, and returns a pointer to the base + address of the resulting allocation, which may or may not have moved from + its original location. Behavior is undefined if + size is 0, or if request size + overflows due to size class and/or alignment constraints. + + The xallocx function resizes the + allocation at ptr in place to be at least + size bytes, and returns the real size of the + allocation. If extra is non-zero, an attempt is + made to resize the allocation to be at least (size + + extra) bytes, though inability to allocate + the extra byte(s) will not by itself result in failure to resize. + Behavior is undefined if size is + 0, or if (size + extra + > SIZE_T_MAX). + + The sallocx function returns the + real size of the allocation at ptr. + + The dallocx function causes the + memory referenced by ptr to be made available for + future allocations. + + The nallocx function allocates no + memory, but it performs the same size computation as the + mallocx function, and returns the real + size of the allocation that would result from the equivalent + mallocx function call. Behavior is + undefined if size is 0, or if + request size overflows due to size class and/or alignment + constraints. + The mallctl function provides a general interface for introspecting the memory allocator, as well as setting modifiable parameters and triggering actions. The @@ -297,15 +397,14 @@ it is legitimate to construct code like the following: + + The malloc_stats_print function + writes human-readable summary statistics via the + write_cb callback function pointer and + cbopaque data passed to + write_cb, or + malloc_message if + write_cb is NULL. This + function can be called repeatedly. General information that never + changes during execution can be omitted by specifying "g" as a character + within the opts string. Note that + malloc_message uses the + mallctl* functions internally, so + inconsistent statistics can be reported if multiple threads use these + functions simultaneously. If is + specified during configuration, “m” and “a” can + be specified to omit merged arena and per arena statistics, respectively; + “b” and “l” can be specified to omit per size + class statistics for bins and large objects, respectively. Unrecognized + characters are silently ignored. Note that thread caching may prevent + some statistics from being completely up to date, since extra locking + would be required to merge counters that track thread cache operations. + + + The malloc_usable_size function + returns the usable size of the allocation pointed to by + ptr. The return value may be larger than the size + that was requested during allocation. The + malloc_usable_size function is not a + mechanism for in-place realloc; rather + it is provided solely as a tool for introspection purposes. Any + discrepancy between the requested allocation size and the size reported + by malloc_usable_size should not be + depended on, since such behavior is entirely implementation-dependent. + Experimental API @@ -358,7 +492,7 @@ for (i = 0; i < nbins; i++) { Initialize newly allocated memory to contain zero bytes. In the growing reallocation case, the real size prior to reallocation defines the boundary between untouched bytes and those - that are initialized to contain zero bytes. If this option is + that are initialized to contain zero bytes. If this macro is absent, newly allocated memory is uninitialized. @@ -373,9 +507,11 @@ for (i = 0; i < nbins; i++) { Use the arena specified by the index - a. This macro does not validate that - a specifies an arena in the valid - range. + a (and by necessity bypass the thread + cache). This macro has no effect for huge regions, nor for regions + that were allocated via an arena other than the one specified. + This macro does not validate that a + specifies an arena index in the valid range. @@ -385,8 +521,9 @@ for (i = 0; i < nbins; i++) { *ptr to the base address of the allocation, and sets *rsize to the real size of the allocation if rsize is not NULL. Behavior - is undefined if size is - 0. + is undefined if size is 0, or + if request size overflows due to size class and/or alignment + constraints. The rallocm function resizes the allocation at *ptr to be at least @@ -396,11 +533,12 @@ for (i = 0; i < nbins; i++) { rsize is not NULL. If extra is non-zero, an attempt is made to resize the allocation to be at least size + + language="C">(size + extra) bytes, though inability to allocate the extra byte(s) will not by itself result in failure. Behavior is - undefined if size is 0, or if - (size + + undefined if size is 0, if + request size overflows due to size class and/or alignment constraints, or + if (size + extra > SIZE_T_MAX). @@ -417,8 +555,9 @@ for (i = 0; i < nbins; i++) { rsize is not NULL it sets *rsize to the real size of the allocation that would result from the equivalent allocm - function call. Behavior is undefined if - size is 0. + function call. Behavior is undefined if size is + 0, or if request size overflows due to size class + and/or alignment constraints. @@ -432,7 +571,14 @@ for (i = 0; i < nbins; i++) { referenced by the symbolic link named /etc/malloc.conf, and the value of the environment variable MALLOC_CONF, will be interpreted, in - that order, from left to right as options. + that order, from left to right as options. Note that + malloc_conf may be read before + main is entered, so the declaration of + malloc_conf should specify an initializer that contains + the final value to be read by jemalloc. malloc_conf is + a compile-time setting, whereas /etc/malloc.conf and MALLOC_CONF + can be safely set any time prior to program invocation. An options string is a comma-separated list of option:value pairs. There is one key corresponding to each - + version (const char *) @@ -619,7 +765,7 @@ for (i = 0; i < nbins; i++) { detecting whether another thread caused a refresh. - + config.debug (bool) @@ -629,7 +775,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.dss (bool) @@ -639,7 +785,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.fill (bool) @@ -649,7 +795,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.lazy_lock (bool) @@ -659,7 +805,7 @@ for (i = 0; i < nbins; i++) { during build configuration. - + config.mremap (bool) @@ -669,7 +815,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.munmap (bool) @@ -679,7 +825,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.prof (bool) @@ -689,7 +835,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.prof_libgcc (bool) @@ -699,7 +845,7 @@ for (i = 0; i < nbins; i++) { specified during build configuration. - + config.prof_libunwind (bool) @@ -709,7 +855,7 @@ for (i = 0; i < nbins; i++) { during build configuration. - + config.stats (bool) @@ -719,7 +865,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.tcache (bool) @@ -729,7 +875,7 @@ for (i = 0; i < nbins; i++) { during build configuration. - + config.tls (bool) @@ -739,7 +885,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.utrace (bool) @@ -749,7 +895,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.valgrind (bool) @@ -759,7 +905,7 @@ for (i = 0; i < nbins; i++) { build configuration. - + config.xmalloc (bool) @@ -784,16 +930,6 @@ for (i = 0; i < nbins; i++) { - - - opt.lg_chunk - (size_t) - r- - - Virtual memory chunk size (log base 2). The default - chunk size is 4 MiB (2^22). - - opt.dss @@ -805,7 +941,23 @@ for (i = 0; i < nbins; i++) { related to mmap 2 allocation. The following settings are supported: “disabled”, “primary”, - and “secondary” (default). + and “secondary”. The default is “secondary” if + config.dss is + true, “disabled” otherwise. + + + + + + opt.lg_chunk + (size_t) + r- + + Virtual memory chunk size (log base 2). If a chunk + size outside the supported size range is specified, the size is + silently clipped to the minimum/maximum supported size. The default + chunk size is 4 MiB (2^22). + @@ -924,7 +1076,8 @@ for (i = 0; i < nbins; i++) { Zero filling enabled/disabled. If enabled, each byte of uninitialized allocated memory will be initialized to 0. Note that this initialization only happens once for each byte, so - realloc and + realloc, + rallocx and rallocm calls do not zero memory that was previously allocated. This is intended for debugging and will impact performance negatively. This option is disabled by default. @@ -1053,7 +1206,7 @@ malloc_conf = "xmalloc:true";]]> opt.prof_active (bool) - r- + rw [] Profiling activated/deactivated. This is a secondary @@ -1165,7 +1318,7 @@ malloc_conf = "xmalloc:true";]]> by default. - + thread.arena (unsigned) @@ -1192,7 +1345,7 @@ malloc_conf = "xmalloc:true";]]> cases. - + thread.allocatedp (uint64_t *) @@ -1219,7 +1372,7 @@ malloc_conf = "xmalloc:true";]]> cases. - + thread.deallocatedp (uint64_t *) @@ -1233,7 +1386,7 @@ malloc_conf = "xmalloc:true";]]> mallctl* calls. - + thread.tcache.enabled (bool) @@ -1247,7 +1400,7 @@ malloc_conf = "xmalloc:true";]]> - + thread.tcache.flush (void) @@ -1286,8 +1439,12 @@ malloc_conf = "xmalloc:true";]]> Set the precedence of dss allocation as related to mmap allocation for arena <i>, or for all arenas if <i> equals arenas.narenas. See - opt.dss for supported + linkend="arenas.narenas">arenas.narenas. Note + that even during huge allocation this setting is read from the arena + that would be chosen for small or large allocation so that applications + can depend on consistent dss versus mmap allocation regardless of + allocation size. See opt.dss for supported settings. @@ -1313,7 +1470,7 @@ malloc_conf = "xmalloc:true";]]> initialized. - + arenas.quantum (size_t) @@ -1322,7 +1479,7 @@ malloc_conf = "xmalloc:true";]]> Quantum size. - + arenas.page (size_t) @@ -1331,7 +1488,7 @@ malloc_conf = "xmalloc:true";]]> Page size. - + arenas.tcache_max (size_t) @@ -1341,7 +1498,7 @@ malloc_conf = "xmalloc:true";]]> Maximum thread-cached size class. - + arenas.nbins (unsigned) @@ -1350,7 +1507,7 @@ malloc_conf = "xmalloc:true";]]> Number of bin size classes. - + arenas.nhbins (unsigned) @@ -1370,7 +1527,7 @@ malloc_conf = "xmalloc:true";]]> Maximum size supported by size class. - + arenas.bin.<i>.nregs (uint32_t) @@ -1379,7 +1536,7 @@ malloc_conf = "xmalloc:true";]]> Number of regions per page run. - + arenas.bin.<i>.run_size (size_t) @@ -1388,7 +1545,7 @@ malloc_conf = "xmalloc:true";]]> Number of bytes per page run. - + arenas.nlruns (size_t) @@ -1397,7 +1554,7 @@ malloc_conf = "xmalloc:true";]]> Total number of large size classes. - + arenas.lrun.<i>.size (size_t) @@ -1407,7 +1564,7 @@ malloc_conf = "xmalloc:true";]]> class. - + arenas.purge (unsigned) @@ -1417,7 +1574,7 @@ malloc_conf = "xmalloc:true";]]> for all arenas if none is specified. - + arenas.extend (unsigned) @@ -1441,7 +1598,7 @@ malloc_conf = "xmalloc:true";]]> - + prof.dump (const char *) @@ -1457,7 +1614,7 @@ malloc_conf = "xmalloc:true";]]> option. - + prof.interval (uint64_t) @@ -1517,7 +1674,7 @@ malloc_conf = "xmalloc:true";]]> entirely devoted to allocator metadata. - + stats.mapped (size_t) @@ -1531,7 +1688,7 @@ malloc_conf = "xmalloc:true";]]> does not include inactive chunks. - + stats.chunks.current (size_t) @@ -1543,7 +1700,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.chunks.total (uint64_t) @@ -1553,7 +1710,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of chunks allocated. - + stats.chunks.high (size_t) @@ -1564,7 +1721,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.huge.allocated (size_t) @@ -1575,7 +1732,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.huge.nmalloc (uint64_t) @@ -1586,7 +1743,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.huge.ndalloc (uint64_t) @@ -1597,7 +1754,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.dss (const char *) @@ -1611,7 +1768,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.nthreads (unsigned) @@ -1621,7 +1778,7 @@ malloc_conf = "xmalloc:true";]]> arena. - + stats.arenas.<i>.pactive (size_t) @@ -1642,7 +1799,7 @@ malloc_conf = "xmalloc:true";]]> similar has not been called. - + stats.arenas.<i>.mapped (size_t) @@ -1652,7 +1809,7 @@ malloc_conf = "xmalloc:true";]]> Number of mapped bytes. - + stats.arenas.<i>.npurge (uint64_t) @@ -1663,7 +1820,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.nmadvise (uint64_t) @@ -1675,9 +1832,9 @@ malloc_conf = "xmalloc:true";]]> similar calls made to purge dirty pages. - + - stats.arenas.<i>.npurged + stats.arenas.<i>.purged (uint64_t) r- [] @@ -1685,7 +1842,7 @@ malloc_conf = "xmalloc:true";]]> Number of pages purged. - + stats.arenas.<i>.small.allocated (size_t) @@ -1696,7 +1853,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.small.nmalloc (uint64_t) @@ -1707,7 +1864,7 @@ malloc_conf = "xmalloc:true";]]> small bins. - + stats.arenas.<i>.small.ndalloc (uint64_t) @@ -1718,7 +1875,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.small.nrequests (uint64_t) @@ -1729,7 +1886,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.large.allocated (size_t) @@ -1740,7 +1897,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.large.nmalloc (uint64_t) @@ -1751,7 +1908,7 @@ malloc_conf = "xmalloc:true";]]> directly by the arena. - + stats.arenas.<i>.large.ndalloc (uint64_t) @@ -1762,7 +1919,7 @@ malloc_conf = "xmalloc:true";]]> directly by the arena. - + stats.arenas.<i>.large.nrequests (uint64_t) @@ -1773,7 +1930,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.bins.<j>.allocated (size_t) @@ -1784,7 +1941,7 @@ malloc_conf = "xmalloc:true";]]> bin. - + stats.arenas.<i>.bins.<j>.nmalloc (uint64_t) @@ -1795,7 +1952,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.bins.<j>.ndalloc (uint64_t) @@ -1806,7 +1963,7 @@ malloc_conf = "xmalloc:true";]]> - + stats.arenas.<i>.bins.<j>.nrequests (uint64_t) @@ -1817,7 +1974,7 @@ malloc_conf = "xmalloc:true";]]> requests. - + stats.arenas.<i>.bins.<j>.nfills (uint64_t) @@ -1827,7 +1984,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of tcache fills. - + stats.arenas.<i>.bins.<j>.nflushes (uint64_t) @@ -1837,7 +1994,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of tcache flushes. - + stats.arenas.<i>.bins.<j>.nruns (uint64_t) @@ -1847,7 +2004,7 @@ malloc_conf = "xmalloc:true";]]> Cumulative number of runs created. - + stats.arenas.<i>.bins.<j>.nreruns (uint64_t) @@ -1858,7 +2015,7 @@ malloc_conf = "xmalloc:true";]]> to allocate changed. - + stats.arenas.<i>.bins.<j>.curruns (size_t) @@ -1868,7 +2025,7 @@ malloc_conf = "xmalloc:true";]]> Current number of runs. - + stats.arenas.<i>.lruns.<j>.nmalloc (uint64_t) @@ -1879,7 +2036,7 @@ malloc_conf = "xmalloc:true";]]> class served directly by the arena. - + stats.arenas.<i>.lruns.<j>.ndalloc (uint64_t) @@ -1890,7 +2047,7 @@ malloc_conf = "xmalloc:true";]]> size class served directly by the arena. - + stats.arenas.<i>.lruns.<j>.nrequests (uint64_t) @@ -1901,7 +2058,7 @@ malloc_conf = "xmalloc:true";]]> class. - + stats.arenas.<i>.lruns.<j>.curruns (size_t) @@ -2027,9 +2184,26 @@ malloc_conf = "xmalloc:true";]]> Non-standard API - The malloc_usable_size function - returns the usable size of the allocation pointed to by - ptr. + The mallocx and + rallocx functions return a pointer to + the allocated memory if successful; otherwise a NULL + pointer is returned to indicate insufficient contiguous memory was + available to service the allocation request. + + The xallocx function returns the + real size of the resulting resized allocation pointed to by + ptr, which is a value less than + size if the allocation could not be adequately + grown in place. + + The sallocx function returns the + real size of the allocation pointed to by ptr. + + + The nallocx returns the real size + that would result from a successful equivalent + mallocx function call, or zero if + insufficient memory is available to perform the size computation. The mallctl, mallctlnametomib, and @@ -2046,12 +2220,6 @@ malloc_conf = "xmalloc:true";]]> is too large or too small; in this case as much data as possible are read despite the error. - - ENOMEM - - *oldlenp is too short to - hold the requested value. - ENOENT @@ -2080,6 +2248,10 @@ malloc_conf = "xmalloc:true";]]> + + The malloc_usable_size function + returns the usable size of the allocation pointed to by + ptr. Experimental API diff --git a/deps/jemalloc/include/jemalloc/internal/arena.h b/deps/jemalloc/include/jemalloc/internal/arena.h index 561c9b6ffe9..9d000c03dec 100644 --- a/deps/jemalloc/include/jemalloc/internal/arena.h +++ b/deps/jemalloc/include/jemalloc/internal/arena.h @@ -158,6 +158,7 @@ struct arena_chunk_map_s { }; typedef rb_tree(arena_chunk_map_t) arena_avail_tree_t; typedef rb_tree(arena_chunk_map_t) arena_run_tree_t; +typedef ql_head(arena_chunk_map_t) arena_chunk_mapelms_t; /* Arena chunk header. */ struct arena_chunk_s { @@ -174,11 +175,12 @@ struct arena_chunk_s { size_t nruns_avail; /* - * Number of available run adjacencies. Clean and dirty available runs - * are not coalesced, which causes virtual memory fragmentation. The - * ratio of (nruns_avail-nruns_adjac):nruns_adjac is used for tracking - * this fragmentation. - * */ + * Number of available run adjacencies that purging could coalesce. + * Clean and dirty available runs are not coalesced, which causes + * virtual memory fragmentation. The ratio of + * (nruns_avail-nruns_adjac):nruns_adjac is used for tracking this + * fragmentation. + */ size_t nruns_adjac; /* @@ -400,12 +402,20 @@ extern arena_bin_info_t arena_bin_info[NBINS]; #define nlclasses (chunk_npages - map_bias) void arena_purge_all(arena_t *arena); -void arena_prof_accum(arena_t *arena, uint64_t accumbytes); void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, uint64_t prof_accumbytes); void arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero); +#ifdef JEMALLOC_JET +typedef void (arena_redzone_corruption_t)(void *, size_t, bool, size_t, + uint8_t); +extern arena_redzone_corruption_t *arena_redzone_corruption; +typedef void (arena_dalloc_junk_small_t)(void *, arena_bin_info_t *); +extern arena_dalloc_junk_small_t *arena_dalloc_junk_small; +#else void arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info); +#endif +void arena_quarantine_junk_small(void *ptr, size_t usize); void *arena_malloc_small(arena_t *arena, size_t size, bool zero); void *arena_malloc_large(arena_t *arena, size_t size, bool zero); void *arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero); @@ -416,10 +426,18 @@ void arena_dalloc_bin(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind, arena_chunk_map_t *mapelm); void arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t pageind); +#ifdef JEMALLOC_JET +typedef void (arena_dalloc_junk_large_t)(void *, size_t); +extern arena_dalloc_junk_large_t *arena_dalloc_junk_large; +#endif void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr); void arena_dalloc_large(arena_t *arena, arena_chunk_t *chunk, void *ptr); -void *arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, +#ifdef JEMALLOC_JET +typedef void (arena_ralloc_junk_large_t)(void *, size_t, size_t); +extern arena_ralloc_junk_large_t *arena_ralloc_junk_large; +#endif +bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero); void *arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t extra, size_t alignment, bool zero, bool try_tcache_alloc, @@ -442,6 +460,7 @@ void arena_postfork_child(arena_t *arena); #ifndef JEMALLOC_ENABLE_INLINE arena_chunk_map_t *arena_mapp_get(arena_chunk_t *chunk, size_t pageind); size_t *arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind); +size_t arena_mapbitsp_read(size_t *mapbitsp); size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind); @@ -452,6 +471,7 @@ size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind); size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind); +void arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits); void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags); void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, @@ -464,12 +484,15 @@ void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, size_t binind, size_t flags); void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, size_t unzeroed); +bool arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes); +bool arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes); +bool arena_prof_accum(arena_t *arena, uint64_t accumbytes); size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits); size_t arena_bin_index(arena_t *arena, arena_bin_t *bin); unsigned arena_run_regind(arena_run_t *run, arena_bin_info_t *bin_info, const void *ptr); prof_ctx_t *arena_prof_ctx_get(const void *ptr); -void arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +void arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); void *arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache); size_t arena_salloc(const void *ptr, bool demote); void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, @@ -478,7 +501,7 @@ void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_ARENA_C_)) # ifdef JEMALLOC_ARENA_INLINE_A -JEMALLOC_INLINE arena_chunk_map_t * +JEMALLOC_ALWAYS_INLINE arena_chunk_map_t * arena_mapp_get(arena_chunk_t *chunk, size_t pageind) { @@ -488,21 +511,28 @@ arena_mapp_get(arena_chunk_t *chunk, size_t pageind) return (&chunk->map[pageind-map_bias]); } -JEMALLOC_INLINE size_t * +JEMALLOC_ALWAYS_INLINE size_t * arena_mapbitsp_get(arena_chunk_t *chunk, size_t pageind) { return (&arena_mapp_get(chunk, pageind)->bits); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t +arena_mapbitsp_read(size_t *mapbitsp) +{ + + return (*mapbitsp); +} + +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_get(arena_chunk_t *chunk, size_t pageind) { - return (*arena_mapbitsp_get(chunk, pageind)); + return (arena_mapbitsp_read(arena_mapbitsp_get(chunk, pageind))); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -512,7 +542,7 @@ arena_mapbits_unallocated_size_get(arena_chunk_t *chunk, size_t pageind) return (mapbits & ~PAGE_MASK); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -523,7 +553,7 @@ arena_mapbits_large_size_get(arena_chunk_t *chunk, size_t pageind) return (mapbits & ~PAGE_MASK); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -534,7 +564,7 @@ arena_mapbits_small_runind_get(arena_chunk_t *chunk, size_t pageind) return (mapbits >> LG_PAGE); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -546,7 +576,7 @@ arena_mapbits_binind_get(arena_chunk_t *chunk, size_t pageind) return (binind); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -555,7 +585,7 @@ arena_mapbits_dirty_get(arena_chunk_t *chunk, size_t pageind) return (mapbits & CHUNK_MAP_DIRTY); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -564,7 +594,7 @@ arena_mapbits_unzeroed_get(arena_chunk_t *chunk, size_t pageind) return (mapbits & CHUNK_MAP_UNZEROED); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -573,7 +603,7 @@ arena_mapbits_large_get(arena_chunk_t *chunk, size_t pageind) return (mapbits & CHUNK_MAP_LARGE); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) { size_t mapbits; @@ -582,86 +612,138 @@ arena_mapbits_allocated_get(arena_chunk_t *chunk, size_t pageind) return (mapbits & CHUNK_MAP_ALLOCATED); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void +arena_mapbitsp_write(size_t *mapbitsp, size_t mapbits) +{ + + *mapbitsp = mapbits; +} + +JEMALLOC_ALWAYS_INLINE void arena_mapbits_unallocated_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags) { - size_t *mapbitsp; + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); - mapbitsp = arena_mapbitsp_get(chunk, pageind); assert((size & PAGE_MASK) == 0); assert((flags & ~CHUNK_MAP_FLAGS_MASK) == 0); assert((flags & (CHUNK_MAP_DIRTY|CHUNK_MAP_UNZEROED)) == flags); - *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags; + arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void arena_mapbits_unallocated_size_set(arena_chunk_t *chunk, size_t pageind, size_t size) { - size_t *mapbitsp; + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); - mapbitsp = arena_mapbitsp_get(chunk, pageind); assert((size & PAGE_MASK) == 0); - assert((*mapbitsp & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); - *mapbitsp = size | (*mapbitsp & PAGE_MASK); + assert((mapbits & (CHUNK_MAP_LARGE|CHUNK_MAP_ALLOCATED)) == 0); + arena_mapbitsp_write(mapbitsp, size | (mapbits & PAGE_MASK)); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void arena_mapbits_large_set(arena_chunk_t *chunk, size_t pageind, size_t size, size_t flags) { - size_t *mapbitsp; + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); size_t unzeroed; - mapbitsp = arena_mapbitsp_get(chunk, pageind); assert((size & PAGE_MASK) == 0); assert((flags & CHUNK_MAP_DIRTY) == flags); - unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ - *mapbitsp = size | CHUNK_MAP_BININD_INVALID | flags | unzeroed | - CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED; + unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ + arena_mapbitsp_write(mapbitsp, size | CHUNK_MAP_BININD_INVALID | flags + | unzeroed | CHUNK_MAP_LARGE | CHUNK_MAP_ALLOCATED); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void arena_mapbits_large_binind_set(arena_chunk_t *chunk, size_t pageind, size_t binind) { - size_t *mapbitsp; + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); assert(binind <= BININD_INVALID); - mapbitsp = arena_mapbitsp_get(chunk, pageind); assert(arena_mapbits_large_size_get(chunk, pageind) == PAGE); - *mapbitsp = (*mapbitsp & ~CHUNK_MAP_BININD_MASK) | (binind << - CHUNK_MAP_BININD_SHIFT); + arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_BININD_MASK) | + (binind << CHUNK_MAP_BININD_SHIFT)); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void arena_mapbits_small_set(arena_chunk_t *chunk, size_t pageind, size_t runind, size_t binind, size_t flags) { - size_t *mapbitsp; + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); size_t unzeroed; assert(binind < BININD_INVALID); - mapbitsp = arena_mapbitsp_get(chunk, pageind); assert(pageind - runind >= map_bias); assert((flags & CHUNK_MAP_DIRTY) == flags); - unzeroed = *mapbitsp & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ - *mapbitsp = (runind << LG_PAGE) | (binind << CHUNK_MAP_BININD_SHIFT) | - flags | unzeroed | CHUNK_MAP_ALLOCATED; + unzeroed = mapbits & CHUNK_MAP_UNZEROED; /* Preserve unzeroed. */ + arena_mapbitsp_write(mapbitsp, (runind << LG_PAGE) | (binind << + CHUNK_MAP_BININD_SHIFT) | flags | unzeroed | CHUNK_MAP_ALLOCATED); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void arena_mapbits_unzeroed_set(arena_chunk_t *chunk, size_t pageind, size_t unzeroed) { - size_t *mapbitsp; + size_t *mapbitsp = arena_mapbitsp_get(chunk, pageind); + size_t mapbits = arena_mapbitsp_read(mapbitsp); - mapbitsp = arena_mapbitsp_get(chunk, pageind); - *mapbitsp = (*mapbitsp & ~CHUNK_MAP_UNZEROED) | unzeroed; + arena_mapbitsp_write(mapbitsp, (mapbits & ~CHUNK_MAP_UNZEROED) | + unzeroed); } -JEMALLOC_INLINE size_t +JEMALLOC_INLINE bool +arena_prof_accum_impl(arena_t *arena, uint64_t accumbytes) +{ + + cassert(config_prof); + assert(prof_interval != 0); + + arena->prof_accumbytes += accumbytes; + if (arena->prof_accumbytes >= prof_interval) { + arena->prof_accumbytes -= prof_interval; + return (true); + } + return (false); +} + +JEMALLOC_INLINE bool +arena_prof_accum_locked(arena_t *arena, uint64_t accumbytes) +{ + + cassert(config_prof); + + if (prof_interval == 0) + return (false); + return (arena_prof_accum_impl(arena, accumbytes)); +} + +JEMALLOC_INLINE bool +arena_prof_accum(arena_t *arena, uint64_t accumbytes) +{ + + cassert(config_prof); + + if (prof_interval == 0) + return (false); + + { + bool ret; + + malloc_mutex_lock(&arena->lock); + ret = arena_prof_accum_impl(arena, accumbytes); + malloc_mutex_unlock(&arena->lock); + return (ret); + } +} + +JEMALLOC_ALWAYS_INLINE size_t arena_ptr_small_binind_get(const void *ptr, size_t mapbits) { size_t binind; @@ -822,10 +904,10 @@ arena_prof_ctx_get(const void *ptr) } JEMALLOC_INLINE void -arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +arena_prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) { arena_chunk_t *chunk; - size_t pageind, mapbits; + size_t pageind; cassert(config_prof); assert(ptr != NULL); @@ -833,10 +915,17 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - mapbits = arena_mapbits_get(chunk, pageind); - assert((mapbits & CHUNK_MAP_ALLOCATED) != 0); - if ((mapbits & CHUNK_MAP_LARGE) == 0) { + assert(arena_mapbits_allocated_get(chunk, pageind) != 0); + + if (usize > SMALL_MAXCLASS || (prof_promote && + ((uintptr_t)ctx != (uintptr_t)1U || arena_mapbits_large_get(chunk, + pageind) != 0))) { + assert(arena_mapbits_large_get(chunk, pageind) != 0); + arena_mapp_get(chunk, pageind)->prof_ctx = ctx; + } else { + assert(arena_mapbits_large_get(chunk, pageind) == 0); if (prof_promote == false) { + size_t mapbits = arena_mapbits_get(chunk, pageind); arena_run_t *run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)((pageind - (mapbits >> LG_PAGE)) << LG_PAGE)); @@ -848,15 +937,14 @@ arena_prof_ctx_set(const void *ptr, prof_ctx_t *ctx) bin_info = &arena_bin_info[binind]; regind = arena_run_regind(run, bin_info, ptr); - *((prof_ctx_t **)((uintptr_t)run + bin_info->ctx0_offset - + (regind * sizeof(prof_ctx_t *)))) = ctx; - } else - assert((uintptr_t)ctx == (uintptr_t)1U); - } else - arena_mapp_get(chunk, pageind)->prof_ctx = ctx; + *((prof_ctx_t **)((uintptr_t)run + + bin_info->ctx0_offset + (regind * sizeof(prof_ctx_t + *)))) = ctx; + } + } } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) { tcache_t *tcache; @@ -887,7 +975,7 @@ arena_malloc(arena_t *arena, size_t size, bool zero, bool try_tcache) } /* Return the size of the allocation pointed to by ptr. */ -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t arena_salloc(const void *ptr, bool demote) { size_t ret; @@ -933,7 +1021,7 @@ arena_salloc(const void *ptr, bool demote) return (ret); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void arena_dalloc(arena_t *arena, arena_chunk_t *chunk, void *ptr, bool try_tcache) { size_t pageind, mapbits; diff --git a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h index 6585f071bbe..4535ce09c09 100644 --- a/deps/jemalloc/include/jemalloc/internal/chunk_dss.h +++ b/deps/jemalloc/include/jemalloc/internal/chunk_dss.h @@ -7,7 +7,7 @@ typedef enum { dss_prec_secondary = 2, dss_prec_limit = 3 -} dss_prec_t ; +} dss_prec_t; #define DSS_PREC_DEFAULT dss_prec_secondary #define DSS_DEFAULT "secondary" diff --git a/deps/jemalloc/include/jemalloc/internal/ckh.h b/deps/jemalloc/include/jemalloc/internal/ckh.h index 05d1fc03e7a..58712a6a763 100644 --- a/deps/jemalloc/include/jemalloc/internal/ckh.h +++ b/deps/jemalloc/include/jemalloc/internal/ckh.h @@ -5,7 +5,7 @@ typedef struct ckh_s ckh_t; typedef struct ckhc_s ckhc_t; /* Typedefs to allow easy function pointer passing. */ -typedef void ckh_hash_t (const void *, unsigned, size_t *, size_t *); +typedef void ckh_hash_t (const void *, size_t[2]); typedef bool ckh_keycomp_t (const void *, const void *); /* Maintain counters used to get an idea of performance. */ @@ -17,7 +17,7 @@ typedef bool ckh_keycomp_t (const void *, const void *); * There are 2^LG_CKH_BUCKET_CELLS cells in each hash table bucket. Try to fit * one bucket per L1 cache line. */ -#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) +#define LG_CKH_BUCKET_CELLS (LG_CACHELINE - LG_SIZEOF_PTR - 1) #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -75,11 +75,9 @@ bool ckh_insert(ckh_t *ckh, const void *key, const void *data); bool ckh_remove(ckh_t *ckh, const void *searchkey, void **key, void **data); bool ckh_search(ckh_t *ckh, const void *seachkey, void **key, void **data); -void ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); +void ckh_string_hash(const void *key, size_t r_hash[2]); bool ckh_string_keycomp(const void *k1, const void *k2); -void ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); +void ckh_pointer_hash(const void *key, size_t r_hash[2]); bool ckh_pointer_keycomp(const void *k1, const void *k2); #endif /* JEMALLOC_H_EXTERNS */ diff --git a/deps/jemalloc/include/jemalloc/internal/hash.h b/deps/jemalloc/include/jemalloc/internal/hash.h index 2f501f5d4c9..c7183ede82d 100644 --- a/deps/jemalloc/include/jemalloc/internal/hash.h +++ b/deps/jemalloc/include/jemalloc/internal/hash.h @@ -1,3 +1,8 @@ +/* + * The following hash function is based on MurmurHash3, placed into the public + * domain by Austin Appleby. See http://code.google.com/p/smhasher/ for + * details. + */ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES @@ -14,55 +19,315 @@ #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -uint64_t hash(const void *key, size_t len, uint64_t seed); +uint32_t hash_x86_32(const void *key, int len, uint32_t seed); +void hash_x86_128(const void *key, const int len, uint32_t seed, + uint64_t r_out[2]); +void hash_x64_128(const void *key, const int len, const uint32_t seed, + uint64_t r_out[2]); +void hash(const void *key, size_t len, const uint32_t seed, + size_t r_hash[2]); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_HASH_C_)) -/* - * The following hash function is based on MurmurHash64A(), placed into the - * public domain by Austin Appleby. See http://murmurhash.googlepages.com/ for - * details. - */ +/******************************************************************************/ +/* Internal implementation. */ +JEMALLOC_INLINE uint32_t +hash_rotl_32(uint32_t x, int8_t r) +{ + + return (x << r) | (x >> (32 - r)); +} + +JEMALLOC_INLINE uint64_t +hash_rotl_64(uint64_t x, int8_t r) +{ + return (x << r) | (x >> (64 - r)); +} + +JEMALLOC_INLINE uint32_t +hash_get_block_32(const uint32_t *p, int i) +{ + + return (p[i]); +} + JEMALLOC_INLINE uint64_t -hash(const void *key, size_t len, uint64_t seed) +hash_get_block_64(const uint64_t *p, int i) { - const uint64_t m = UINT64_C(0xc6a4a7935bd1e995); - const int r = 47; - uint64_t h = seed ^ (len * m); - const uint64_t *data = (const uint64_t *)key; - const uint64_t *end = data + (len/8); - const unsigned char *data2; - assert(((uintptr_t)key & 0x7) == 0); + return (p[i]); +} + +JEMALLOC_INLINE uint32_t +hash_fmix_32(uint32_t h) +{ - while(data != end) { - uint64_t k = *data++; + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; - k *= m; - k ^= k >> r; - k *= m; + return (h); +} - h ^= k; - h *= m; +JEMALLOC_INLINE uint64_t +hash_fmix_64(uint64_t k) +{ + + k ^= k >> 33; + k *= QU(0xff51afd7ed558ccdLLU); + k ^= k >> 33; + k *= QU(0xc4ceb9fe1a85ec53LLU); + k ^= k >> 33; + + return (k); +} + +JEMALLOC_INLINE uint32_t +hash_x86_32(const void *key, int len, uint32_t seed) +{ + const uint8_t *data = (const uint8_t *) key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + const uint32_t c1 = 0xcc9e2d51; + const uint32_t c2 = 0x1b873593; + + /* body */ + { + const uint32_t *blocks = (const uint32_t *) (data + nblocks*4); + int i; + + for (i = -nblocks; i; i++) { + uint32_t k1 = hash_get_block_32(blocks, i); + + k1 *= c1; + k1 = hash_rotl_32(k1, 15); + k1 *= c2; + + h1 ^= k1; + h1 = hash_rotl_32(h1, 13); + h1 = h1*5 + 0xe6546b64; + } } - data2 = (const unsigned char *)data; - switch(len & 7) { - case 7: h ^= ((uint64_t)(data2[6])) << 48; - case 6: h ^= ((uint64_t)(data2[5])) << 40; - case 5: h ^= ((uint64_t)(data2[4])) << 32; - case 4: h ^= ((uint64_t)(data2[3])) << 24; - case 3: h ^= ((uint64_t)(data2[2])) << 16; - case 2: h ^= ((uint64_t)(data2[1])) << 8; - case 1: h ^= ((uint64_t)(data2[0])); - h *= m; + /* tail */ + { + const uint8_t *tail = (const uint8_t *) (data + nblocks*4); + + uint32_t k1 = 0; + + switch (len & 3) { + case 3: k1 ^= tail[2] << 16; + case 2: k1 ^= tail[1] << 8; + case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); + k1 *= c2; h1 ^= k1; + } } - h ^= h >> r; - h *= m; - h ^= h >> r; + /* finalization */ + h1 ^= len; - return (h); + h1 = hash_fmix_32(h1); + + return (h1); +} + +UNUSED JEMALLOC_INLINE void +hash_x86_128(const void *key, const int len, uint32_t seed, + uint64_t r_out[2]) +{ + const uint8_t * data = (const uint8_t *) key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + const uint32_t c1 = 0x239b961b; + const uint32_t c2 = 0xab0e9789; + const uint32_t c3 = 0x38b34ae5; + const uint32_t c4 = 0xa1e38b93; + + /* body */ + { + const uint32_t *blocks = (const uint32_t *) (data + nblocks*16); + int i; + + for (i = -nblocks; i; i++) { + uint32_t k1 = hash_get_block_32(blocks, i*4 + 0); + uint32_t k2 = hash_get_block_32(blocks, i*4 + 1); + uint32_t k3 = hash_get_block_32(blocks, i*4 + 2); + uint32_t k4 = hash_get_block_32(blocks, i*4 + 3); + + k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + + h1 = hash_rotl_32(h1, 19); h1 += h2; + h1 = h1*5 + 0x561ccd1b; + + k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; + + h2 = hash_rotl_32(h2, 17); h2 += h3; + h2 = h2*5 + 0x0bcaa747; + + k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; + + h3 = hash_rotl_32(h3, 15); h3 += h4; + h3 = h3*5 + 0x96cd1c35; + + k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; + + h4 = hash_rotl_32(h4, 13); h4 += h1; + h4 = h4*5 + 0x32ac3b17; + } + } + + /* tail */ + { + const uint8_t *tail = (const uint8_t *) (data + nblocks*16); + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch (len & 15) { + case 15: k4 ^= tail[14] << 16; + case 14: k4 ^= tail[13] << 8; + case 13: k4 ^= tail[12] << 0; + k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; + + case 12: k3 ^= tail[11] << 24; + case 11: k3 ^= tail[10] << 16; + case 10: k3 ^= tail[ 9] << 8; + case 9: k3 ^= tail[ 8] << 0; + k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; + + case 8: k2 ^= tail[ 7] << 24; + case 7: k2 ^= tail[ 6] << 16; + case 6: k2 ^= tail[ 5] << 8; + case 5: k2 ^= tail[ 4] << 0; + k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; + + case 4: k1 ^= tail[ 3] << 24; + case 3: k1 ^= tail[ 2] << 16; + case 2: k1 ^= tail[ 1] << 8; + case 1: k1 ^= tail[ 0] << 0; + k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + } + } + + /* finalization */ + h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len; + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + h1 = hash_fmix_32(h1); + h2 = hash_fmix_32(h2); + h3 = hash_fmix_32(h3); + h4 = hash_fmix_32(h4); + + h1 += h2; h1 += h3; h1 += h4; + h2 += h1; h3 += h1; h4 += h1; + + r_out[0] = (((uint64_t) h2) << 32) | h1; + r_out[1] = (((uint64_t) h4) << 32) | h3; +} + +UNUSED JEMALLOC_INLINE void +hash_x64_128(const void *key, const int len, const uint32_t seed, + uint64_t r_out[2]) +{ + const uint8_t *data = (const uint8_t *) key; + const int nblocks = len / 16; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + const uint64_t c1 = QU(0x87c37b91114253d5LLU); + const uint64_t c2 = QU(0x4cf5ad432745937fLLU); + + /* body */ + { + const uint64_t *blocks = (const uint64_t *) (data); + int i; + + for (i = 0; i < nblocks; i++) { + uint64_t k1 = hash_get_block_64(blocks, i*2 + 0); + uint64_t k2 = hash_get_block_64(blocks, i*2 + 1); + + k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; + + h1 = hash_rotl_64(h1, 27); h1 += h2; + h1 = h1*5 + 0x52dce729; + + k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; + + h2 = hash_rotl_64(h2, 31); h2 += h1; + h2 = h2*5 + 0x38495ab5; + } + } + + /* tail */ + { + const uint8_t *tail = (const uint8_t*)(data + nblocks*16); + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (len & 15) { + case 15: k2 ^= ((uint64_t)(tail[14])) << 48; + case 14: k2 ^= ((uint64_t)(tail[13])) << 40; + case 13: k2 ^= ((uint64_t)(tail[12])) << 32; + case 12: k2 ^= ((uint64_t)(tail[11])) << 24; + case 11: k2 ^= ((uint64_t)(tail[10])) << 16; + case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; + case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; + k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; + + case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; + case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; + case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; + case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; + case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; + case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; + case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; + case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; + k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; + } + } + + /* finalization */ + h1 ^= len; h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = hash_fmix_64(h1); + h2 = hash_fmix_64(h2); + + h1 += h2; + h2 += h1; + + r_out[0] = h1; + r_out[1] = h2; +} + +/******************************************************************************/ +/* API. */ +JEMALLOC_INLINE void +hash(const void *key, size_t len, const uint32_t seed, size_t r_hash[2]) +{ +#if (LG_SIZEOF_PTR == 3 && !defined(JEMALLOC_BIG_ENDIAN)) + hash_x64_128(key, len, seed, (uint64_t *)r_hash); +#else + uint64_t hashes[2]; + hash_x86_128(key, len, seed, hashes); + r_hash[0] = (size_t)hashes[0]; + r_hash[1] = (size_t)hashes[1]; +#endif } #endif diff --git a/deps/jemalloc/include/jemalloc/internal/huge.h b/deps/jemalloc/include/jemalloc/internal/huge.h index d987d370767..a2b9c779191 100644 --- a/deps/jemalloc/include/jemalloc/internal/huge.h +++ b/deps/jemalloc/include/jemalloc/internal/huge.h @@ -17,14 +17,20 @@ extern size_t huge_allocated; /* Protects chunk-related data structures. */ extern malloc_mutex_t huge_mtx; -void *huge_malloc(size_t size, bool zero); -void *huge_palloc(size_t size, size_t alignment, bool zero); -void *huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, +void *huge_malloc(size_t size, bool zero, dss_prec_t dss_prec); +void *huge_palloc(size_t size, size_t alignment, bool zero, + dss_prec_t dss_prec); +bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra); void *huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache_dalloc); + size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec); +#ifdef JEMALLOC_JET +typedef void (huge_dalloc_junk_t)(void *, size_t); +extern huge_dalloc_junk_t *huge_dalloc_junk; +#endif void huge_dalloc(void *ptr, bool unmap); size_t huge_salloc(const void *ptr); +dss_prec_t huge_dss_prec_get(arena_t *arena); prof_ctx_t *huge_prof_ctx_get(const void *ptr); void huge_prof_ctx_set(const void *ptr, prof_ctx_t *ctx); bool huge_boot(void); diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in index 475821acb61..574bbb14186 100644 --- a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal.h.in @@ -1,5 +1,5 @@ #ifndef JEMALLOC_INTERNAL_H -#define JEMALLOC_INTERNAL_H +#define JEMALLOC_INTERNAL_H #include #ifdef _WIN32 # include @@ -54,8 +54,7 @@ typedef intptr_t ssize_t; #endif #include -#define JEMALLOC_NO_DEMANGLE -#include "../jemalloc@install_suffix@.h" +#include "jemalloc_internal_defs.h" #ifdef JEMALLOC_UTRACE #include @@ -66,13 +65,18 @@ typedef intptr_t ssize_t; #include #endif -#include "jemalloc/internal/private_namespace.h" - -#ifdef JEMALLOC_CC_SILENCE -#define UNUSED JEMALLOC_ATTR(unused) +#define JEMALLOC_NO_DEMANGLE +#ifdef JEMALLOC_JET +# define JEMALLOC_N(n) jet_##n +# include "jemalloc/internal/public_namespace.h" +# define JEMALLOC_NO_RENAME +# include "../jemalloc@install_suffix@.h" +# undef JEMALLOC_NO_RENAME #else -#define UNUSED +# define JEMALLOC_N(n) @private_namespace@##n +# include "../jemalloc@install_suffix@.h" #endif +#include "jemalloc/internal/private_namespace.h" static const bool config_debug = #ifdef JEMALLOC_DEBUG @@ -221,27 +225,12 @@ static const bool config_ivsalloc = * JEMALLOC_H_INLINES : Inline functions. */ /******************************************************************************/ -#define JEMALLOC_H_TYPES - -#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) +#define JEMALLOC_H_TYPES -#define ZU(z) ((size_t)z) +#include "jemalloc/internal/jemalloc_internal_macros.h" -#ifndef __DECONST -# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) -#endif - -#ifdef JEMALLOC_DEBUG - /* Disable inlining to make debugging easier. */ -# define JEMALLOC_INLINE -# define inline -#else -# define JEMALLOC_ENABLE_INLINE -# define JEMALLOC_INLINE static inline -# ifdef _MSC_VER -# define inline _inline -# endif -#endif +#define MALLOCX_LG_ALIGN_MASK ((int)0x3f) +#define ALLOCM_LG_ALIGN_MASK ((int)0x3f) /* Smallest size class to support. */ #define LG_TINY_MIN 3 @@ -270,6 +259,9 @@ static const bool config_ivsalloc = # ifdef __arm__ # define LG_QUANTUM 3 # endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif # ifdef __hppa__ # define LG_QUANTUM 4 # endif @@ -279,7 +271,7 @@ static const bool config_ivsalloc = # ifdef __powerpc__ # define LG_QUANTUM 4 # endif -# ifdef __s390x__ +# ifdef __s390__ # define LG_QUANTUM 4 # endif # ifdef __SH4__ @@ -359,7 +351,11 @@ static const bool config_ivsalloc = # include # define alloca _alloca # else -# include +# ifdef JEMALLOC_HAS_ALLOCA_H +# include +# else +# include +# endif # endif # define VARIABLE_ARRAY(type, name, count) \ type *name = alloca(sizeof(type) * count) @@ -428,15 +424,18 @@ static const bool config_ivsalloc = } while (0) #else #define RUNNING_ON_VALGRIND ((unsigned)0) -#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) -#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) -#define VALGRIND_FREELIKE_BLOCK(addr, rzB) -#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) -#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) -#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) +#define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ + do {} while (0) +#define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \ + do {} while (0) +#define VALGRIND_FREELIKE_BLOCK(addr, rzB) do {} while (0) +#define VALGRIND_MAKE_MEM_NOACCESS(_qzz_addr, _qzz_len) do {} while (0) +#define VALGRIND_MAKE_MEM_UNDEFINED(_qzz_addr, _qzz_len) do {} while (0) +#define VALGRIND_MAKE_MEM_DEFINED(_qzz_addr, _qzz_len) do {} while (0) +#define JEMALLOC_VALGRIND_MALLOC(cond, ptr, usize, zero) do {} while (0) #define JEMALLOC_VALGRIND_REALLOC(ptr, usize, old_ptr, old_usize, \ - old_rzsize, zero) -#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) + old_rzsize, zero) do {} while (0) +#define JEMALLOC_VALGRIND_FREE(ptr, rzsize) do {} while (0) #endif #include "jemalloc/internal/util.h" @@ -463,7 +462,7 @@ static const bool config_ivsalloc = #undef JEMALLOC_H_TYPES /******************************************************************************/ -#define JEMALLOC_H_STRUCTS +#define JEMALLOC_H_STRUCTS #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" @@ -492,14 +491,14 @@ typedef struct { uint64_t deallocated; } thread_allocated_t; /* - * The JEMALLOC_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro + * The JEMALLOC_ARG_CONCAT() wrapper is necessary to pass {0, 0} via a cpp macro * argument. */ -#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_CONCAT({0, 0}) +#define THREAD_ALLOCATED_INITIALIZER JEMALLOC_ARG_CONCAT({0, 0}) #undef JEMALLOC_H_STRUCTS /******************************************************************************/ -#define JEMALLOC_H_EXTERNS +#define JEMALLOC_H_EXTERNS extern bool opt_abort; extern bool opt_junk; @@ -559,7 +558,7 @@ void jemalloc_postfork_child(void); #undef JEMALLOC_H_EXTERNS /******************************************************************************/ -#define JEMALLOC_H_INLINES +#define JEMALLOC_H_INLINES #include "jemalloc/internal/util.h" #include "jemalloc/internal/atomic.h" @@ -591,13 +590,14 @@ arena_t *choose_arena(arena_t *arena); * for allocations. */ malloc_tsd_externs(arenas, arena_t *) -malloc_tsd_funcs(JEMALLOC_INLINE, arenas, arena_t *, NULL, arenas_cleanup) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, arenas, arena_t *, NULL, + arenas_cleanup) /* * Compute usable size that would result from allocating an object with the * specified size. */ -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t s2u(size_t size) { @@ -612,7 +612,7 @@ s2u(size_t size) * Compute usable size that would result from allocating an object with the * specified size and alignment. */ -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t sa2u(size_t size, size_t alignment) { size_t usize; @@ -733,32 +733,36 @@ choose_arena(arena_t *arena) #include "jemalloc/internal/quarantine.h" #ifndef JEMALLOC_ENABLE_INLINE -void *imallocx(size_t size, bool try_tcache, arena_t *arena); +void *imalloct(size_t size, bool try_tcache, arena_t *arena); void *imalloc(size_t size); -void *icallocx(size_t size, bool try_tcache, arena_t *arena); +void *icalloct(size_t size, bool try_tcache, arena_t *arena); void *icalloc(size_t size); -void *ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, +void *ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena); void *ipalloc(size_t usize, size_t alignment, bool zero); size_t isalloc(const void *ptr, bool demote); size_t ivsalloc(const void *ptr, bool demote); size_t u2rz(size_t usize); size_t p2rz(const void *ptr); -void idallocx(void *ptr, bool try_tcache); +void idalloct(void *ptr, bool try_tcache); void idalloc(void *ptr); -void iqallocx(void *ptr, bool try_tcache); +void iqalloct(void *ptr, bool try_tcache); void iqalloc(void *ptr); -void *irallocx(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero, bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, +void *iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); +void *iralloct(void *ptr, size_t size, size_t extra, size_t alignment, + bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena); void *iralloc(void *ptr, size_t size, size_t extra, size_t alignment, - bool zero, bool no_move); + bool zero); +bool ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, + bool zero); malloc_tsd_protos(JEMALLOC_ATTR(unused), thread_allocated, thread_allocated_t) #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_C_)) -JEMALLOC_INLINE void * -imallocx(size_t size, bool try_tcache, arena_t *arena) +JEMALLOC_ALWAYS_INLINE void * +imalloct(size_t size, bool try_tcache, arena_t *arena) { assert(size != 0); @@ -766,35 +770,35 @@ imallocx(size_t size, bool try_tcache, arena_t *arena) if (size <= arena_maxclass) return (arena_malloc(arena, size, false, try_tcache)); else - return (huge_malloc(size, false)); + return (huge_malloc(size, false, huge_dss_prec_get(arena))); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * imalloc(size_t size) { - return (imallocx(size, true, NULL)); + return (imalloct(size, true, NULL)); } -JEMALLOC_INLINE void * -icallocx(size_t size, bool try_tcache, arena_t *arena) +JEMALLOC_ALWAYS_INLINE void * +icalloct(size_t size, bool try_tcache, arena_t *arena) { if (size <= arena_maxclass) return (arena_malloc(arena, size, true, try_tcache)); else - return (huge_malloc(size, true)); + return (huge_malloc(size, true, huge_dss_prec_get(arena))); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * icalloc(size_t size) { - return (icallocx(size, true, NULL)); + return (icalloct(size, true, NULL)); } -JEMALLOC_INLINE void * -ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, +JEMALLOC_ALWAYS_INLINE void * +ipalloct(size_t usize, size_t alignment, bool zero, bool try_tcache, arena_t *arena) { void *ret; @@ -809,20 +813,20 @@ ipallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, ret = arena_palloc(choose_arena(arena), usize, alignment, zero); } else if (alignment <= chunksize) - ret = huge_malloc(usize, zero); + ret = huge_malloc(usize, zero, huge_dss_prec_get(arena)); else - ret = huge_palloc(usize, alignment, zero); + ret = huge_palloc(usize, alignment, zero, huge_dss_prec_get(arena)); } assert(ALIGNMENT_ADDR2BASE(ret, alignment) == ret); return (ret); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * ipalloc(size_t usize, size_t alignment, bool zero) { - return (ipallocx(usize, alignment, zero, true, NULL)); + return (ipalloct(usize, alignment, zero, true, NULL)); } /* @@ -830,7 +834,7 @@ ipalloc(size_t usize, size_t alignment, bool zero) * void *ptr = [...] * size_t sz = isalloc(ptr, config_prof); */ -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t isalloc(const void *ptr, bool demote) { size_t ret; @@ -849,12 +853,12 @@ isalloc(const void *ptr, bool demote) return (ret); } -JEMALLOC_INLINE size_t +JEMALLOC_ALWAYS_INLINE size_t ivsalloc(const void *ptr, bool demote) { /* Return 0 if ptr is not within a chunk managed by jemalloc. */ - if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == NULL) + if (rtree_get(chunks_rtree, (uintptr_t)CHUNK_ADDR2BASE(ptr)) == 0) return (0); return (isalloc(ptr, demote)); @@ -882,8 +886,8 @@ p2rz(const void *ptr) return (u2rz(usize)); } -JEMALLOC_INLINE void -idallocx(void *ptr, bool try_tcache) +JEMALLOC_ALWAYS_INLINE void +idalloct(void *ptr, bool try_tcache) { arena_chunk_t *chunk; @@ -896,35 +900,67 @@ idallocx(void *ptr, bool try_tcache) huge_dalloc(ptr, true); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void idalloc(void *ptr) { - idallocx(ptr, true); + idalloct(ptr, true); } -JEMALLOC_INLINE void -iqallocx(void *ptr, bool try_tcache) +JEMALLOC_ALWAYS_INLINE void +iqalloct(void *ptr, bool try_tcache) { if (config_fill && opt_quarantine) quarantine(ptr); else - idallocx(ptr, try_tcache); + idalloct(ptr, try_tcache); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void iqalloc(void *ptr) { - iqallocx(ptr, true); + iqalloct(ptr, true); } -JEMALLOC_INLINE void * -irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, - bool no_move, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) +JEMALLOC_ALWAYS_INLINE void * +iralloct_realign(void *ptr, size_t oldsize, size_t size, size_t extra, + size_t alignment, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena) +{ + void *p; + size_t usize, copysize; + + usize = sa2u(size + extra, alignment); + if (usize == 0) + return (NULL); + p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); + if (p == NULL) { + if (extra == 0) + return (NULL); + /* Try again, without extra this time. */ + usize = sa2u(size, alignment); + if (usize == 0) + return (NULL); + p = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); + if (p == NULL) + return (NULL); + } + /* + * Copy at most size bytes (not size+extra), since the caller has no + * expectation that the extra bytes will be reliably preserved. + */ + copysize = (size < oldsize) ? size : oldsize; + memcpy(p, ptr, copysize); + iqalloct(ptr, try_tcache_dalloc); + return (p); +} + +JEMALLOC_ALWAYS_INLINE void * +iralloct(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, + bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena) { - void *ret; size_t oldsize; assert(ptr != NULL); @@ -934,72 +970,54 @@ irallocx(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { - size_t usize, copysize; - /* * Existing object alignment is inadequate; allocate new space * and copy. */ - if (no_move) - return (NULL); - usize = sa2u(size + extra, alignment); - if (usize == 0) - return (NULL); - ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); - if (ret == NULL) { - if (extra == 0) - return (NULL); - /* Try again, without extra this time. */ - usize = sa2u(size, alignment); - if (usize == 0) - return (NULL); - ret = ipallocx(usize, alignment, zero, try_tcache_alloc, - arena); - if (ret == NULL) - return (NULL); - } - /* - * Copy at most size bytes (not size+extra), since the caller - * has no expectation that the extra bytes will be reliably - * preserved. - */ - copysize = (size < oldsize) ? size : oldsize; - memcpy(ret, ptr, copysize); - iqallocx(ptr, try_tcache_dalloc); - return (ret); + return (iralloct_realign(ptr, oldsize, size, extra, alignment, + zero, try_tcache_alloc, try_tcache_dalloc, arena)); } - if (no_move) { - if (size <= arena_maxclass) { - return (arena_ralloc_no_move(ptr, oldsize, size, - extra, zero)); - } else { - return (huge_ralloc_no_move(ptr, oldsize, size, - extra)); - } + if (size + extra <= arena_maxclass) { + return (arena_ralloc(arena, ptr, oldsize, size, extra, + alignment, zero, try_tcache_alloc, + try_tcache_dalloc)); } else { - if (size + extra <= arena_maxclass) { - return (arena_ralloc(arena, ptr, oldsize, size, extra, - alignment, zero, try_tcache_alloc, - try_tcache_dalloc)); - } else { - return (huge_ralloc(ptr, oldsize, size, extra, - alignment, zero, try_tcache_dalloc)); - } + return (huge_ralloc(ptr, oldsize, size, extra, + alignment, zero, try_tcache_dalloc, huge_dss_prec_get(arena))); } } -JEMALLOC_INLINE void * -iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero, - bool no_move) +JEMALLOC_ALWAYS_INLINE void * +iralloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) +{ + + return (iralloct(ptr, size, extra, alignment, zero, true, true, NULL)); +} + +JEMALLOC_ALWAYS_INLINE bool +ixalloc(void *ptr, size_t size, size_t extra, size_t alignment, bool zero) { + size_t oldsize; + + assert(ptr != NULL); + assert(size != 0); - return (irallocx(ptr, size, extra, alignment, zero, no_move, true, true, - NULL)); + oldsize = isalloc(ptr, config_prof); + if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) + != 0) { + /* Existing object alignment is inadequate. */ + return (true); + } + + if (size <= arena_maxclass) + return (arena_ralloc_no_move(ptr, oldsize, size, extra, zero)); + else + return (huge_ralloc_no_move(ptr, oldsize, size, extra)); } malloc_tsd_externs(thread_allocated, thread_allocated_t) -malloc_tsd_funcs(JEMALLOC_INLINE, thread_allocated, thread_allocated_t, +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, thread_allocated, thread_allocated_t, THREAD_ALLOCATED_INITIALIZER, malloc_tsd_no_cleanup) #endif diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in new file mode 100644 index 00000000000..c166fbd9e41 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -0,0 +1,205 @@ +#ifndef JEMALLOC_INTERNAL_DEFS_H_ +#define JEMALLOC_INTERNAL_DEFS_H_ +/* + * If JEMALLOC_PREFIX is defined via --with-jemalloc-prefix, it will cause all + * public APIs to be prefixed. This makes it possible, with some care, to use + * multiple allocators simultaneously. + */ +#undef JEMALLOC_PREFIX +#undef JEMALLOC_CPREFIX + +/* + * JEMALLOC_PRIVATE_NAMESPACE is used as a prefix for all library-private APIs. + * For shared libraries, symbol visibility mechanisms prevent these symbols + * from being exported, but for static libraries, naming collisions are a real + * possibility. + */ +#undef JEMALLOC_PRIVATE_NAMESPACE + +/* + * Hyper-threaded CPUs may need a special instruction inside spin loops in + * order to yield to another virtual CPU. + */ +#undef CPU_SPINWAIT + +/* Defined if the equivalent of FreeBSD's atomic(9) functions are available. */ +#undef JEMALLOC_ATOMIC9 + +/* + * Defined if OSAtomic*() functions are available, as provided by Darwin, and + * documented in the atomic(3) manual page. + */ +#undef JEMALLOC_OSATOMIC + +/* + * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and + * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the + * functions are defined in libgcc instead of being inlines) + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 + +/* + * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and + * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite + * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the + * functions are defined in libgcc instead of being inlines) + */ +#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 + +/* + * Defined if OSSpin*() functions are available, as provided by Darwin, and + * documented in the spinlock(3) manual page. + */ +#undef JEMALLOC_OSSPIN + +/* + * Defined if _malloc_thread_cleanup() exists. At least in the case of + * FreeBSD, pthread_key_create() allocates, which if used during malloc + * bootstrapping will cause recursion into the pthreads library. Therefore, if + * _malloc_thread_cleanup() exists, use it as the basis for thread cleanup in + * malloc_tsd. + */ +#undef JEMALLOC_MALLOC_THREAD_CLEANUP + +/* + * Defined if threaded initialization is known to be safe on this platform. + * Among other things, it must be possible to initialize a mutex without + * triggering allocation in order for threaded allocation to be safe. + */ +#undef JEMALLOC_THREADED_INIT + +/* + * Defined if the pthreads implementation defines + * _pthread_mutex_init_calloc_cb(), in which case the function is used in order + * to avoid recursive allocation during mutex initialization. + */ +#undef JEMALLOC_MUTEX_INIT_CB + +/* Defined if sbrk() is supported. */ +#undef JEMALLOC_HAVE_SBRK + +/* Non-empty if the tls_model attribute is supported. */ +#undef JEMALLOC_TLS_MODEL + +/* JEMALLOC_CC_SILENCE enables code that silences unuseful compiler warnings. */ +#undef JEMALLOC_CC_SILENCE + +/* JEMALLOC_CODE_COVERAGE enables test code coverage analysis. */ +#undef JEMALLOC_CODE_COVERAGE + +/* + * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables + * inline functions. + */ +#undef JEMALLOC_DEBUG + +/* JEMALLOC_STATS enables statistics calculation. */ +#undef JEMALLOC_STATS + +/* JEMALLOC_PROF enables allocation profiling. */ +#undef JEMALLOC_PROF + +/* Use libunwind for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBUNWIND + +/* Use libgcc for profile backtracing if defined. */ +#undef JEMALLOC_PROF_LIBGCC + +/* Use gcc intrinsics for profile backtracing if defined. */ +#undef JEMALLOC_PROF_GCC + +/* + * JEMALLOC_TCACHE enables a thread-specific caching layer for small objects. + * This makes it possible to allocate/deallocate objects without any locking + * when the cache is in the steady state. + */ +#undef JEMALLOC_TCACHE + +/* + * JEMALLOC_DSS enables use of sbrk(2) to allocate chunks from the data storage + * segment (DSS). + */ +#undef JEMALLOC_DSS + +/* Support memory filling (junk/zero/quarantine/redzone). */ +#undef JEMALLOC_FILL + +/* Support utrace(2)-based tracing. */ +#undef JEMALLOC_UTRACE + +/* Support Valgrind. */ +#undef JEMALLOC_VALGRIND + +/* Support optional abort() on OOM. */ +#undef JEMALLOC_XMALLOC + +/* Support lazy locking (avoid locking unless a second thread is launched). */ +#undef JEMALLOC_LAZY_LOCK + +/* One page is 2^STATIC_PAGE_SHIFT bytes. */ +#undef STATIC_PAGE_SHIFT + +/* + * If defined, use munmap() to unmap freed chunks, rather than storing them for + * later reuse. This is disabled by default on Linux because common sequences + * of mmap()/munmap() calls will cause virtual memory map holes. + */ +#undef JEMALLOC_MUNMAP + +/* + * If defined, use mremap(...MREMAP_FIXED...) for huge realloc(). This is + * disabled by default because it is Linux-specific and it will cause virtual + * memory map holes, much like munmap(2) does. + */ +#undef JEMALLOC_MREMAP + +/* TLS is used to map arenas and magazine caches to threads. */ +#undef JEMALLOC_TLS + +/* + * JEMALLOC_IVSALLOC enables ivsalloc(), which verifies that pointers reside + * within jemalloc-owned chunks before dereferencing them. + */ +#undef JEMALLOC_IVSALLOC + +/* + * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. + */ +#undef JEMALLOC_ZONE +#undef JEMALLOC_ZONE_VERSION + +/* + * Methods for purging unused pages differ between operating systems. + * + * madvise(..., MADV_DONTNEED) : On Linux, this immediately discards pages, + * such that new pages will be demand-zeroed if + * the address region is later touched. + * madvise(..., MADV_FREE) : On FreeBSD and Darwin, this marks pages as being + * unused, such that they will be discarded rather + * than swapped out. + */ +#undef JEMALLOC_PURGE_MADVISE_DONTNEED +#undef JEMALLOC_PURGE_MADVISE_FREE + +/* + * Define if operating system has alloca.h header. + */ +#undef JEMALLOC_HAS_ALLOCA_H + +/* C99 restrict keyword supported. */ +#undef JEMALLOC_HAS_RESTRICT + +/* For use by hash code. */ +#undef JEMALLOC_BIG_ENDIAN + +/* sizeof(int) == 2^LG_SIZEOF_INT. */ +#undef LG_SIZEOF_INT + +/* sizeof(long) == 2^LG_SIZEOF_LONG. */ +#undef LG_SIZEOF_LONG + +/* sizeof(intmax_t) == 2^LG_SIZEOF_INTMAX_T. */ +#undef LG_SIZEOF_INTMAX_T + +#endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h new file mode 100644 index 00000000000..4e2392302c7 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -0,0 +1,51 @@ +/* + * JEMALLOC_ALWAYS_INLINE and JEMALLOC_INLINE are used within header files for + * functions that are static inline functions if inlining is enabled, and + * single-definition library-private functions if inlining is disabled. + * + * JEMALLOC_ALWAYS_INLINE_C and JEMALLOC_INLINE_C are for use in .c files, in + * which case the denoted functions are always static, regardless of whether + * inlining is enabled. + */ +#if defined(JEMALLOC_DEBUG) || defined(JEMALLOC_CODE_COVERAGE) + /* Disable inlining to make debugging/profiling easier. */ +# define JEMALLOC_ALWAYS_INLINE +# define JEMALLOC_ALWAYS_INLINE_C static +# define JEMALLOC_INLINE +# define JEMALLOC_INLINE_C static +# define inline +#else +# define JEMALLOC_ENABLE_INLINE +# ifdef JEMALLOC_HAVE_ATTR +# define JEMALLOC_ALWAYS_INLINE \ + static inline JEMALLOC_ATTR(unused) JEMALLOC_ATTR(always_inline) +# define JEMALLOC_ALWAYS_INLINE_C \ + static inline JEMALLOC_ATTR(always_inline) +# else +# define JEMALLOC_ALWAYS_INLINE static inline +# define JEMALLOC_ALWAYS_INLINE_C static inline +# endif +# define JEMALLOC_INLINE static inline +# define JEMALLOC_INLINE_C static inline +# ifdef _MSC_VER +# define inline _inline +# endif +#endif + +#ifdef JEMALLOC_CC_SILENCE +# define UNUSED JEMALLOC_ATTR(unused) +#else +# define UNUSED +#endif + +#define ZU(z) ((size_t)z) +#define QU(q) ((uint64_t)q) +#define QI(q) ((int64_t)q) + +#ifndef __DECONST +# define __DECONST(type, var) ((type)(uintptr_t)(const void *)(var)) +#endif + +#ifndef JEMALLOC_HAS_RESTRICT +# define restrict +#endif diff --git a/deps/jemalloc/include/jemalloc/internal/private_namespace.h b/deps/jemalloc/include/jemalloc/internal/private_namespace.h deleted file mode 100644 index 06241cd2fa1..00000000000 --- a/deps/jemalloc/include/jemalloc/internal/private_namespace.h +++ /dev/null @@ -1,367 +0,0 @@ -#define a0calloc JEMALLOC_N(a0calloc) -#define a0free JEMALLOC_N(a0free) -#define a0malloc JEMALLOC_N(a0malloc) -#define arena_alloc_junk_small JEMALLOC_N(arena_alloc_junk_small) -#define arena_bin_index JEMALLOC_N(arena_bin_index) -#define arena_bin_info JEMALLOC_N(arena_bin_info) -#define arena_boot JEMALLOC_N(arena_boot) -#define arena_dalloc JEMALLOC_N(arena_dalloc) -#define arena_dalloc_bin JEMALLOC_N(arena_dalloc_bin) -#define arena_dalloc_bin_locked JEMALLOC_N(arena_dalloc_bin_locked) -#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) -#define arena_dalloc_large JEMALLOC_N(arena_dalloc_large) -#define arena_dalloc_large_locked JEMALLOC_N(arena_dalloc_large_locked) -#define arena_dalloc_small JEMALLOC_N(arena_dalloc_small) -#define arena_dss_prec_get JEMALLOC_N(arena_dss_prec_get) -#define arena_dss_prec_set JEMALLOC_N(arena_dss_prec_set) -#define arena_malloc JEMALLOC_N(arena_malloc) -#define arena_malloc_large JEMALLOC_N(arena_malloc_large) -#define arena_malloc_small JEMALLOC_N(arena_malloc_small) -#define arena_mapbits_allocated_get JEMALLOC_N(arena_mapbits_allocated_get) -#define arena_mapbits_binind_get JEMALLOC_N(arena_mapbits_binind_get) -#define arena_mapbits_dirty_get JEMALLOC_N(arena_mapbits_dirty_get) -#define arena_mapbits_get JEMALLOC_N(arena_mapbits_get) -#define arena_mapbits_large_binind_set JEMALLOC_N(arena_mapbits_large_binind_set) -#define arena_mapbits_large_get JEMALLOC_N(arena_mapbits_large_get) -#define arena_mapbits_large_set JEMALLOC_N(arena_mapbits_large_set) -#define arena_mapbits_large_size_get JEMALLOC_N(arena_mapbits_large_size_get) -#define arena_mapbits_small_runind_get JEMALLOC_N(arena_mapbits_small_runind_get) -#define arena_mapbits_small_set JEMALLOC_N(arena_mapbits_small_set) -#define arena_mapbits_unallocated_set JEMALLOC_N(arena_mapbits_unallocated_set) -#define arena_mapbits_unallocated_size_get JEMALLOC_N(arena_mapbits_unallocated_size_get) -#define arena_mapbits_unallocated_size_set JEMALLOC_N(arena_mapbits_unallocated_size_set) -#define arena_mapbits_unzeroed_get JEMALLOC_N(arena_mapbits_unzeroed_get) -#define arena_mapbits_unzeroed_set JEMALLOC_N(arena_mapbits_unzeroed_set) -#define arena_mapbitsp_get JEMALLOC_N(arena_mapbitsp_get) -#define arena_mapp_get JEMALLOC_N(arena_mapp_get) -#define arena_maxclass JEMALLOC_N(arena_maxclass) -#define arena_new JEMALLOC_N(arena_new) -#define arena_palloc JEMALLOC_N(arena_palloc) -#define arena_postfork_child JEMALLOC_N(arena_postfork_child) -#define arena_postfork_parent JEMALLOC_N(arena_postfork_parent) -#define arena_prefork JEMALLOC_N(arena_prefork) -#define arena_prof_accum JEMALLOC_N(arena_prof_accum) -#define arena_prof_ctx_get JEMALLOC_N(arena_prof_ctx_get) -#define arena_prof_ctx_set JEMALLOC_N(arena_prof_ctx_set) -#define arena_prof_promoted JEMALLOC_N(arena_prof_promoted) -#define arena_ptr_small_binind_get JEMALLOC_N(arena_ptr_small_binind_get) -#define arena_purge_all JEMALLOC_N(arena_purge_all) -#define arena_ralloc JEMALLOC_N(arena_ralloc) -#define arena_ralloc_no_move JEMALLOC_N(arena_ralloc_no_move) -#define arena_run_regind JEMALLOC_N(arena_run_regind) -#define arena_salloc JEMALLOC_N(arena_salloc) -#define arena_stats_merge JEMALLOC_N(arena_stats_merge) -#define arena_tcache_fill_small JEMALLOC_N(arena_tcache_fill_small) -#define arenas JEMALLOC_N(arenas) -#define arenas_booted JEMALLOC_N(arenas_booted) -#define arenas_cleanup JEMALLOC_N(arenas_cleanup) -#define arenas_extend JEMALLOC_N(arenas_extend) -#define arenas_initialized JEMALLOC_N(arenas_initialized) -#define arenas_lock JEMALLOC_N(arenas_lock) -#define arenas_tls JEMALLOC_N(arenas_tls) -#define arenas_tsd JEMALLOC_N(arenas_tsd) -#define arenas_tsd_boot JEMALLOC_N(arenas_tsd_boot) -#define arenas_tsd_cleanup_wrapper JEMALLOC_N(arenas_tsd_cleanup_wrapper) -#define arenas_tsd_get JEMALLOC_N(arenas_tsd_get) -#define arenas_tsd_set JEMALLOC_N(arenas_tsd_set) -#define atomic_add_u JEMALLOC_N(atomic_add_u) -#define atomic_add_uint32 JEMALLOC_N(atomic_add_uint32) -#define atomic_add_uint64 JEMALLOC_N(atomic_add_uint64) -#define atomic_add_z JEMALLOC_N(atomic_add_z) -#define atomic_sub_u JEMALLOC_N(atomic_sub_u) -#define atomic_sub_uint32 JEMALLOC_N(atomic_sub_uint32) -#define atomic_sub_uint64 JEMALLOC_N(atomic_sub_uint64) -#define atomic_sub_z JEMALLOC_N(atomic_sub_z) -#define base_alloc JEMALLOC_N(base_alloc) -#define base_boot JEMALLOC_N(base_boot) -#define base_calloc JEMALLOC_N(base_calloc) -#define base_node_alloc JEMALLOC_N(base_node_alloc) -#define base_node_dealloc JEMALLOC_N(base_node_dealloc) -#define base_postfork_child JEMALLOC_N(base_postfork_child) -#define base_postfork_parent JEMALLOC_N(base_postfork_parent) -#define base_prefork JEMALLOC_N(base_prefork) -#define bitmap_full JEMALLOC_N(bitmap_full) -#define bitmap_get JEMALLOC_N(bitmap_get) -#define bitmap_info_init JEMALLOC_N(bitmap_info_init) -#define bitmap_info_ngroups JEMALLOC_N(bitmap_info_ngroups) -#define bitmap_init JEMALLOC_N(bitmap_init) -#define bitmap_set JEMALLOC_N(bitmap_set) -#define bitmap_sfu JEMALLOC_N(bitmap_sfu) -#define bitmap_size JEMALLOC_N(bitmap_size) -#define bitmap_unset JEMALLOC_N(bitmap_unset) -#define bt_init JEMALLOC_N(bt_init) -#define buferror JEMALLOC_N(buferror) -#define choose_arena JEMALLOC_N(choose_arena) -#define choose_arena_hard JEMALLOC_N(choose_arena_hard) -#define chunk_alloc JEMALLOC_N(chunk_alloc) -#define chunk_alloc_dss JEMALLOC_N(chunk_alloc_dss) -#define chunk_alloc_mmap JEMALLOC_N(chunk_alloc_mmap) -#define chunk_boot JEMALLOC_N(chunk_boot) -#define chunk_dealloc JEMALLOC_N(chunk_dealloc) -#define chunk_dealloc_mmap JEMALLOC_N(chunk_dealloc_mmap) -#define chunk_dss_boot JEMALLOC_N(chunk_dss_boot) -#define chunk_dss_postfork_child JEMALLOC_N(chunk_dss_postfork_child) -#define chunk_dss_postfork_parent JEMALLOC_N(chunk_dss_postfork_parent) -#define chunk_dss_prec_get JEMALLOC_N(chunk_dss_prec_get) -#define chunk_dss_prec_set JEMALLOC_N(chunk_dss_prec_set) -#define chunk_dss_prefork JEMALLOC_N(chunk_dss_prefork) -#define chunk_in_dss JEMALLOC_N(chunk_in_dss) -#define chunk_npages JEMALLOC_N(chunk_npages) -#define chunk_postfork_child JEMALLOC_N(chunk_postfork_child) -#define chunk_postfork_parent JEMALLOC_N(chunk_postfork_parent) -#define chunk_prefork JEMALLOC_N(chunk_prefork) -#define chunk_unmap JEMALLOC_N(chunk_unmap) -#define chunks_mtx JEMALLOC_N(chunks_mtx) -#define chunks_rtree JEMALLOC_N(chunks_rtree) -#define chunksize JEMALLOC_N(chunksize) -#define chunksize_mask JEMALLOC_N(chunksize_mask) -#define ckh_bucket_search JEMALLOC_N(ckh_bucket_search) -#define ckh_count JEMALLOC_N(ckh_count) -#define ckh_delete JEMALLOC_N(ckh_delete) -#define ckh_evict_reloc_insert JEMALLOC_N(ckh_evict_reloc_insert) -#define ckh_insert JEMALLOC_N(ckh_insert) -#define ckh_isearch JEMALLOC_N(ckh_isearch) -#define ckh_iter JEMALLOC_N(ckh_iter) -#define ckh_new JEMALLOC_N(ckh_new) -#define ckh_pointer_hash JEMALLOC_N(ckh_pointer_hash) -#define ckh_pointer_keycomp JEMALLOC_N(ckh_pointer_keycomp) -#define ckh_rebuild JEMALLOC_N(ckh_rebuild) -#define ckh_remove JEMALLOC_N(ckh_remove) -#define ckh_search JEMALLOC_N(ckh_search) -#define ckh_string_hash JEMALLOC_N(ckh_string_hash) -#define ckh_string_keycomp JEMALLOC_N(ckh_string_keycomp) -#define ckh_try_bucket_insert JEMALLOC_N(ckh_try_bucket_insert) -#define ckh_try_insert JEMALLOC_N(ckh_try_insert) -#define ctl_boot JEMALLOC_N(ctl_boot) -#define ctl_bymib JEMALLOC_N(ctl_bymib) -#define ctl_byname JEMALLOC_N(ctl_byname) -#define ctl_nametomib JEMALLOC_N(ctl_nametomib) -#define ctl_postfork_child JEMALLOC_N(ctl_postfork_child) -#define ctl_postfork_parent JEMALLOC_N(ctl_postfork_parent) -#define ctl_prefork JEMALLOC_N(ctl_prefork) -#define dss_prec_names JEMALLOC_N(dss_prec_names) -#define extent_tree_ad_first JEMALLOC_N(extent_tree_ad_first) -#define extent_tree_ad_insert JEMALLOC_N(extent_tree_ad_insert) -#define extent_tree_ad_iter JEMALLOC_N(extent_tree_ad_iter) -#define extent_tree_ad_iter_recurse JEMALLOC_N(extent_tree_ad_iter_recurse) -#define extent_tree_ad_iter_start JEMALLOC_N(extent_tree_ad_iter_start) -#define extent_tree_ad_last JEMALLOC_N(extent_tree_ad_last) -#define extent_tree_ad_new JEMALLOC_N(extent_tree_ad_new) -#define extent_tree_ad_next JEMALLOC_N(extent_tree_ad_next) -#define extent_tree_ad_nsearch JEMALLOC_N(extent_tree_ad_nsearch) -#define extent_tree_ad_prev JEMALLOC_N(extent_tree_ad_prev) -#define extent_tree_ad_psearch JEMALLOC_N(extent_tree_ad_psearch) -#define extent_tree_ad_remove JEMALLOC_N(extent_tree_ad_remove) -#define extent_tree_ad_reverse_iter JEMALLOC_N(extent_tree_ad_reverse_iter) -#define extent_tree_ad_reverse_iter_recurse JEMALLOC_N(extent_tree_ad_reverse_iter_recurse) -#define extent_tree_ad_reverse_iter_start JEMALLOC_N(extent_tree_ad_reverse_iter_start) -#define extent_tree_ad_search JEMALLOC_N(extent_tree_ad_search) -#define extent_tree_szad_first JEMALLOC_N(extent_tree_szad_first) -#define extent_tree_szad_insert JEMALLOC_N(extent_tree_szad_insert) -#define extent_tree_szad_iter JEMALLOC_N(extent_tree_szad_iter) -#define extent_tree_szad_iter_recurse JEMALLOC_N(extent_tree_szad_iter_recurse) -#define extent_tree_szad_iter_start JEMALLOC_N(extent_tree_szad_iter_start) -#define extent_tree_szad_last JEMALLOC_N(extent_tree_szad_last) -#define extent_tree_szad_new JEMALLOC_N(extent_tree_szad_new) -#define extent_tree_szad_next JEMALLOC_N(extent_tree_szad_next) -#define extent_tree_szad_nsearch JEMALLOC_N(extent_tree_szad_nsearch) -#define extent_tree_szad_prev JEMALLOC_N(extent_tree_szad_prev) -#define extent_tree_szad_psearch JEMALLOC_N(extent_tree_szad_psearch) -#define extent_tree_szad_remove JEMALLOC_N(extent_tree_szad_remove) -#define extent_tree_szad_reverse_iter JEMALLOC_N(extent_tree_szad_reverse_iter) -#define extent_tree_szad_reverse_iter_recurse JEMALLOC_N(extent_tree_szad_reverse_iter_recurse) -#define extent_tree_szad_reverse_iter_start JEMALLOC_N(extent_tree_szad_reverse_iter_start) -#define extent_tree_szad_search JEMALLOC_N(extent_tree_szad_search) -#define get_errno JEMALLOC_N(get_errno) -#define hash JEMALLOC_N(hash) -#define huge_allocated JEMALLOC_N(huge_allocated) -#define huge_boot JEMALLOC_N(huge_boot) -#define huge_dalloc JEMALLOC_N(huge_dalloc) -#define huge_malloc JEMALLOC_N(huge_malloc) -#define huge_mtx JEMALLOC_N(huge_mtx) -#define huge_ndalloc JEMALLOC_N(huge_ndalloc) -#define huge_nmalloc JEMALLOC_N(huge_nmalloc) -#define huge_palloc JEMALLOC_N(huge_palloc) -#define huge_postfork_child JEMALLOC_N(huge_postfork_child) -#define huge_postfork_parent JEMALLOC_N(huge_postfork_parent) -#define huge_prefork JEMALLOC_N(huge_prefork) -#define huge_prof_ctx_get JEMALLOC_N(huge_prof_ctx_get) -#define huge_prof_ctx_set JEMALLOC_N(huge_prof_ctx_set) -#define huge_ralloc JEMALLOC_N(huge_ralloc) -#define huge_ralloc_no_move JEMALLOC_N(huge_ralloc_no_move) -#define huge_salloc JEMALLOC_N(huge_salloc) -#define iallocm JEMALLOC_N(iallocm) -#define icalloc JEMALLOC_N(icalloc) -#define icallocx JEMALLOC_N(icallocx) -#define idalloc JEMALLOC_N(idalloc) -#define idallocx JEMALLOC_N(idallocx) -#define imalloc JEMALLOC_N(imalloc) -#define imallocx JEMALLOC_N(imallocx) -#define ipalloc JEMALLOC_N(ipalloc) -#define ipallocx JEMALLOC_N(ipallocx) -#define iqalloc JEMALLOC_N(iqalloc) -#define iqallocx JEMALLOC_N(iqallocx) -#define iralloc JEMALLOC_N(iralloc) -#define irallocx JEMALLOC_N(irallocx) -#define isalloc JEMALLOC_N(isalloc) -#define isthreaded JEMALLOC_N(isthreaded) -#define ivsalloc JEMALLOC_N(ivsalloc) -#define jemalloc_postfork_child JEMALLOC_N(jemalloc_postfork_child) -#define jemalloc_postfork_parent JEMALLOC_N(jemalloc_postfork_parent) -#define jemalloc_prefork JEMALLOC_N(jemalloc_prefork) -#define malloc_cprintf JEMALLOC_N(malloc_cprintf) -#define malloc_mutex_init JEMALLOC_N(malloc_mutex_init) -#define malloc_mutex_lock JEMALLOC_N(malloc_mutex_lock) -#define malloc_mutex_postfork_child JEMALLOC_N(malloc_mutex_postfork_child) -#define malloc_mutex_postfork_parent JEMALLOC_N(malloc_mutex_postfork_parent) -#define malloc_mutex_prefork JEMALLOC_N(malloc_mutex_prefork) -#define malloc_mutex_unlock JEMALLOC_N(malloc_mutex_unlock) -#define malloc_printf JEMALLOC_N(malloc_printf) -#define malloc_snprintf JEMALLOC_N(malloc_snprintf) -#define malloc_strtoumax JEMALLOC_N(malloc_strtoumax) -#define malloc_tsd_boot JEMALLOC_N(malloc_tsd_boot) -#define malloc_tsd_cleanup_register JEMALLOC_N(malloc_tsd_cleanup_register) -#define malloc_tsd_dalloc JEMALLOC_N(malloc_tsd_dalloc) -#define malloc_tsd_malloc JEMALLOC_N(malloc_tsd_malloc) -#define malloc_tsd_no_cleanup JEMALLOC_N(malloc_tsd_no_cleanup) -#define malloc_vcprintf JEMALLOC_N(malloc_vcprintf) -#define malloc_vsnprintf JEMALLOC_N(malloc_vsnprintf) -#define malloc_write JEMALLOC_N(malloc_write) -#define map_bias JEMALLOC_N(map_bias) -#define mb_write JEMALLOC_N(mb_write) -#define mutex_boot JEMALLOC_N(mutex_boot) -#define narenas_auto JEMALLOC_N(narenas_auto) -#define narenas_total JEMALLOC_N(narenas_total) -#define narenas_total_get JEMALLOC_N(narenas_total_get) -#define ncpus JEMALLOC_N(ncpus) -#define nhbins JEMALLOC_N(nhbins) -#define opt_abort JEMALLOC_N(opt_abort) -#define opt_junk JEMALLOC_N(opt_junk) -#define opt_lg_chunk JEMALLOC_N(opt_lg_chunk) -#define opt_lg_dirty_mult JEMALLOC_N(opt_lg_dirty_mult) -#define opt_lg_prof_interval JEMALLOC_N(opt_lg_prof_interval) -#define opt_lg_prof_sample JEMALLOC_N(opt_lg_prof_sample) -#define opt_lg_tcache_max JEMALLOC_N(opt_lg_tcache_max) -#define opt_narenas JEMALLOC_N(opt_narenas) -#define opt_prof JEMALLOC_N(opt_prof) -#define opt_prof_accum JEMALLOC_N(opt_prof_accum) -#define opt_prof_active JEMALLOC_N(opt_prof_active) -#define opt_prof_final JEMALLOC_N(opt_prof_final) -#define opt_prof_gdump JEMALLOC_N(opt_prof_gdump) -#define opt_prof_leak JEMALLOC_N(opt_prof_leak) -#define opt_prof_prefix JEMALLOC_N(opt_prof_prefix) -#define opt_quarantine JEMALLOC_N(opt_quarantine) -#define opt_redzone JEMALLOC_N(opt_redzone) -#define opt_stats_print JEMALLOC_N(opt_stats_print) -#define opt_tcache JEMALLOC_N(opt_tcache) -#define opt_utrace JEMALLOC_N(opt_utrace) -#define opt_valgrind JEMALLOC_N(opt_valgrind) -#define opt_xmalloc JEMALLOC_N(opt_xmalloc) -#define opt_zero JEMALLOC_N(opt_zero) -#define p2rz JEMALLOC_N(p2rz) -#define pages_purge JEMALLOC_N(pages_purge) -#define pow2_ceil JEMALLOC_N(pow2_ceil) -#define prof_backtrace JEMALLOC_N(prof_backtrace) -#define prof_boot0 JEMALLOC_N(prof_boot0) -#define prof_boot1 JEMALLOC_N(prof_boot1) -#define prof_boot2 JEMALLOC_N(prof_boot2) -#define prof_ctx_get JEMALLOC_N(prof_ctx_get) -#define prof_ctx_set JEMALLOC_N(prof_ctx_set) -#define prof_free JEMALLOC_N(prof_free) -#define prof_gdump JEMALLOC_N(prof_gdump) -#define prof_idump JEMALLOC_N(prof_idump) -#define prof_interval JEMALLOC_N(prof_interval) -#define prof_lookup JEMALLOC_N(prof_lookup) -#define prof_malloc JEMALLOC_N(prof_malloc) -#define prof_mdump JEMALLOC_N(prof_mdump) -#define prof_postfork_child JEMALLOC_N(prof_postfork_child) -#define prof_postfork_parent JEMALLOC_N(prof_postfork_parent) -#define prof_prefork JEMALLOC_N(prof_prefork) -#define prof_promote JEMALLOC_N(prof_promote) -#define prof_realloc JEMALLOC_N(prof_realloc) -#define prof_sample_accum_update JEMALLOC_N(prof_sample_accum_update) -#define prof_sample_threshold_update JEMALLOC_N(prof_sample_threshold_update) -#define prof_tdata_booted JEMALLOC_N(prof_tdata_booted) -#define prof_tdata_cleanup JEMALLOC_N(prof_tdata_cleanup) -#define prof_tdata_get JEMALLOC_N(prof_tdata_get) -#define prof_tdata_init JEMALLOC_N(prof_tdata_init) -#define prof_tdata_initialized JEMALLOC_N(prof_tdata_initialized) -#define prof_tdata_tls JEMALLOC_N(prof_tdata_tls) -#define prof_tdata_tsd JEMALLOC_N(prof_tdata_tsd) -#define prof_tdata_tsd_boot JEMALLOC_N(prof_tdata_tsd_boot) -#define prof_tdata_tsd_cleanup_wrapper JEMALLOC_N(prof_tdata_tsd_cleanup_wrapper) -#define prof_tdata_tsd_get JEMALLOC_N(prof_tdata_tsd_get) -#define prof_tdata_tsd_set JEMALLOC_N(prof_tdata_tsd_set) -#define quarantine JEMALLOC_N(quarantine) -#define quarantine_boot JEMALLOC_N(quarantine_boot) -#define quarantine_tsd_boot JEMALLOC_N(quarantine_tsd_boot) -#define quarantine_tsd_cleanup_wrapper JEMALLOC_N(quarantine_tsd_cleanup_wrapper) -#define quarantine_tsd_get JEMALLOC_N(quarantine_tsd_get) -#define quarantine_tsd_set JEMALLOC_N(quarantine_tsd_set) -#define register_zone JEMALLOC_N(register_zone) -#define rtree_get JEMALLOC_N(rtree_get) -#define rtree_get_locked JEMALLOC_N(rtree_get_locked) -#define rtree_new JEMALLOC_N(rtree_new) -#define rtree_postfork_child JEMALLOC_N(rtree_postfork_child) -#define rtree_postfork_parent JEMALLOC_N(rtree_postfork_parent) -#define rtree_prefork JEMALLOC_N(rtree_prefork) -#define rtree_set JEMALLOC_N(rtree_set) -#define s2u JEMALLOC_N(s2u) -#define sa2u JEMALLOC_N(sa2u) -#define set_errno JEMALLOC_N(set_errno) -#define stats_cactive JEMALLOC_N(stats_cactive) -#define stats_cactive_add JEMALLOC_N(stats_cactive_add) -#define stats_cactive_get JEMALLOC_N(stats_cactive_get) -#define stats_cactive_sub JEMALLOC_N(stats_cactive_sub) -#define stats_chunks JEMALLOC_N(stats_chunks) -#define stats_print JEMALLOC_N(stats_print) -#define tcache_alloc_easy JEMALLOC_N(tcache_alloc_easy) -#define tcache_alloc_large JEMALLOC_N(tcache_alloc_large) -#define tcache_alloc_small JEMALLOC_N(tcache_alloc_small) -#define tcache_alloc_small_hard JEMALLOC_N(tcache_alloc_small_hard) -#define tcache_arena_associate JEMALLOC_N(tcache_arena_associate) -#define tcache_arena_dissociate JEMALLOC_N(tcache_arena_dissociate) -#define tcache_bin_flush_large JEMALLOC_N(tcache_bin_flush_large) -#define tcache_bin_flush_small JEMALLOC_N(tcache_bin_flush_small) -#define tcache_bin_info JEMALLOC_N(tcache_bin_info) -#define tcache_boot0 JEMALLOC_N(tcache_boot0) -#define tcache_boot1 JEMALLOC_N(tcache_boot1) -#define tcache_booted JEMALLOC_N(tcache_booted) -#define tcache_create JEMALLOC_N(tcache_create) -#define tcache_dalloc_large JEMALLOC_N(tcache_dalloc_large) -#define tcache_dalloc_small JEMALLOC_N(tcache_dalloc_small) -#define tcache_destroy JEMALLOC_N(tcache_destroy) -#define tcache_enabled_booted JEMALLOC_N(tcache_enabled_booted) -#define tcache_enabled_get JEMALLOC_N(tcache_enabled_get) -#define tcache_enabled_initialized JEMALLOC_N(tcache_enabled_initialized) -#define tcache_enabled_set JEMALLOC_N(tcache_enabled_set) -#define tcache_enabled_tls JEMALLOC_N(tcache_enabled_tls) -#define tcache_enabled_tsd JEMALLOC_N(tcache_enabled_tsd) -#define tcache_enabled_tsd_boot JEMALLOC_N(tcache_enabled_tsd_boot) -#define tcache_enabled_tsd_cleanup_wrapper JEMALLOC_N(tcache_enabled_tsd_cleanup_wrapper) -#define tcache_enabled_tsd_get JEMALLOC_N(tcache_enabled_tsd_get) -#define tcache_enabled_tsd_set JEMALLOC_N(tcache_enabled_tsd_set) -#define tcache_event JEMALLOC_N(tcache_event) -#define tcache_event_hard JEMALLOC_N(tcache_event_hard) -#define tcache_flush JEMALLOC_N(tcache_flush) -#define tcache_get JEMALLOC_N(tcache_get) -#define tcache_initialized JEMALLOC_N(tcache_initialized) -#define tcache_maxclass JEMALLOC_N(tcache_maxclass) -#define tcache_salloc JEMALLOC_N(tcache_salloc) -#define tcache_stats_merge JEMALLOC_N(tcache_stats_merge) -#define tcache_thread_cleanup JEMALLOC_N(tcache_thread_cleanup) -#define tcache_tls JEMALLOC_N(tcache_tls) -#define tcache_tsd JEMALLOC_N(tcache_tsd) -#define tcache_tsd_boot JEMALLOC_N(tcache_tsd_boot) -#define tcache_tsd_cleanup_wrapper JEMALLOC_N(tcache_tsd_cleanup_wrapper) -#define tcache_tsd_get JEMALLOC_N(tcache_tsd_get) -#define tcache_tsd_set JEMALLOC_N(tcache_tsd_set) -#define thread_allocated_booted JEMALLOC_N(thread_allocated_booted) -#define thread_allocated_initialized JEMALLOC_N(thread_allocated_initialized) -#define thread_allocated_tls JEMALLOC_N(thread_allocated_tls) -#define thread_allocated_tsd JEMALLOC_N(thread_allocated_tsd) -#define thread_allocated_tsd_boot JEMALLOC_N(thread_allocated_tsd_boot) -#define thread_allocated_tsd_cleanup_wrapper JEMALLOC_N(thread_allocated_tsd_cleanup_wrapper) -#define thread_allocated_tsd_get JEMALLOC_N(thread_allocated_tsd_get) -#define thread_allocated_tsd_set JEMALLOC_N(thread_allocated_tsd_set) -#define u2rz JEMALLOC_N(u2rz) diff --git a/deps/jemalloc/include/jemalloc/internal/private_namespace.sh b/deps/jemalloc/include/jemalloc/internal/private_namespace.sh new file mode 100755 index 00000000000..cd25eb3061e --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/private_namespace.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +for symbol in `cat $1` ; do + echo "#define ${symbol} JEMALLOC_N(${symbol})" +done diff --git a/deps/jemalloc/include/jemalloc/internal/private_symbols.txt b/deps/jemalloc/include/jemalloc/internal/private_symbols.txt new file mode 100644 index 00000000000..93516d242b6 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/private_symbols.txt @@ -0,0 +1,413 @@ +a0calloc +a0free +a0malloc +arena_alloc_junk_small +arena_bin_index +arena_bin_info +arena_boot +arena_dalloc +arena_dalloc_bin +arena_dalloc_bin_locked +arena_dalloc_junk_large +arena_dalloc_junk_small +arena_dalloc_large +arena_dalloc_large_locked +arena_dalloc_small +arena_dss_prec_get +arena_dss_prec_set +arena_malloc +arena_malloc_large +arena_malloc_small +arena_mapbits_allocated_get +arena_mapbits_binind_get +arena_mapbits_dirty_get +arena_mapbits_get +arena_mapbits_large_binind_set +arena_mapbits_large_get +arena_mapbits_large_set +arena_mapbits_large_size_get +arena_mapbits_small_runind_get +arena_mapbits_small_set +arena_mapbits_unallocated_set +arena_mapbits_unallocated_size_get +arena_mapbits_unallocated_size_set +arena_mapbits_unzeroed_get +arena_mapbits_unzeroed_set +arena_mapbitsp_get +arena_mapbitsp_read +arena_mapbitsp_write +arena_mapp_get +arena_maxclass +arena_new +arena_palloc +arena_postfork_child +arena_postfork_parent +arena_prefork +arena_prof_accum +arena_prof_accum_impl +arena_prof_accum_locked +arena_prof_ctx_get +arena_prof_ctx_set +arena_prof_promoted +arena_ptr_small_binind_get +arena_purge_all +arena_quarantine_junk_small +arena_ralloc +arena_ralloc_junk_large +arena_ralloc_no_move +arena_redzone_corruption +arena_run_regind +arena_salloc +arena_stats_merge +arena_tcache_fill_small +arenas +arenas_booted +arenas_cleanup +arenas_extend +arenas_initialized +arenas_lock +arenas_tls +arenas_tsd +arenas_tsd_boot +arenas_tsd_cleanup_wrapper +arenas_tsd_get +arenas_tsd_get_wrapper +arenas_tsd_init_head +arenas_tsd_set +atomic_add_u +atomic_add_uint32 +atomic_add_uint64 +atomic_add_z +atomic_sub_u +atomic_sub_uint32 +atomic_sub_uint64 +atomic_sub_z +base_alloc +base_boot +base_calloc +base_node_alloc +base_node_dealloc +base_postfork_child +base_postfork_parent +base_prefork +bitmap_full +bitmap_get +bitmap_info_init +bitmap_info_ngroups +bitmap_init +bitmap_set +bitmap_sfu +bitmap_size +bitmap_unset +bt_init +buferror +choose_arena +choose_arena_hard +chunk_alloc +chunk_alloc_dss +chunk_alloc_mmap +chunk_boot +chunk_dealloc +chunk_dealloc_mmap +chunk_dss_boot +chunk_dss_postfork_child +chunk_dss_postfork_parent +chunk_dss_prec_get +chunk_dss_prec_set +chunk_dss_prefork +chunk_in_dss +chunk_npages +chunk_postfork_child +chunk_postfork_parent +chunk_prefork +chunk_unmap +chunks_mtx +chunks_rtree +chunksize +chunksize_mask +ckh_bucket_search +ckh_count +ckh_delete +ckh_evict_reloc_insert +ckh_insert +ckh_isearch +ckh_iter +ckh_new +ckh_pointer_hash +ckh_pointer_keycomp +ckh_rebuild +ckh_remove +ckh_search +ckh_string_hash +ckh_string_keycomp +ckh_try_bucket_insert +ckh_try_insert +ctl_boot +ctl_bymib +ctl_byname +ctl_nametomib +ctl_postfork_child +ctl_postfork_parent +ctl_prefork +dss_prec_names +extent_tree_ad_first +extent_tree_ad_insert +extent_tree_ad_iter +extent_tree_ad_iter_recurse +extent_tree_ad_iter_start +extent_tree_ad_last +extent_tree_ad_new +extent_tree_ad_next +extent_tree_ad_nsearch +extent_tree_ad_prev +extent_tree_ad_psearch +extent_tree_ad_remove +extent_tree_ad_reverse_iter +extent_tree_ad_reverse_iter_recurse +extent_tree_ad_reverse_iter_start +extent_tree_ad_search +extent_tree_szad_first +extent_tree_szad_insert +extent_tree_szad_iter +extent_tree_szad_iter_recurse +extent_tree_szad_iter_start +extent_tree_szad_last +extent_tree_szad_new +extent_tree_szad_next +extent_tree_szad_nsearch +extent_tree_szad_prev +extent_tree_szad_psearch +extent_tree_szad_remove +extent_tree_szad_reverse_iter +extent_tree_szad_reverse_iter_recurse +extent_tree_szad_reverse_iter_start +extent_tree_szad_search +get_errno +hash +hash_fmix_32 +hash_fmix_64 +hash_get_block_32 +hash_get_block_64 +hash_rotl_32 +hash_rotl_64 +hash_x64_128 +hash_x86_128 +hash_x86_32 +huge_allocated +huge_boot +huge_dalloc +huge_dalloc_junk +huge_dss_prec_get +huge_malloc +huge_mtx +huge_ndalloc +huge_nmalloc +huge_palloc +huge_postfork_child +huge_postfork_parent +huge_prefork +huge_prof_ctx_get +huge_prof_ctx_set +huge_ralloc +huge_ralloc_no_move +huge_salloc +iallocm +icalloc +icalloct +idalloc +idalloct +imalloc +imalloct +ipalloc +ipalloct +iqalloc +iqalloct +iralloc +iralloct +iralloct_realign +isalloc +isthreaded +ivsalloc +ixalloc +jemalloc_postfork_child +jemalloc_postfork_parent +jemalloc_prefork +malloc_cprintf +malloc_mutex_init +malloc_mutex_lock +malloc_mutex_postfork_child +malloc_mutex_postfork_parent +malloc_mutex_prefork +malloc_mutex_unlock +malloc_printf +malloc_snprintf +malloc_strtoumax +malloc_tsd_boot +malloc_tsd_cleanup_register +malloc_tsd_dalloc +malloc_tsd_malloc +malloc_tsd_no_cleanup +malloc_vcprintf +malloc_vsnprintf +malloc_write +map_bias +mb_write +mutex_boot +narenas_auto +narenas_total +narenas_total_get +ncpus +nhbins +opt_abort +opt_dss +opt_junk +opt_lg_chunk +opt_lg_dirty_mult +opt_lg_prof_interval +opt_lg_prof_sample +opt_lg_tcache_max +opt_narenas +opt_prof +opt_prof_accum +opt_prof_active +opt_prof_final +opt_prof_gdump +opt_prof_leak +opt_prof_prefix +opt_quarantine +opt_redzone +opt_stats_print +opt_tcache +opt_utrace +opt_valgrind +opt_xmalloc +opt_zero +p2rz +pages_purge +pow2_ceil +prof_backtrace +prof_boot0 +prof_boot1 +prof_boot2 +prof_bt_count +prof_ctx_get +prof_ctx_set +prof_dump_open +prof_free +prof_gdump +prof_idump +prof_interval +prof_lookup +prof_malloc +prof_mdump +prof_postfork_child +prof_postfork_parent +prof_prefork +prof_promote +prof_realloc +prof_sample_accum_update +prof_sample_threshold_update +prof_tdata_booted +prof_tdata_cleanup +prof_tdata_get +prof_tdata_init +prof_tdata_initialized +prof_tdata_tls +prof_tdata_tsd +prof_tdata_tsd_boot +prof_tdata_tsd_cleanup_wrapper +prof_tdata_tsd_get +prof_tdata_tsd_get_wrapper +prof_tdata_tsd_init_head +prof_tdata_tsd_set +quarantine +quarantine_alloc_hook +quarantine_boot +quarantine_booted +quarantine_cleanup +quarantine_init +quarantine_tls +quarantine_tsd +quarantine_tsd_boot +quarantine_tsd_cleanup_wrapper +quarantine_tsd_get +quarantine_tsd_get_wrapper +quarantine_tsd_init_head +quarantine_tsd_set +register_zone +rtree_delete +rtree_get +rtree_get_locked +rtree_new +rtree_postfork_child +rtree_postfork_parent +rtree_prefork +rtree_set +s2u +sa2u +set_errno +small_size2bin +stats_cactive +stats_cactive_add +stats_cactive_get +stats_cactive_sub +stats_chunks +stats_print +tcache_alloc_easy +tcache_alloc_large +tcache_alloc_small +tcache_alloc_small_hard +tcache_arena_associate +tcache_arena_dissociate +tcache_bin_flush_large +tcache_bin_flush_small +tcache_bin_info +tcache_boot0 +tcache_boot1 +tcache_booted +tcache_create +tcache_dalloc_large +tcache_dalloc_small +tcache_destroy +tcache_enabled_booted +tcache_enabled_get +tcache_enabled_initialized +tcache_enabled_set +tcache_enabled_tls +tcache_enabled_tsd +tcache_enabled_tsd_boot +tcache_enabled_tsd_cleanup_wrapper +tcache_enabled_tsd_get +tcache_enabled_tsd_get_wrapper +tcache_enabled_tsd_init_head +tcache_enabled_tsd_set +tcache_event +tcache_event_hard +tcache_flush +tcache_get +tcache_initialized +tcache_maxclass +tcache_salloc +tcache_stats_merge +tcache_thread_cleanup +tcache_tls +tcache_tsd +tcache_tsd_boot +tcache_tsd_cleanup_wrapper +tcache_tsd_get +tcache_tsd_get_wrapper +tcache_tsd_init_head +tcache_tsd_set +thread_allocated_booted +thread_allocated_initialized +thread_allocated_tls +thread_allocated_tsd +thread_allocated_tsd_boot +thread_allocated_tsd_cleanup_wrapper +thread_allocated_tsd_get +thread_allocated_tsd_get_wrapper +thread_allocated_tsd_init_head +thread_allocated_tsd_set +tsd_init_check_recursion +tsd_init_finish +u2rz diff --git a/deps/jemalloc/include/jemalloc/internal/private_unnamespace.sh b/deps/jemalloc/include/jemalloc/internal/private_unnamespace.sh new file mode 100755 index 00000000000..23fed8e8034 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/private_unnamespace.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +for symbol in `cat $1` ; do + echo "#undef ${symbol}" +done diff --git a/deps/jemalloc/include/jemalloc/internal/prng.h b/deps/jemalloc/include/jemalloc/internal/prng.h index 83a5462b4dd..7b2b06512ff 100644 --- a/deps/jemalloc/include/jemalloc/internal/prng.h +++ b/deps/jemalloc/include/jemalloc/internal/prng.h @@ -25,7 +25,7 @@ * uint32_t state : Seed value. * const uint32_t a, c : See above discussion. */ -#define prng32(r, lg_range, state, a, c) do { \ +#define prng32(r, lg_range, state, a, c) do { \ assert(lg_range > 0); \ assert(lg_range <= 32); \ \ @@ -35,7 +35,7 @@ } while (false) /* Same as prng32(), but 64 bits of pseudo-randomness, using uint64_t. */ -#define prng64(r, lg_range, state, a, c) do { \ +#define prng64(r, lg_range, state, a, c) do { \ assert(lg_range > 0); \ assert(lg_range <= 64); \ \ diff --git a/deps/jemalloc/include/jemalloc/internal/prof.h b/deps/jemalloc/include/jemalloc/internal/prof.h index 47f22ad2da1..6f162d21e84 100644 --- a/deps/jemalloc/include/jemalloc/internal/prof.h +++ b/deps/jemalloc/include/jemalloc/internal/prof.h @@ -8,7 +8,11 @@ typedef struct prof_ctx_s prof_ctx_t; typedef struct prof_tdata_s prof_tdata_t; /* Option defaults. */ -#define PROF_PREFIX_DEFAULT "jeprof" +#ifdef JEMALLOC_PROF +# define PROF_PREFIX_DEFAULT "jeprof" +#else +# define PROF_PREFIX_DEFAULT "" +#endif #define LG_PROF_SAMPLE_DEFAULT 19 #define LG_PROF_INTERVAL_DEFAULT -1 @@ -129,6 +133,7 @@ struct prof_ctx_s { * limbo due to one of: * - Initializing per thread counters associated with this ctx. * - Preparing to destroy this ctx. + * - Dumping a heap profile that includes this ctx. * nlimbo must be 1 (single destroyer) in order to safely destroy the * ctx. */ @@ -145,7 +150,11 @@ struct prof_ctx_s { * this context. */ ql_head(prof_thr_cnt_t) cnts_ql; + + /* Linkage for list of contexts to be dumped. */ + ql_elm(prof_ctx_t) dump_link; }; +typedef ql_head(prof_ctx_t) prof_ctx_list_t; struct prof_tdata_s { /* @@ -195,7 +204,12 @@ extern bool opt_prof_gdump; /* High-water memory dumping. */ extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ -extern char opt_prof_prefix[PATH_MAX + 1]; +extern char opt_prof_prefix[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; /* * Profile dump interval, measured in bytes allocated. Each arena triggers a @@ -215,6 +229,11 @@ extern bool prof_promote; void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt, unsigned nignore); prof_thr_cnt_t *prof_lookup(prof_bt_t *bt); +#ifdef JEMALLOC_JET +size_t prof_bt_count(void); +typedef int (prof_dump_open_t)(bool, const char *); +extern prof_dump_open_t *prof_dump_open; +#endif void prof_idump(void); bool prof_mdump(const char *filename); void prof_gdump(void); @@ -237,7 +256,7 @@ void prof_postfork_child(void); \ assert(size == s2u(size)); \ \ - prof_tdata = prof_tdata_get(); \ + prof_tdata = prof_tdata_get(true); \ if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { \ if (prof_tdata != NULL) \ ret = (prof_thr_cnt_t *)(uintptr_t)1U; \ @@ -286,14 +305,14 @@ void prof_postfork_child(void); #ifndef JEMALLOC_ENABLE_INLINE malloc_tsd_protos(JEMALLOC_ATTR(unused), prof_tdata, prof_tdata_t *) -prof_tdata_t *prof_tdata_get(void); +prof_tdata_t *prof_tdata_get(bool create); void prof_sample_threshold_update(prof_tdata_t *prof_tdata); prof_ctx_t *prof_ctx_get(const void *ptr); -void prof_ctx_set(const void *ptr, prof_ctx_t *ctx); +void prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx); bool prof_sample_accum_update(size_t size); -void prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt); -void prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, - size_t old_size, prof_ctx_t *old_ctx); +void prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt); +void prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, + size_t old_usize, prof_ctx_t *old_ctx); void prof_free(const void *ptr, size_t size); #endif @@ -304,17 +323,15 @@ malloc_tsd_funcs(JEMALLOC_INLINE, prof_tdata, prof_tdata_t *, NULL, prof_tdata_cleanup) JEMALLOC_INLINE prof_tdata_t * -prof_tdata_get(void) +prof_tdata_get(bool create) { prof_tdata_t *prof_tdata; cassert(config_prof); prof_tdata = *prof_tdata_tsd_get(); - if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) { - if (prof_tdata == NULL) - prof_tdata = prof_tdata_init(); - } + if (create && prof_tdata == NULL) + prof_tdata = prof_tdata_init(); return (prof_tdata); } @@ -322,6 +339,20 @@ prof_tdata_get(void) JEMALLOC_INLINE void prof_sample_threshold_update(prof_tdata_t *prof_tdata) { + /* + * The body of this function is compiled out unless heap profiling is + * enabled, so that it is possible to compile jemalloc with floating + * point support completely disabled. Avoiding floating point code is + * important on memory-constrained systems, but it also enables a + * workaround for versions of glibc that don't properly save/restore + * floating point registers during dynamic lazy symbol loading (which + * internally calls into whatever malloc implementation happens to be + * integrated into the application). Note that some compilers (e.g. + * gcc 4.8) may use floating point registers for fast memory moves, so + * jemalloc must be compiled with such optimizations disabled (e.g. + * -mno-sse) in order for the workaround to be complete. + */ +#ifdef JEMALLOC_PROF uint64_t r; double u; @@ -343,7 +374,7 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata) * Luc Devroye * Springer-Verlag, New York, 1986 * pp 500 - * (http://cg.scs.carleton.ca/~luc/rnbookindex.html) + * (http://luc.devroye.org/rnbookindex.html) */ prng64(r, 53, prof_tdata->prng_state, UINT64_C(6364136223846793005), UINT64_C(1442695040888963407)); @@ -351,6 +382,7 @@ prof_sample_threshold_update(prof_tdata_t *prof_tdata) prof_tdata->threshold = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << opt_lg_prof_sample)))) + (uint64_t)1U; +#endif } JEMALLOC_INLINE prof_ctx_t * @@ -373,7 +405,7 @@ prof_ctx_get(const void *ptr) } JEMALLOC_INLINE void -prof_ctx_set(const void *ptr, prof_ctx_t *ctx) +prof_ctx_set(const void *ptr, size_t usize, prof_ctx_t *ctx) { arena_chunk_t *chunk; @@ -383,7 +415,7 @@ prof_ctx_set(const void *ptr, prof_ctx_t *ctx) chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); if (chunk != ptr) { /* Region. */ - arena_prof_ctx_set(ptr, ctx); + arena_prof_ctx_set(ptr, usize, ctx); } else huge_prof_ctx_set(ptr, ctx); } @@ -397,7 +429,7 @@ prof_sample_accum_update(size_t size) /* Sampling logic is unnecessary if the interval is 1. */ assert(opt_lg_prof_sample != 0); - prof_tdata = *prof_tdata_tsd_get(); + prof_tdata = prof_tdata_get(false); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (true); @@ -418,20 +450,20 @@ prof_sample_accum_update(size_t size) } JEMALLOC_INLINE void -prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) +prof_malloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt) { cassert(config_prof); assert(ptr != NULL); - assert(size == isalloc(ptr, true)); + assert(usize == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(size)) { + if (prof_sample_accum_update(usize)) { /* * Don't sample. For malloc()-like allocation, it is * always possible to tell in advance how large an * object's usable size will be, so there should never - * be a difference between the size passed to + * be a difference between the usize passed to * PROF_ALLOC_PREP() and prof_malloc(). */ assert((uintptr_t)cnt == (uintptr_t)1U); @@ -439,17 +471,17 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) } if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); + prof_ctx_set(ptr, usize, cnt->ctx); cnt->epoch++; /*********/ mb_write(); /*********/ cnt->cnts.curobjs++; - cnt->cnts.curbytes += size; + cnt->cnts.curbytes += usize; if (opt_prof_accum) { cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += size; + cnt->cnts.accumbytes += usize; } /*********/ mb_write(); @@ -459,12 +491,12 @@ prof_malloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt) mb_write(); /*********/ } else - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); + prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U); } JEMALLOC_INLINE void -prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, - size_t old_size, prof_ctx_t *old_ctx) +prof_realloc(const void *ptr, size_t usize, prof_thr_cnt_t *cnt, + size_t old_usize, prof_ctx_t *old_ctx) { prof_thr_cnt_t *told_cnt; @@ -472,15 +504,15 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, assert(ptr != NULL || (uintptr_t)cnt <= (uintptr_t)1U); if (ptr != NULL) { - assert(size == isalloc(ptr, true)); + assert(usize == isalloc(ptr, true)); if (opt_lg_prof_sample != 0) { - if (prof_sample_accum_update(size)) { + if (prof_sample_accum_update(usize)) { /* - * Don't sample. The size passed to + * Don't sample. The usize passed to * PROF_ALLOC_PREP() was larger than what * actually got allocated, so a backtrace was * captured for this allocation, even though - * its actual size was insufficient to cross + * its actual usize was insufficient to cross * the sample threshold. */ cnt = (prof_thr_cnt_t *)(uintptr_t)1U; @@ -497,7 +529,7 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, */ malloc_mutex_lock(old_ctx->lock); old_ctx->cnt_merged.curobjs--; - old_ctx->cnt_merged.curbytes -= old_size; + old_ctx->cnt_merged.curbytes -= old_usize; malloc_mutex_unlock(old_ctx->lock); told_cnt = (prof_thr_cnt_t *)(uintptr_t)1U; } @@ -507,23 +539,23 @@ prof_realloc(const void *ptr, size_t size, prof_thr_cnt_t *cnt, if ((uintptr_t)told_cnt > (uintptr_t)1U) told_cnt->epoch++; if ((uintptr_t)cnt > (uintptr_t)1U) { - prof_ctx_set(ptr, cnt->ctx); + prof_ctx_set(ptr, usize, cnt->ctx); cnt->epoch++; } else if (ptr != NULL) - prof_ctx_set(ptr, (prof_ctx_t *)(uintptr_t)1U); + prof_ctx_set(ptr, usize, (prof_ctx_t *)(uintptr_t)1U); /*********/ mb_write(); /*********/ if ((uintptr_t)told_cnt > (uintptr_t)1U) { told_cnt->cnts.curobjs--; - told_cnt->cnts.curbytes -= old_size; + told_cnt->cnts.curbytes -= old_usize; } if ((uintptr_t)cnt > (uintptr_t)1U) { cnt->cnts.curobjs++; - cnt->cnts.curbytes += size; + cnt->cnts.curbytes += usize; if (opt_prof_accum) { cnt->cnts.accumobjs++; - cnt->cnts.accumbytes += size; + cnt->cnts.accumbytes += usize; } } /*********/ diff --git a/deps/jemalloc/include/jemalloc/internal/public_namespace.sh b/deps/jemalloc/include/jemalloc/internal/public_namespace.sh new file mode 100755 index 00000000000..362109f7127 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/public_namespace.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +for nm in `cat $1` ; do + n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'` + echo "#define je_${n} JEMALLOC_N(${n})" +done diff --git a/deps/jemalloc/include/jemalloc/internal/public_unnamespace.sh b/deps/jemalloc/include/jemalloc/internal/public_unnamespace.sh new file mode 100755 index 00000000000..4239d17754c --- /dev/null +++ b/deps/jemalloc/include/jemalloc/internal/public_unnamespace.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +for nm in `cat $1` ; do + n=`echo ${nm} |tr ':' ' ' |awk '{print $1}'` + echo "#undef je_${n}" +done diff --git a/deps/jemalloc/include/jemalloc/internal/ql.h b/deps/jemalloc/include/jemalloc/internal/ql.h index a9ed2393f0c..f70c5f6f391 100644 --- a/deps/jemalloc/include/jemalloc/internal/ql.h +++ b/deps/jemalloc/include/jemalloc/internal/ql.h @@ -1,61 +1,61 @@ /* * List definitions. */ -#define ql_head(a_type) \ +#define ql_head(a_type) \ struct { \ a_type *qlh_first; \ } -#define ql_head_initializer(a_head) {NULL} +#define ql_head_initializer(a_head) {NULL} -#define ql_elm(a_type) qr(a_type) +#define ql_elm(a_type) qr(a_type) /* List functions. */ -#define ql_new(a_head) do { \ +#define ql_new(a_head) do { \ (a_head)->qlh_first = NULL; \ } while (0) -#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field) +#define ql_elm_new(a_elm, a_field) qr_new((a_elm), a_field) -#define ql_first(a_head) ((a_head)->qlh_first) +#define ql_first(a_head) ((a_head)->qlh_first) -#define ql_last(a_head, a_field) \ +#define ql_last(a_head, a_field) \ ((ql_first(a_head) != NULL) \ ? qr_prev(ql_first(a_head), a_field) : NULL) -#define ql_next(a_head, a_elm, a_field) \ +#define ql_next(a_head, a_elm, a_field) \ ((ql_last(a_head, a_field) != (a_elm)) \ ? qr_next((a_elm), a_field) : NULL) -#define ql_prev(a_head, a_elm, a_field) \ +#define ql_prev(a_head, a_elm, a_field) \ ((ql_first(a_head) != (a_elm)) ? qr_prev((a_elm), a_field) \ : NULL) -#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ +#define ql_before_insert(a_head, a_qlelm, a_elm, a_field) do { \ qr_before_insert((a_qlelm), (a_elm), a_field); \ if (ql_first(a_head) == (a_qlelm)) { \ ql_first(a_head) = (a_elm); \ } \ } while (0) -#define ql_after_insert(a_qlelm, a_elm, a_field) \ +#define ql_after_insert(a_qlelm, a_elm, a_field) \ qr_after_insert((a_qlelm), (a_elm), a_field) -#define ql_head_insert(a_head, a_elm, a_field) do { \ +#define ql_head_insert(a_head, a_elm, a_field) do { \ if (ql_first(a_head) != NULL) { \ qr_before_insert(ql_first(a_head), (a_elm), a_field); \ } \ ql_first(a_head) = (a_elm); \ } while (0) -#define ql_tail_insert(a_head, a_elm, a_field) do { \ +#define ql_tail_insert(a_head, a_elm, a_field) do { \ if (ql_first(a_head) != NULL) { \ qr_before_insert(ql_first(a_head), (a_elm), a_field); \ } \ ql_first(a_head) = qr_next((a_elm), a_field); \ } while (0) -#define ql_remove(a_head, a_elm, a_field) do { \ +#define ql_remove(a_head, a_elm, a_field) do { \ if (ql_first(a_head) == (a_elm)) { \ ql_first(a_head) = qr_next(ql_first(a_head), a_field); \ } \ @@ -66,18 +66,18 @@ struct { \ } \ } while (0) -#define ql_head_remove(a_head, a_type, a_field) do { \ +#define ql_head_remove(a_head, a_type, a_field) do { \ a_type *t = ql_first(a_head); \ ql_remove((a_head), t, a_field); \ } while (0) -#define ql_tail_remove(a_head, a_type, a_field) do { \ +#define ql_tail_remove(a_head, a_type, a_field) do { \ a_type *t = ql_last(a_head, a_field); \ ql_remove((a_head), t, a_field); \ } while (0) -#define ql_foreach(a_var, a_head, a_field) \ +#define ql_foreach(a_var, a_head, a_field) \ qr_foreach((a_var), ql_first(a_head), a_field) -#define ql_reverse_foreach(a_var, a_head, a_field) \ +#define ql_reverse_foreach(a_var, a_head, a_field) \ qr_reverse_foreach((a_var), ql_first(a_head), a_field) diff --git a/deps/jemalloc/include/jemalloc/internal/qr.h b/deps/jemalloc/include/jemalloc/internal/qr.h index fe22352fedd..602944b9b4f 100644 --- a/deps/jemalloc/include/jemalloc/internal/qr.h +++ b/deps/jemalloc/include/jemalloc/internal/qr.h @@ -1,28 +1,28 @@ /* Ring definitions. */ -#define qr(a_type) \ +#define qr(a_type) \ struct { \ a_type *qre_next; \ a_type *qre_prev; \ } /* Ring functions. */ -#define qr_new(a_qr, a_field) do { \ +#define qr_new(a_qr, a_field) do { \ (a_qr)->a_field.qre_next = (a_qr); \ (a_qr)->a_field.qre_prev = (a_qr); \ } while (0) -#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next) +#define qr_next(a_qr, a_field) ((a_qr)->a_field.qre_next) -#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev) +#define qr_prev(a_qr, a_field) ((a_qr)->a_field.qre_prev) -#define qr_before_insert(a_qrelm, a_qr, a_field) do { \ +#define qr_before_insert(a_qrelm, a_qr, a_field) do { \ (a_qr)->a_field.qre_prev = (a_qrelm)->a_field.qre_prev; \ (a_qr)->a_field.qre_next = (a_qrelm); \ (a_qr)->a_field.qre_prev->a_field.qre_next = (a_qr); \ (a_qrelm)->a_field.qre_prev = (a_qr); \ } while (0) -#define qr_after_insert(a_qrelm, a_qr, a_field) \ +#define qr_after_insert(a_qrelm, a_qr, a_field) \ do \ { \ (a_qr)->a_field.qre_next = (a_qrelm)->a_field.qre_next; \ @@ -31,7 +31,7 @@ struct { \ (a_qrelm)->a_field.qre_next = (a_qr); \ } while (0) -#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ +#define qr_meld(a_qr_a, a_qr_b, a_field) do { \ void *t; \ (a_qr_a)->a_field.qre_prev->a_field.qre_next = (a_qr_b); \ (a_qr_b)->a_field.qre_prev->a_field.qre_next = (a_qr_a); \ @@ -42,10 +42,10 @@ struct { \ /* qr_meld() and qr_split() are functionally equivalent, so there's no need to * have two copies of the code. */ -#define qr_split(a_qr_a, a_qr_b, a_field) \ +#define qr_split(a_qr_a, a_qr_b, a_field) \ qr_meld((a_qr_a), (a_qr_b), a_field) -#define qr_remove(a_qr, a_field) do { \ +#define qr_remove(a_qr, a_field) do { \ (a_qr)->a_field.qre_prev->a_field.qre_next \ = (a_qr)->a_field.qre_next; \ (a_qr)->a_field.qre_next->a_field.qre_prev \ @@ -54,13 +54,13 @@ struct { \ (a_qr)->a_field.qre_prev = (a_qr); \ } while (0) -#define qr_foreach(var, a_qr, a_field) \ +#define qr_foreach(var, a_qr, a_field) \ for ((var) = (a_qr); \ (var) != NULL; \ (var) = (((var)->a_field.qre_next != (a_qr)) \ ? (var)->a_field.qre_next : NULL)) -#define qr_reverse_foreach(var, a_qr, a_field) \ +#define qr_reverse_foreach(var, a_qr, a_field) \ for ((var) = ((a_qr) != NULL) ? qr_prev(a_qr, a_field) : NULL; \ (var) != NULL; \ (var) = (((var) != (a_qr)) \ diff --git a/deps/jemalloc/include/jemalloc/internal/quarantine.h b/deps/jemalloc/include/jemalloc/internal/quarantine.h index 38f3d696e93..16f677f73da 100644 --- a/deps/jemalloc/include/jemalloc/internal/quarantine.h +++ b/deps/jemalloc/include/jemalloc/internal/quarantine.h @@ -1,6 +1,9 @@ /******************************************************************************/ #ifdef JEMALLOC_H_TYPES +typedef struct quarantine_obj_s quarantine_obj_t; +typedef struct quarantine_s quarantine_t; + /* Default per thread quarantine size if valgrind is enabled. */ #define JEMALLOC_VALGRIND_QUARANTINE_DEFAULT (ZU(1) << 24) @@ -8,17 +11,57 @@ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +struct quarantine_obj_s { + void *ptr; + size_t usize; +}; + +struct quarantine_s { + size_t curbytes; + size_t curobjs; + size_t first; +#define LG_MAXOBJS_INIT 10 + size_t lg_maxobjs; + quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */ +}; + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS +quarantine_t *quarantine_init(size_t lg_maxobjs); void quarantine(void *ptr); +void quarantine_cleanup(void *arg); bool quarantine_boot(void); #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ #ifdef JEMALLOC_H_INLINES +#ifndef JEMALLOC_ENABLE_INLINE +malloc_tsd_protos(JEMALLOC_ATTR(unused), quarantine, quarantine_t *) + +void quarantine_alloc_hook(void); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_QUARANTINE_C_)) +malloc_tsd_externs(quarantine, quarantine_t *) +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, quarantine, quarantine_t *, NULL, + quarantine_cleanup) + +JEMALLOC_ALWAYS_INLINE void +quarantine_alloc_hook(void) +{ + quarantine_t *quarantine; + + assert(config_fill && opt_quarantine); + + quarantine = *quarantine_tsd_get(); + if (quarantine == NULL) + quarantine_init(LG_MAXOBJS_INIT); +} +#endif + #endif /* JEMALLOC_H_INLINES */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/rb.h b/deps/jemalloc/include/jemalloc/internal/rb.h index 7b675f09051..423802eb2dc 100644 --- a/deps/jemalloc/include/jemalloc/internal/rb.h +++ b/deps/jemalloc/include/jemalloc/internal/rb.h @@ -22,10 +22,6 @@ #ifndef RB_H_ #define RB_H_ -#if 0 -__FBSDID("$FreeBSD: head/lib/libc/stdlib/rb.h 204493 2010-02-28 22:57:13Z jasone $"); -#endif - #ifdef RB_COMPACT /* Node structure. */ #define rb_node(a_type) \ diff --git a/deps/jemalloc/include/jemalloc/internal/rtree.h b/deps/jemalloc/include/jemalloc/internal/rtree.h index 9bd98548cfe..bc74769f50e 100644 --- a/deps/jemalloc/include/jemalloc/internal/rtree.h +++ b/deps/jemalloc/include/jemalloc/internal/rtree.h @@ -14,17 +14,18 @@ typedef struct rtree_s rtree_t; * Size of each radix tree node (must be a power of 2). This impacts tree * depth. */ -#if (LG_SIZEOF_PTR == 2) -# define RTREE_NODESIZE (1U << 14) -#else -# define RTREE_NODESIZE CACHELINE -#endif +#define RTREE_NODESIZE (1U << 16) + +typedef void *(rtree_alloc_t)(size_t); +typedef void (rtree_dalloc_t)(void *); #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS struct rtree_s { + rtree_alloc_t *alloc; + rtree_dalloc_t *dalloc; malloc_mutex_t mutex; void **root; unsigned height; @@ -35,7 +36,8 @@ struct rtree_s { /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -rtree_t *rtree_new(unsigned bits); +rtree_t *rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc); +void rtree_delete(rtree_t *rtree); void rtree_prefork(rtree_t *rtree); void rtree_postfork_parent(rtree_t *rtree); void rtree_postfork_child(rtree_t *rtree); @@ -45,20 +47,20 @@ void rtree_postfork_child(rtree_t *rtree); #ifdef JEMALLOC_H_INLINES #ifndef JEMALLOC_ENABLE_INLINE -#ifndef JEMALLOC_DEBUG -void *rtree_get_locked(rtree_t *rtree, uintptr_t key); +#ifdef JEMALLOC_DEBUG +uint8_t rtree_get_locked(rtree_t *rtree, uintptr_t key); #endif -void *rtree_get(rtree_t *rtree, uintptr_t key); -bool rtree_set(rtree_t *rtree, uintptr_t key, void *val); +uint8_t rtree_get(rtree_t *rtree, uintptr_t key); +bool rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val); #endif #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_RTREE_C_)) #define RTREE_GET_GENERATE(f) \ /* The least significant bits of the key are ignored. */ \ -JEMALLOC_INLINE void * \ +JEMALLOC_INLINE uint8_t \ f(rtree_t *rtree, uintptr_t key) \ { \ - void *ret; \ + uint8_t ret; \ uintptr_t subkey; \ unsigned i, lshift, height, bits; \ void **node, **child; \ @@ -68,12 +70,12 @@ f(rtree_t *rtree, uintptr_t key) \ i < height - 1; \ i++, lshift += bits, node = child) { \ bits = rtree->level2bits[i]; \ - subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ + subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR + \ 3)) - bits); \ child = (void**)node[subkey]; \ if (child == NULL) { \ RTREE_UNLOCK(&rtree->mutex); \ - return (NULL); \ + return (0); \ } \ } \ \ @@ -84,7 +86,10 @@ f(rtree_t *rtree, uintptr_t key) \ bits = rtree->level2bits[i]; \ subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - \ bits); \ - ret = node[subkey]; \ + { \ + uint8_t *leaf = (uint8_t *)node; \ + ret = leaf[subkey]; \ + } \ RTREE_UNLOCK(&rtree->mutex); \ \ RTREE_GET_VALIDATE \ @@ -123,7 +128,7 @@ RTREE_GET_GENERATE(rtree_get) #undef RTREE_GET_VALIDATE JEMALLOC_INLINE bool -rtree_set(rtree_t *rtree, uintptr_t key, void *val) +rtree_set(rtree_t *rtree, uintptr_t key, uint8_t val) { uintptr_t subkey; unsigned i, lshift, height, bits; @@ -138,14 +143,14 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val) bits); child = (void**)node[subkey]; if (child == NULL) { - child = (void**)base_alloc(sizeof(void *) << - rtree->level2bits[i+1]); + size_t size = ((i + 1 < height - 1) ? sizeof(void *) + : (sizeof(uint8_t))) << rtree->level2bits[i+1]; + child = (void**)rtree->alloc(size); if (child == NULL) { malloc_mutex_unlock(&rtree->mutex); return (true); } - memset(child, 0, sizeof(void *) << - rtree->level2bits[i+1]); + memset(child, 0, size); node[subkey] = child; } } @@ -153,7 +158,10 @@ rtree_set(rtree_t *rtree, uintptr_t key, void *val) /* node is a leaf, so it contains values rather than node pointers. */ bits = rtree->level2bits[i]; subkey = (key << lshift) >> ((ZU(1) << (LG_SIZEOF_PTR+3)) - bits); - node[subkey] = val; + { + uint8_t *leaf = (uint8_t *)node; + leaf[subkey] = val; + } malloc_mutex_unlock(&rtree->mutex); return (false); diff --git a/deps/jemalloc/include/jemalloc/internal/tcache.h b/deps/jemalloc/include/jemalloc/internal/tcache.h index 38d735c86c8..c3d4b58d4dc 100644 --- a/deps/jemalloc/include/jemalloc/internal/tcache.h +++ b/deps/jemalloc/include/jemalloc/internal/tcache.h @@ -140,11 +140,11 @@ void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size); #if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_TCACHE_C_)) /* Map of thread-specific caches. */ malloc_tsd_externs(tcache, tcache_t *) -malloc_tsd_funcs(JEMALLOC_INLINE, tcache, tcache_t *, NULL, +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache, tcache_t *, NULL, tcache_thread_cleanup) /* Per thread flag that allows thread caches to be disabled. */ malloc_tsd_externs(tcache_enabled, tcache_enabled_t) -malloc_tsd_funcs(JEMALLOC_INLINE, tcache_enabled, tcache_enabled_t, +malloc_tsd_funcs(JEMALLOC_ALWAYS_INLINE, tcache_enabled, tcache_enabled_t, tcache_enabled_default, malloc_tsd_no_cleanup) JEMALLOC_INLINE void @@ -206,7 +206,7 @@ tcache_enabled_set(bool enabled) } } -JEMALLOC_INLINE tcache_t * +JEMALLOC_ALWAYS_INLINE tcache_t * tcache_get(bool create) { tcache_t *tcache; @@ -258,7 +258,7 @@ tcache_get(bool create) return (tcache); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void tcache_event(tcache_t *tcache) { @@ -271,7 +271,7 @@ tcache_event(tcache_t *tcache) tcache_event_hard(tcache); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * tcache_alloc_easy(tcache_bin_t *tbin) { void *ret; @@ -287,7 +287,7 @@ tcache_alloc_easy(tcache_bin_t *tbin) return (ret); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) { void *ret; @@ -297,6 +297,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) binind = SMALL_SIZE2BIN(size); assert(binind < NBINS); tbin = &tcache->tbins[binind]; + size = arena_bin_info[binind].reg_size; ret = tcache_alloc_easy(tbin); if (ret == NULL) { ret = tcache_alloc_small_hard(tcache, tbin, binind); @@ -313,6 +314,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) } else if (opt_zero) memset(ret, 0, size); } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { if (config_fill && opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], @@ -330,7 +332,7 @@ tcache_alloc_small(tcache_t *tcache, size_t size, bool zero) return (ret); } -JEMALLOC_INLINE void * +JEMALLOC_ALWAYS_INLINE void * tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) { void *ret; @@ -367,6 +369,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) else if (opt_zero) memset(ret, 0, size); } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { VALGRIND_MAKE_MEM_UNDEFINED(ret, size); memset(ret, 0, size); @@ -382,7 +385,7 @@ tcache_alloc_large(tcache_t *tcache, size_t size, bool zero) return (ret); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) { tcache_bin_t *tbin; @@ -406,7 +409,7 @@ tcache_dalloc_small(tcache_t *tcache, void *ptr, size_t binind) tcache_event(tcache); } -JEMALLOC_INLINE void +JEMALLOC_ALWAYS_INLINE void tcache_dalloc_large(tcache_t *tcache, void *ptr, size_t size) { size_t binind; diff --git a/deps/jemalloc/include/jemalloc/internal/tsd.h b/deps/jemalloc/include/jemalloc/internal/tsd.h index 0037cf35e70..9fb4a23ec6b 100644 --- a/deps/jemalloc/include/jemalloc/internal/tsd.h +++ b/deps/jemalloc/include/jemalloc/internal/tsd.h @@ -6,6 +6,12 @@ typedef bool (*malloc_tsd_cleanup_t)(void); +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +typedef struct tsd_init_block_s tsd_init_block_t; +typedef struct tsd_init_head_s tsd_init_head_t; +#endif + /* * TLS/TSD-agnostic macro-based implementation of thread-specific data. There * are four macros that support (at least) three use cases: file-private, @@ -75,12 +81,13 @@ extern __thread a_type a_name##_tls; \ extern pthread_key_t a_name##_tsd; \ extern bool a_name##_booted; #elif (defined(_WIN32)) -#define malloc_tsd_externs(a_name, a_type) \ +#define malloc_tsd_externs(a_name, a_type) \ extern DWORD a_name##_tsd; \ extern bool a_name##_booted; #else #define malloc_tsd_externs(a_name, a_type) \ extern pthread_key_t a_name##_tsd; \ +extern tsd_init_head_t a_name##_tsd_init_head; \ extern bool a_name##_booted; #endif @@ -105,6 +112,10 @@ a_attr bool a_name##_booted = false; #else #define malloc_tsd_data(a_attr, a_name, a_type, a_initializer) \ a_attr pthread_key_t a_name##_tsd; \ +a_attr tsd_init_head_t a_name##_tsd_init_head = { \ + ql_head_initializer(blocks), \ + MALLOC_MUTEX_INITIALIZER \ +}; \ a_attr bool a_name##_booted = false; #endif @@ -333,8 +344,14 @@ a_name##_tsd_get_wrapper(void) \ pthread_getspecific(a_name##_tsd); \ \ if (wrapper == NULL) { \ + tsd_init_block_t block; \ + wrapper = tsd_init_check_recursion( \ + &a_name##_tsd_init_head, &block); \ + if (wrapper) \ + return (wrapper); \ wrapper = (a_name##_tsd_wrapper_t *) \ malloc_tsd_malloc(sizeof(a_name##_tsd_wrapper_t)); \ + block.data = wrapper; \ if (wrapper == NULL) { \ malloc_write(": Error allocating" \ " TSD for "#a_name"\n"); \ @@ -350,6 +367,7 @@ a_name##_tsd_get_wrapper(void) \ " TSD for "#a_name"\n"); \ abort(); \ } \ + tsd_init_finish(&a_name##_tsd_init_head, &block); \ } \ return (wrapper); \ } \ @@ -379,6 +397,19 @@ a_name##_tsd_set(a_type *val) \ /******************************************************************************/ #ifdef JEMALLOC_H_STRUCTS +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +struct tsd_init_block_s { + ql_elm(tsd_init_block_t) link; + pthread_t thread; + void *data; +}; +struct tsd_init_head_s { + ql_head(tsd_init_block_t) blocks; + malloc_mutex_t lock; +}; +#endif + #endif /* JEMALLOC_H_STRUCTS */ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS @@ -388,6 +419,12 @@ void malloc_tsd_dalloc(void *wrapper); void malloc_tsd_no_cleanup(void *); void malloc_tsd_cleanup_register(bool (*f)(void)); void malloc_tsd_boot(void); +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +void *tsd_init_check_recursion(tsd_init_head_t *head, + tsd_init_block_t *block); +void tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block); +#endif #endif /* JEMALLOC_H_EXTERNS */ /******************************************************************************/ diff --git a/deps/jemalloc/include/jemalloc/internal/util.h b/deps/jemalloc/include/jemalloc/internal/util.h index 8479693631a..6b938f74688 100644 --- a/deps/jemalloc/include/jemalloc/internal/util.h +++ b/deps/jemalloc/include/jemalloc/internal/util.h @@ -14,7 +14,7 @@ * Wrap a cpp argument that contains commas such that it isn't broken up into * multiple arguments. */ -#define JEMALLOC_CONCAT(...) __VA_ARGS__ +#define JEMALLOC_ARG_CONCAT(...) __VA_ARGS__ /* * Silence compiler warnings due to uninitialized values. This is used @@ -42,12 +42,6 @@ } while (0) #endif -/* Use to assert a particular configuration, e.g., cassert(config_debug). */ -#define cassert(c) do { \ - if ((c) == false) \ - assert(false); \ -} while (0) - #ifndef not_reached #define not_reached() do { \ if (config_debug) { \ @@ -69,10 +63,18 @@ } while (0) #endif +#ifndef assert_not_implemented #define assert_not_implemented(e) do { \ if (config_debug && !(e)) \ not_implemented(); \ } while (0) +#endif + +/* Use to assert a particular configuration, e.g., cassert(config_debug). */ +#define cassert(c) do { \ + if ((c) == false) \ + not_reached(); \ +} while (0) #endif /* JEMALLOC_H_TYPES */ /******************************************************************************/ @@ -82,8 +84,9 @@ /******************************************************************************/ #ifdef JEMALLOC_H_EXTERNS -int buferror(char *buf, size_t buflen); -uintmax_t malloc_strtoumax(const char *nptr, char **endptr, int base); +int buferror(int err, char *buf, size_t buflen); +uintmax_t malloc_strtoumax(const char *restrict nptr, + char **restrict endptr, int base); void malloc_write(const char *s); /* @@ -107,7 +110,6 @@ void malloc_printf(const char *format, ...) #ifndef JEMALLOC_ENABLE_INLINE size_t pow2_ceil(size_t x); -void malloc_write(const char *s); void set_errno(int errnum); int get_errno(void); #endif diff --git a/deps/jemalloc/include/jemalloc/jemalloc.h.in b/deps/jemalloc/include/jemalloc/jemalloc.h.in deleted file mode 100644 index 31b1304a20a..00000000000 --- a/deps/jemalloc/include/jemalloc/jemalloc.h.in +++ /dev/null @@ -1,157 +0,0 @@ -#ifndef JEMALLOC_H_ -#define JEMALLOC_H_ -#ifdef __cplusplus -extern "C" { -#endif - -#include -#include - -#define JEMALLOC_VERSION "@jemalloc_version@" -#define JEMALLOC_VERSION_MAJOR @jemalloc_version_major@ -#define JEMALLOC_VERSION_MINOR @jemalloc_version_minor@ -#define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ -#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ -#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" - -#include "jemalloc_defs@install_suffix@.h" - -#ifdef JEMALLOC_EXPERIMENTAL -#define ALLOCM_LG_ALIGN(la) (la) -#if LG_SIZEOF_PTR == 2 -#define ALLOCM_ALIGN(a) (ffs(a)-1) -#else -#define ALLOCM_ALIGN(a) ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) -#endif -#define ALLOCM_ZERO ((int)0x40) -#define ALLOCM_NO_MOVE ((int)0x80) -/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */ -#define ALLOCM_ARENA(a) ((int)(((a)+1) << 8)) - -#define ALLOCM_SUCCESS 0 -#define ALLOCM_ERR_OOM 1 -#define ALLOCM_ERR_NOT_MOVED 2 -#endif - -/* - * The je_ prefix on the following public symbol declarations is an artifact of - * namespace management, and should be omitted in application code unless - * JEMALLOC_NO_DEMANGLE is defined (see below). - */ -extern JEMALLOC_EXPORT const char *je_malloc_conf; -extern JEMALLOC_EXPORT void (*je_malloc_message)(void *cbopaque, - const char *s); - -JEMALLOC_EXPORT void *je_malloc(size_t size) JEMALLOC_ATTR(malloc); -JEMALLOC_EXPORT void *je_calloc(size_t num, size_t size) - JEMALLOC_ATTR(malloc); -JEMALLOC_EXPORT int je_posix_memalign(void **memptr, size_t alignment, - size_t size) JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT void *je_aligned_alloc(size_t alignment, size_t size) - JEMALLOC_ATTR(malloc); -JEMALLOC_EXPORT void *je_realloc(void *ptr, size_t size); -JEMALLOC_EXPORT void je_free(void *ptr); - -#ifdef JEMALLOC_OVERRIDE_MEMALIGN -JEMALLOC_EXPORT void * je_memalign(size_t alignment, size_t size) - JEMALLOC_ATTR(malloc); -#endif - -#ifdef JEMALLOC_OVERRIDE_VALLOC -JEMALLOC_EXPORT void * je_valloc(size_t size) JEMALLOC_ATTR(malloc); -#endif - -JEMALLOC_EXPORT size_t je_malloc_usable_size( - JEMALLOC_USABLE_SIZE_CONST void *ptr); -JEMALLOC_EXPORT void je_malloc_stats_print(void (*write_cb)(void *, - const char *), void *je_cbopaque, const char *opts); -JEMALLOC_EXPORT int je_mallctl(const char *name, void *oldp, - size_t *oldlenp, void *newp, size_t newlen); -JEMALLOC_EXPORT int je_mallctlnametomib(const char *name, size_t *mibp, - size_t *miblenp); -JEMALLOC_EXPORT int je_mallctlbymib(const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen); - -#ifdef JEMALLOC_EXPERIMENTAL -JEMALLOC_EXPORT int je_allocm(void **ptr, size_t *rsize, size_t size, - int flags) JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int je_rallocm(void **ptr, size_t *rsize, size_t size, - size_t extra, int flags) JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int je_sallocm(const void *ptr, size_t *rsize, int flags) - JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int je_dallocm(void *ptr, int flags) - JEMALLOC_ATTR(nonnull(1)); -JEMALLOC_EXPORT int je_nallocm(size_t *rsize, size_t size, int flags); -#endif - -/* - * By default application code must explicitly refer to mangled symbol names, - * so that it is possible to use jemalloc in conjunction with another allocator - * in the same application. Define JEMALLOC_MANGLE in order to cause automatic - * name mangling that matches the API prefixing that happened as a result of - * --with-mangling and/or --with-jemalloc-prefix configuration settings. - */ -#ifdef JEMALLOC_MANGLE -#ifndef JEMALLOC_NO_DEMANGLE -#define JEMALLOC_NO_DEMANGLE -#endif -#define malloc_conf je_malloc_conf -#define malloc_message je_malloc_message -#define malloc je_malloc -#define calloc je_calloc -#define posix_memalign je_posix_memalign -#define aligned_alloc je_aligned_alloc -#define realloc je_realloc -#define free je_free -#define malloc_usable_size je_malloc_usable_size -#define malloc_stats_print je_malloc_stats_print -#define mallctl je_mallctl -#define mallctlnametomib je_mallctlnametomib -#define mallctlbymib je_mallctlbymib -#define memalign je_memalign -#define valloc je_valloc -#ifdef JEMALLOC_EXPERIMENTAL -#define allocm je_allocm -#define rallocm je_rallocm -#define sallocm je_sallocm -#define dallocm je_dallocm -#define nallocm je_nallocm -#endif -#endif - -/* - * The je_* macros can be used as stable alternative names for the public - * jemalloc API if JEMALLOC_NO_DEMANGLE is defined. This is primarily meant - * for use in jemalloc itself, but it can be used by application code to - * provide isolation from the name mangling specified via --with-mangling - * and/or --with-jemalloc-prefix. - */ -#ifndef JEMALLOC_NO_DEMANGLE -#undef je_malloc_conf -#undef je_malloc_message -#undef je_malloc -#undef je_calloc -#undef je_posix_memalign -#undef je_aligned_alloc -#undef je_realloc -#undef je_free -#undef je_malloc_usable_size -#undef je_malloc_stats_print -#undef je_mallctl -#undef je_mallctlnametomib -#undef je_mallctlbymib -#undef je_memalign -#undef je_valloc -#ifdef JEMALLOC_EXPERIMENTAL -#undef je_allocm -#undef je_rallocm -#undef je_sallocm -#undef je_dallocm -#undef je_nallocm -#endif -#endif - -#ifdef __cplusplus -}; -#endif -#endif /* JEMALLOC_H_ */ diff --git a/deps/jemalloc/include/jemalloc/jemalloc.sh b/deps/jemalloc/include/jemalloc/jemalloc.sh new file mode 100755 index 00000000000..e4738ebae99 --- /dev/null +++ b/deps/jemalloc/include/jemalloc/jemalloc.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +objroot=$1 + +cat < +#include + +#define JEMALLOC_VERSION "@jemalloc_version@" +#define JEMALLOC_VERSION_MAJOR @jemalloc_version_major@ +#define JEMALLOC_VERSION_MINOR @jemalloc_version_minor@ +#define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ +#define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ +#define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" + +# define MALLOCX_LG_ALIGN(la) (la) +# if LG_SIZEOF_PTR == 2 +# define MALLOCX_ALIGN(a) (ffs(a)-1) +# else +# define MALLOCX_ALIGN(a) \ + ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) +# endif +# define MALLOCX_ZERO ((int)0x40) +/* Bias arena index bits so that 0 encodes "MALLOCX_ARENA() unspecified". */ +# define MALLOCX_ARENA(a) ((int)(((a)+1) << 8)) + +#ifdef JEMALLOC_EXPERIMENTAL +# define ALLOCM_LG_ALIGN(la) (la) +# if LG_SIZEOF_PTR == 2 +# define ALLOCM_ALIGN(a) (ffs(a)-1) +# else +# define ALLOCM_ALIGN(a) \ + ((a < (size_t)INT_MAX) ? ffs(a)-1 : ffs(a>>32)+31) +# endif +# define ALLOCM_ZERO ((int)0x40) +# define ALLOCM_NO_MOVE ((int)0x80) +/* Bias arena index bits so that 0 encodes "ALLOCM_ARENA() unspecified". */ +# define ALLOCM_ARENA(a) ((int)(((a)+1) << 8)) +# define ALLOCM_SUCCESS 0 +# define ALLOCM_ERR_OOM 1 +# define ALLOCM_ERR_NOT_MOVED 2 +#endif + +#ifdef JEMALLOC_HAVE_ATTR +# define JEMALLOC_ATTR(s) __attribute__((s)) +# define JEMALLOC_EXPORT JEMALLOC_ATTR(visibility("default")) +# define JEMALLOC_ALIGNED(s) JEMALLOC_ATTR(aligned(s)) +# define JEMALLOC_SECTION(s) JEMALLOC_ATTR(section(s)) +# define JEMALLOC_NOINLINE JEMALLOC_ATTR(noinline) +#elif _MSC_VER +# define JEMALLOC_ATTR(s) +# ifdef DLLEXPORT +# define JEMALLOC_EXPORT __declspec(dllexport) +# else +# define JEMALLOC_EXPORT __declspec(dllimport) +# endif +# define JEMALLOC_ALIGNED(s) __declspec(align(s)) +# define JEMALLOC_SECTION(s) __declspec(allocate(s)) +# define JEMALLOC_NOINLINE __declspec(noinline) +#else +# define JEMALLOC_ATTR(s) +# define JEMALLOC_EXPORT +# define JEMALLOC_ALIGNED(s) +# define JEMALLOC_SECTION(s) +# define JEMALLOC_NOINLINE +#endif diff --git a/deps/jemalloc/include/jemalloc/jemalloc_mangle.sh b/deps/jemalloc/include/jemalloc/jemalloc_mangle.sh new file mode 100755 index 00000000000..df328b78dac --- /dev/null +++ b/deps/jemalloc/include/jemalloc/jemalloc_mangle.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +public_symbols_txt=$1 +symbol_prefix=$2 + +cat <nactive + + add_pages) << LG_PAGE) - CHUNK_CEILING((arena->nactive - + sub_pages) << LG_PAGE); + if (cactive_diff != 0) + stats_cactive_add(cactive_diff); + } +} + +static void +arena_run_split_remove(arena_t *arena, arena_chunk_t *chunk, size_t run_ind, + size_t flag_dirty, size_t need_pages) +{ + size_t total_pages, rem_pages; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); - flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); total_pages = arena_mapbits_unallocated_size_get(chunk, run_ind) >> LG_PAGE; assert(arena_mapbits_dirty_get(chunk, run_ind+total_pages-1) == flag_dirty); - need_pages = (size >> LG_PAGE); - assert(need_pages > 0); assert(need_pages <= total_pages); rem_pages = total_pages - need_pages; arena_avail_remove(arena, chunk, run_ind, total_pages, true, true); - if (config_stats) { - /* - * Update stats_cactive if nactive is crossing a chunk - * multiple. - */ - size_t cactive_diff = CHUNK_CEILING((arena->nactive + - need_pages) << LG_PAGE) - CHUNK_CEILING(arena->nactive << - LG_PAGE); - if (cactive_diff != 0) - stats_cactive_add(cactive_diff); - } + arena_cactive_update(arena, need_pages, 0); arena->nactive += need_pages; /* Keep track of trailing unused pages for later use. */ if (rem_pages > 0) { if (flag_dirty != 0) { - arena_mapbits_unallocated_set(chunk, run_ind+need_pages, - (rem_pages << LG_PAGE), CHUNK_MAP_DIRTY); + arena_mapbits_unallocated_set(chunk, + run_ind+need_pages, (rem_pages << LG_PAGE), + flag_dirty); arena_mapbits_unallocated_set(chunk, run_ind+total_pages-1, (rem_pages << LG_PAGE), - CHUNK_MAP_DIRTY); + flag_dirty); } else { arena_mapbits_unallocated_set(chunk, run_ind+need_pages, (rem_pages << LG_PAGE), @@ -426,156 +405,219 @@ arena_run_split(arena_t *arena, arena_run_t *run, size_t size, bool large, arena_avail_insert(arena, chunk, run_ind+need_pages, rem_pages, false, true); } +} - /* - * Update the page map separately for large vs. small runs, since it is - * possible to avoid iteration for large mallocs. - */ - if (large) { - if (zero) { - if (flag_dirty == 0) { - /* - * The run is clean, so some pages may be - * zeroed (i.e. never before touched). - */ - for (i = 0; i < need_pages; i++) { - if (arena_mapbits_unzeroed_get(chunk, - run_ind+i) != 0) { - VALGRIND_MAKE_MEM_UNDEFINED( - (void *)((uintptr_t) - chunk + ((run_ind+i) << - LG_PAGE)), PAGE); - memset((void *)((uintptr_t) - chunk + ((run_ind+i) << - LG_PAGE)), 0, PAGE); - } else if (config_debug) { - VALGRIND_MAKE_MEM_DEFINED( - (void *)((uintptr_t) - chunk + ((run_ind+i) << - LG_PAGE)), PAGE); - arena_chunk_validate_zeroed( - chunk, run_ind+i); - } +static void +arena_run_split_large_helper(arena_t *arena, arena_run_t *run, size_t size, + bool remove, bool zero) +{ + arena_chunk_t *chunk; + size_t flag_dirty, run_ind, need_pages, i; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); + need_pages = (size >> LG_PAGE); + assert(need_pages > 0); + + if (remove) { + arena_run_split_remove(arena, chunk, run_ind, flag_dirty, + need_pages); + } + + if (zero) { + if (flag_dirty == 0) { + /* + * The run is clean, so some pages may be zeroed (i.e. + * never before touched). + */ + for (i = 0; i < need_pages; i++) { + if (arena_mapbits_unzeroed_get(chunk, run_ind+i) + != 0) + arena_run_zero(chunk, run_ind+i, 1); + else if (config_debug) { + arena_run_page_validate_zeroed(chunk, + run_ind+i); + } else { + arena_run_page_mark_zeroed(chunk, + run_ind+i); } - } else { - /* - * The run is dirty, so all pages must be - * zeroed. - */ - VALGRIND_MAKE_MEM_UNDEFINED((void - *)((uintptr_t)chunk + (run_ind << - LG_PAGE)), (need_pages << LG_PAGE)); - memset((void *)((uintptr_t)chunk + (run_ind << - LG_PAGE)), 0, (need_pages << LG_PAGE)); } + } else { + /* The run is dirty, so all pages must be zeroed. */ + arena_run_zero(chunk, run_ind, need_pages); } - - /* - * Set the last element first, in case the run only contains one - * page (i.e. both statements set the same element). - */ - arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, - flag_dirty); - arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); } else { - assert(zero == false); - /* - * Propagate the dirty and unzeroed flags to the allocated - * small run, so that arena_dalloc_bin_run() has the ability to - * conditionally trim clean pages. - */ - arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); - /* - * The first page will always be dirtied during small run - * initialization, so a validation failure here would not - * actually cause an observable failure. - */ - if (config_debug && flag_dirty == 0 && - arena_mapbits_unzeroed_get(chunk, run_ind) == 0) - arena_chunk_validate_zeroed(chunk, run_ind); - for (i = 1; i < need_pages - 1; i++) { - arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); - if (config_debug && flag_dirty == 0 && - arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) - arena_chunk_validate_zeroed(chunk, run_ind+i); - } - arena_mapbits_small_set(chunk, run_ind+need_pages-1, - need_pages-1, binind, flag_dirty); - if (config_debug && flag_dirty == 0 && - arena_mapbits_unzeroed_get(chunk, run_ind+need_pages-1) == - 0) { - arena_chunk_validate_zeroed(chunk, - run_ind+need_pages-1); - } + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } + + /* + * Set the last element first, in case the run only contains one page + * (i.e. both statements set the same element). + */ + arena_mapbits_large_set(chunk, run_ind+need_pages-1, 0, flag_dirty); + arena_mapbits_large_set(chunk, run_ind, size, flag_dirty); +} + +static void +arena_run_split_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) +{ + + arena_run_split_large_helper(arena, run, size, true, zero); +} + +static void +arena_run_init_large(arena_t *arena, arena_run_t *run, size_t size, bool zero) +{ + + arena_run_split_large_helper(arena, run, size, false, zero); +} + +static void +arena_run_split_small(arena_t *arena, arena_run_t *run, size_t size, + size_t binind) +{ + arena_chunk_t *chunk; + size_t flag_dirty, run_ind, need_pages, i; + + assert(binind != BININD_INVALID); + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (unsigned)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + flag_dirty = arena_mapbits_dirty_get(chunk, run_ind); + need_pages = (size >> LG_PAGE); + assert(need_pages > 0); + + arena_run_split_remove(arena, chunk, run_ind, flag_dirty, need_pages); + + /* + * Propagate the dirty and unzeroed flags to the allocated small run, + * so that arena_dalloc_bin_run() has the ability to conditionally trim + * clean pages. + */ + arena_mapbits_small_set(chunk, run_ind, 0, binind, flag_dirty); + /* + * The first page will always be dirtied during small run + * initialization, so a validation failure here would not actually + * cause an observable failure. + */ + if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, + run_ind) == 0) + arena_run_page_validate_zeroed(chunk, run_ind); + for (i = 1; i < need_pages - 1; i++) { + arena_mapbits_small_set(chunk, run_ind+i, i, binind, 0); + if (config_debug && flag_dirty == 0 && + arena_mapbits_unzeroed_get(chunk, run_ind+i) == 0) + arena_run_page_validate_zeroed(chunk, run_ind+i); + } + arena_mapbits_small_set(chunk, run_ind+need_pages-1, need_pages-1, + binind, flag_dirty); + if (config_debug && flag_dirty == 0 && arena_mapbits_unzeroed_get(chunk, + run_ind+need_pages-1) == 0) + arena_run_page_validate_zeroed(chunk, run_ind+need_pages-1); + VALGRIND_MAKE_MEM_UNDEFINED((void *)((uintptr_t)chunk + + (run_ind << LG_PAGE)), (need_pages << LG_PAGE)); } static arena_chunk_t * -arena_chunk_alloc(arena_t *arena) +arena_chunk_init_spare(arena_t *arena) { arena_chunk_t *chunk; - size_t i; - if (arena->spare != NULL) { - chunk = arena->spare; - arena->spare = NULL; + assert(arena->spare != NULL); - assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); - assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); - assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == - arena_maxclass); - assert(arena_mapbits_unallocated_size_get(chunk, - chunk_npages-1) == arena_maxclass); - assert(arena_mapbits_dirty_get(chunk, map_bias) == - arena_mapbits_dirty_get(chunk, chunk_npages-1)); - } else { - bool zero; - size_t unzeroed; + chunk = arena->spare; + arena->spare = NULL; - zero = false; - malloc_mutex_unlock(&arena->lock); - chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, - false, &zero, arena->dss_prec); - malloc_mutex_lock(&arena->lock); - if (chunk == NULL) - return (NULL); - if (config_stats) - arena->stats.mapped += chunksize; + assert(arena_mapbits_allocated_get(chunk, map_bias) == 0); + assert(arena_mapbits_allocated_get(chunk, chunk_npages-1) == 0); + assert(arena_mapbits_unallocated_size_get(chunk, map_bias) == + arena_maxclass); + assert(arena_mapbits_unallocated_size_get(chunk, chunk_npages-1) == + arena_maxclass); + assert(arena_mapbits_dirty_get(chunk, map_bias) == + arena_mapbits_dirty_get(chunk, chunk_npages-1)); - chunk->arena = arena; + return (chunk); +} - /* - * Claim that no pages are in use, since the header is merely - * overhead. - */ - chunk->ndirty = 0; +static arena_chunk_t * +arena_chunk_init_hard(arena_t *arena) +{ + arena_chunk_t *chunk; + bool zero; + size_t unzeroed, i; - chunk->nruns_avail = 0; - chunk->nruns_adjac = 0; + assert(arena->spare == NULL); - /* - * Initialize the map to contain one maximal free untouched run. - * Mark the pages as zeroed iff chunk_alloc() returned a zeroed - * chunk. - */ - unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; - arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, - unzeroed); - /* - * There is no need to initialize the internal page map entries - * unless the chunk is not zeroed. - */ - if (zero == false) { - for (i = map_bias+1; i < chunk_npages-1; i++) - arena_mapbits_unzeroed_set(chunk, i, unzeroed); - } else if (config_debug) { + zero = false; + malloc_mutex_unlock(&arena->lock); + chunk = (arena_chunk_t *)chunk_alloc(chunksize, chunksize, false, + &zero, arena->dss_prec); + malloc_mutex_lock(&arena->lock); + if (chunk == NULL) + return (NULL); + if (config_stats) + arena->stats.mapped += chunksize; + + chunk->arena = arena; + + /* + * Claim that no pages are in use, since the header is merely overhead. + */ + chunk->ndirty = 0; + + chunk->nruns_avail = 0; + chunk->nruns_adjac = 0; + + /* + * Initialize the map to contain one maximal free untouched run. Mark + * the pages as zeroed iff chunk_alloc() returned a zeroed chunk. + */ + unzeroed = zero ? 0 : CHUNK_MAP_UNZEROED; + arena_mapbits_unallocated_set(chunk, map_bias, arena_maxclass, + unzeroed); + /* + * There is no need to initialize the internal page map entries unless + * the chunk is not zeroed. + */ + if (zero == false) { + VALGRIND_MAKE_MEM_UNDEFINED((void *)arena_mapp_get(chunk, + map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, + chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, + map_bias+1))); + for (i = map_bias+1; i < chunk_npages-1; i++) + arena_mapbits_unzeroed_set(chunk, i, unzeroed); + } else { + VALGRIND_MAKE_MEM_DEFINED((void *)arena_mapp_get(chunk, + map_bias+1), (size_t)((uintptr_t) arena_mapp_get(chunk, + chunk_npages-1) - (uintptr_t)arena_mapp_get(chunk, + map_bias+1))); + if (config_debug) { for (i = map_bias+1; i < chunk_npages-1; i++) { assert(arena_mapbits_unzeroed_get(chunk, i) == unzeroed); } } - arena_mapbits_unallocated_set(chunk, chunk_npages-1, - arena_maxclass, unzeroed); + } + arena_mapbits_unallocated_set(chunk, chunk_npages-1, arena_maxclass, + unzeroed); + + return (chunk); +} + +static arena_chunk_t * +arena_chunk_alloc(arena_t *arena) +{ + arena_chunk_t *chunk; + + if (arena->spare != NULL) + chunk = arena_chunk_init_spare(arena); + else { + chunk = arena_chunk_init_hard(arena); + if (chunk == NULL) + return (NULL); } /* Insert the run into the runs_avail tree. */ @@ -618,8 +660,7 @@ arena_chunk_dealloc(arena_t *arena, arena_chunk_t *chunk) } static arena_run_t * -arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, - bool zero) +arena_run_alloc_large_helper(arena_t *arena, size_t size, bool zero) { arena_run_t *run; arena_chunk_map_t *mapelm, key; @@ -634,7 +675,7 @@ arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << LG_PAGE)); - arena_run_split(arena, run, size, large, binind, zero); + arena_run_split_large(arena, run, size, zero); return (run); } @@ -642,19 +683,72 @@ arena_run_alloc_helper(arena_t *arena, size_t size, bool large, size_t binind, } static arena_run_t * -arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, - bool zero) +arena_run_alloc_large(arena_t *arena, size_t size, bool zero) +{ + arena_chunk_t *chunk; + arena_run_t *run; + + assert(size <= arena_maxclass); + assert((size & PAGE_MASK) == 0); + + /* Search the arena's chunks for the lowest best fit. */ + run = arena_run_alloc_large_helper(arena, size, zero); + if (run != NULL) + return (run); + + /* + * No usable runs. Create a new chunk from which to allocate the run. + */ + chunk = arena_chunk_alloc(arena); + if (chunk != NULL) { + run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); + arena_run_split_large(arena, run, size, zero); + return (run); + } + + /* + * arena_chunk_alloc() failed, but another thread may have made + * sufficient memory available while this one dropped arena->lock in + * arena_chunk_alloc(), so search one more time. + */ + return (arena_run_alloc_large_helper(arena, size, zero)); +} + +static arena_run_t * +arena_run_alloc_small_helper(arena_t *arena, size_t size, size_t binind) +{ + arena_run_t *run; + arena_chunk_map_t *mapelm, key; + + key.bits = size | CHUNK_MAP_KEY; + mapelm = arena_avail_tree_nsearch(&arena->runs_avail, &key); + if (mapelm != NULL) { + arena_chunk_t *run_chunk = CHUNK_ADDR2BASE(mapelm); + size_t pageind = (((uintptr_t)mapelm - + (uintptr_t)run_chunk->map) / sizeof(arena_chunk_map_t)) + + map_bias; + + run = (arena_run_t *)((uintptr_t)run_chunk + (pageind << + LG_PAGE)); + arena_run_split_small(arena, run, size, binind); + return (run); + } + + return (NULL); +} + +static arena_run_t * +arena_run_alloc_small(arena_t *arena, size_t size, size_t binind) { arena_chunk_t *chunk; arena_run_t *run; assert(size <= arena_maxclass); assert((size & PAGE_MASK) == 0); - assert((large && binind == BININD_INVALID) || (large == false && binind - != BININD_INVALID)); + assert(binind != BININD_INVALID); /* Search the arena's chunks for the lowest best fit. */ - run = arena_run_alloc_helper(arena, size, large, binind, zero); + run = arena_run_alloc_small_helper(arena, size, binind); if (run != NULL) return (run); @@ -664,7 +758,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, chunk = arena_chunk_alloc(arena); if (chunk != NULL) { run = (arena_run_t *)((uintptr_t)chunk + (map_bias << LG_PAGE)); - arena_run_split(arena, run, size, large, binind, zero); + arena_run_split_small(arena, run, size, binind); return (run); } @@ -673,7 +767,7 @@ arena_run_alloc(arena_t *arena, size_t size, bool large, size_t binind, * sufficient memory available while this one dropped arena->lock in * arena_chunk_alloc(), so search one more time. */ - return (arena_run_alloc_helper(arena, size, large, binind, zero)); + return (arena_run_alloc_small_helper(arena, size, binind)); } static inline void @@ -699,48 +793,42 @@ arena_maybe_purge(arena_t *arena) arena_purge(arena, false); } -static inline size_t -arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) +static arena_chunk_t * +chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) { - size_t npurged; - ql_head(arena_chunk_map_t) mapelms; - arena_chunk_map_t *mapelm; - size_t pageind, npages; - size_t nmadvise; + size_t *ndirty = (size_t *)arg; - ql_new(&mapelms); + assert(chunk->ndirty != 0); + *ndirty += chunk->ndirty; + return (NULL); +} + +static size_t +arena_compute_npurgatory(arena_t *arena, bool all) +{ + size_t npurgatory, npurgeable; /* - * If chunk is the spare, temporarily re-allocate it, 1) so that its - * run is reinserted into runs_avail, and 2) so that it cannot be - * completely discarded by another thread while arena->lock is dropped - * by this thread. Note that the arena_run_dalloc() call will - * implicitly deallocate the chunk, so no explicit action is required - * in this function to deallocate the chunk. - * - * Note that once a chunk contains dirty pages, it cannot again contain - * a single run unless 1) it is a dirty run, or 2) this function purges - * dirty pages and causes the transition to a single clean run. Thus - * (chunk == arena->spare) is possible, but it is not possible for - * this function to be called on the spare unless it contains a dirty - * run. + * Compute the minimum number of pages that this thread should try to + * purge. */ - if (chunk == arena->spare) { - assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); - assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); + npurgeable = arena->ndirty - arena->npurgatory; - arena_chunk_alloc(arena); - } + if (all == false) { + size_t threshold = (arena->nactive >> opt_lg_dirty_mult); - if (config_stats) - arena->stats.purged += chunk->ndirty; + npurgatory = npurgeable - threshold; + } else + npurgatory = npurgeable; - /* - * Operate on all dirty runs if there is no clean/dirty run - * fragmentation. - */ - if (chunk->nruns_adjac == 0) - all = true; + return (npurgatory); +} + +static void +arena_chunk_stash_dirty(arena_t *arena, arena_chunk_t *chunk, bool all, + arena_chunk_mapelms_t *mapelms) +{ + size_t pageind, npages; /* * Temporarily allocate free dirty runs within chunk. If all is false, @@ -748,7 +836,7 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) * all dirty runs. */ for (pageind = map_bias; pageind < chunk_npages; pageind += npages) { - mapelm = arena_mapp_get(chunk, pageind); + arena_chunk_map_t *mapelm = arena_mapp_get(chunk, pageind); if (arena_mapbits_allocated_get(chunk, pageind) == 0) { size_t run_size = arena_mapbits_unallocated_size_get(chunk, pageind); @@ -764,11 +852,11 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) arena_run_t *run = (arena_run_t *)((uintptr_t) chunk + (uintptr_t)(pageind << LG_PAGE)); - arena_run_split(arena, run, run_size, true, - BININD_INVALID, false); + arena_run_split_large(arena, run, run_size, + false); /* Append to list for later processing. */ ql_elm_new(mapelm, u.ql_link); - ql_tail_insert(&mapelms, mapelm, u.ql_link); + ql_tail_insert(mapelms, mapelm, u.ql_link); } } else { /* Skip run. */ @@ -792,12 +880,20 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) assert(pageind == chunk_npages); assert(chunk->ndirty == 0 || all == false); assert(chunk->nruns_adjac == 0); +} + +static size_t +arena_chunk_purge_stashed(arena_t *arena, arena_chunk_t *chunk, + arena_chunk_mapelms_t *mapelms) +{ + size_t npurged, pageind, npages, nmadvise; + arena_chunk_map_t *mapelm; malloc_mutex_unlock(&arena->lock); if (config_stats) nmadvise = 0; npurged = 0; - ql_foreach(mapelm, &mapelms, u.ql_link) { + ql_foreach(mapelm, mapelms, u.ql_link) { bool unzeroed; size_t flag_unzeroed, i; @@ -831,30 +927,75 @@ arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) if (config_stats) arena->stats.nmadvise += nmadvise; + return (npurged); +} + +static void +arena_chunk_unstash_purged(arena_t *arena, arena_chunk_t *chunk, + arena_chunk_mapelms_t *mapelms) +{ + arena_chunk_map_t *mapelm; + size_t pageind; + /* Deallocate runs. */ - for (mapelm = ql_first(&mapelms); mapelm != NULL; - mapelm = ql_first(&mapelms)) { + for (mapelm = ql_first(mapelms); mapelm != NULL; + mapelm = ql_first(mapelms)) { arena_run_t *run; pageind = (((uintptr_t)mapelm - (uintptr_t)chunk->map) / sizeof(arena_chunk_map_t)) + map_bias; run = (arena_run_t *)((uintptr_t)chunk + (uintptr_t)(pageind << LG_PAGE)); - ql_remove(&mapelms, mapelm, u.ql_link); + ql_remove(mapelms, mapelm, u.ql_link); arena_run_dalloc(arena, run, false, true); } - - return (npurged); } -static arena_chunk_t * -chunks_dirty_iter_cb(arena_chunk_tree_t *tree, arena_chunk_t *chunk, void *arg) +static inline size_t +arena_chunk_purge(arena_t *arena, arena_chunk_t *chunk, bool all) { - size_t *ndirty = (size_t *)arg; + size_t npurged; + arena_chunk_mapelms_t mapelms; - assert(chunk->ndirty != 0); - *ndirty += chunk->ndirty; - return (NULL); + ql_new(&mapelms); + + /* + * If chunk is the spare, temporarily re-allocate it, 1) so that its + * run is reinserted into runs_avail, and 2) so that it cannot be + * completely discarded by another thread while arena->lock is dropped + * by this thread. Note that the arena_run_dalloc() call will + * implicitly deallocate the chunk, so no explicit action is required + * in this function to deallocate the chunk. + * + * Note that once a chunk contains dirty pages, it cannot again contain + * a single run unless 1) it is a dirty run, or 2) this function purges + * dirty pages and causes the transition to a single clean run. Thus + * (chunk == arena->spare) is possible, but it is not possible for + * this function to be called on the spare unless it contains a dirty + * run. + */ + if (chunk == arena->spare) { + assert(arena_mapbits_dirty_get(chunk, map_bias) != 0); + assert(arena_mapbits_dirty_get(chunk, chunk_npages-1) != 0); + + arena_chunk_alloc(arena); + } + + if (config_stats) + arena->stats.purged += chunk->ndirty; + + /* + * Operate on all dirty runs if there is no clean/dirty run + * fragmentation. + */ + if (chunk->nruns_adjac == 0) + all = true; + + arena_chunk_stash_dirty(arena, chunk, all, &mapelms); + npurged = arena_chunk_purge_stashed(arena, chunk, &mapelms); + arena_chunk_unstash_purged(arena, chunk, &mapelms); + + return (npurged); } static void @@ -877,21 +1018,11 @@ arena_purge(arena_t *arena, bool all) arena->stats.npurge++; /* - * Compute the minimum number of pages that this thread should try to - * purge, and add the result to arena->npurgatory. This will keep - * multiple threads from racing to reduce ndirty below the threshold. + * Add the minimum number of pages this thread should try to purge to + * arena->npurgatory. This will keep multiple threads from racing to + * reduce ndirty below the threshold. */ - { - size_t npurgeable = arena->ndirty - arena->npurgatory; - - if (all == false) { - size_t threshold = (arena->nactive >> - opt_lg_dirty_mult); - - npurgatory = npurgeable - threshold; - } else - npurgatory = npurgeable; - } + npurgatory = arena_compute_npurgatory(arena, all); arena->npurgatory += npurgatory; while (npurgatory > 0) { @@ -958,61 +1089,12 @@ arena_purge_all(arena_t *arena) } static void -arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) +arena_run_coalesce(arena_t *arena, arena_chunk_t *chunk, size_t *p_size, + size_t *p_run_ind, size_t *p_run_pages, size_t flag_dirty) { - arena_chunk_t *chunk; - size_t size, run_ind, run_pages, flag_dirty; - - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); - run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); - assert(run_ind >= map_bias); - assert(run_ind < chunk_npages); - if (arena_mapbits_large_get(chunk, run_ind) != 0) { - size = arena_mapbits_large_size_get(chunk, run_ind); - assert(size == PAGE || - arena_mapbits_large_size_get(chunk, - run_ind+(size>>LG_PAGE)-1) == 0); - } else { - size_t binind = arena_bin_index(arena, run->bin); - arena_bin_info_t *bin_info = &arena_bin_info[binind]; - size = bin_info->run_size; - } - run_pages = (size >> LG_PAGE); - if (config_stats) { - /* - * Update stats_cactive if nactive is crossing a chunk - * multiple. - */ - size_t cactive_diff = CHUNK_CEILING(arena->nactive << LG_PAGE) - - CHUNK_CEILING((arena->nactive - run_pages) << LG_PAGE); - if (cactive_diff != 0) - stats_cactive_sub(cactive_diff); - } - arena->nactive -= run_pages; - - /* - * The run is dirty if the caller claims to have dirtied it, as well as - * if it was already dirty before being allocated and the caller - * doesn't claim to have cleaned it. - */ - assert(arena_mapbits_dirty_get(chunk, run_ind) == - arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); - if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) - dirty = true; - flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; - - /* Mark pages as unallocated in the chunk map. */ - if (dirty) { - arena_mapbits_unallocated_set(chunk, run_ind, size, - CHUNK_MAP_DIRTY); - arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, - CHUNK_MAP_DIRTY); - } else { - arena_mapbits_unallocated_set(chunk, run_ind, size, - arena_mapbits_unzeroed_get(chunk, run_ind)); - arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, - arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); - } + size_t size = *p_size; + size_t run_ind = *p_run_ind; + size_t run_pages = *p_run_pages; /* Try to coalesce forward. */ if (run_ind + run_pages < chunk_npages && @@ -1042,8 +1124,9 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) } /* Try to coalesce backward. */ - if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, run_ind-1) - == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == flag_dirty) { + if (run_ind > map_bias && arena_mapbits_allocated_get(chunk, + run_ind-1) == 0 && arena_mapbits_dirty_get(chunk, run_ind-1) == + flag_dirty) { size_t prun_size = arena_mapbits_unallocated_size_get(chunk, run_ind-1); size_t prun_pages = prun_size >> LG_PAGE; @@ -1068,6 +1151,62 @@ arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) size); } + *p_size = size; + *p_run_ind = run_ind; + *p_run_pages = run_pages; +} + +static void +arena_run_dalloc(arena_t *arena, arena_run_t *run, bool dirty, bool cleaned) +{ + arena_chunk_t *chunk; + size_t size, run_ind, run_pages, flag_dirty; + + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); + run_ind = (size_t)(((uintptr_t)run - (uintptr_t)chunk) >> LG_PAGE); + assert(run_ind >= map_bias); + assert(run_ind < chunk_npages); + if (arena_mapbits_large_get(chunk, run_ind) != 0) { + size = arena_mapbits_large_size_get(chunk, run_ind); + assert(size == PAGE || + arena_mapbits_large_size_get(chunk, + run_ind+(size>>LG_PAGE)-1) == 0); + } else { + size_t binind = arena_bin_index(arena, run->bin); + arena_bin_info_t *bin_info = &arena_bin_info[binind]; + size = bin_info->run_size; + } + run_pages = (size >> LG_PAGE); + arena_cactive_update(arena, 0, run_pages); + arena->nactive -= run_pages; + + /* + * The run is dirty if the caller claims to have dirtied it, as well as + * if it was already dirty before being allocated and the caller + * doesn't claim to have cleaned it. + */ + assert(arena_mapbits_dirty_get(chunk, run_ind) == + arena_mapbits_dirty_get(chunk, run_ind+run_pages-1)); + if (cleaned == false && arena_mapbits_dirty_get(chunk, run_ind) != 0) + dirty = true; + flag_dirty = dirty ? CHUNK_MAP_DIRTY : 0; + + /* Mark pages as unallocated in the chunk map. */ + if (dirty) { + arena_mapbits_unallocated_set(chunk, run_ind, size, + CHUNK_MAP_DIRTY); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + CHUNK_MAP_DIRTY); + } else { + arena_mapbits_unallocated_set(chunk, run_ind, size, + arena_mapbits_unzeroed_get(chunk, run_ind)); + arena_mapbits_unallocated_set(chunk, run_ind+run_pages-1, size, + arena_mapbits_unzeroed_get(chunk, run_ind+run_pages-1)); + } + + arena_run_coalesce(arena, chunk, &size, &run_ind, &run_pages, + flag_dirty); + /* Insert into runs_avail, now that coalescing is complete. */ assert(arena_mapbits_unallocated_size_get(chunk, run_ind) == arena_mapbits_unallocated_size_get(chunk, run_ind+run_pages-1)); @@ -1235,14 +1374,12 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) malloc_mutex_unlock(&bin->lock); /******************************/ malloc_mutex_lock(&arena->lock); - run = arena_run_alloc(arena, bin_info->run_size, false, binind, false); + run = arena_run_alloc_small(arena, bin_info->run_size, binind); if (run != NULL) { bitmap_t *bitmap = (bitmap_t *)((uintptr_t)run + (uintptr_t)bin_info->bitmap_offset); /* Initialize run internals. */ - VALGRIND_MAKE_MEM_UNDEFINED(run, bin_info->reg0_offset - - bin_info->redzone_size); run->bin = bin; run->nextind = 0; run->nfree = bin_info->nregs; @@ -1260,7 +1397,7 @@ arena_bin_nonfull_run_get(arena_t *arena, arena_bin_t *bin) } /* - * arena_run_alloc() failed, but another thread may have made + * arena_run_alloc_small() failed, but another thread may have made * sufficient memory available while this one dropped bin->lock above, * so search one more time. */ @@ -1295,12 +1432,12 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) arena_chunk_t *chunk; /* - * arena_run_alloc() may have allocated run, or it may - * have pulled run from the bin's run tree. Therefore - * it is unsafe to make any assumptions about how run - * has previously been used, and arena_bin_lower_run() - * must be called, as if a region were just deallocated - * from the run. + * arena_run_alloc_small() may have allocated run, or + * it may have pulled run from the bin's run tree. + * Therefore it is unsafe to make any assumptions about + * how run has previously been used, and + * arena_bin_lower_run() must be called, as if a region + * were just deallocated from the run. */ chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(run); if (run->nfree == bin_info->nregs) @@ -1321,21 +1458,6 @@ arena_bin_malloc_hard(arena_t *arena, arena_bin_t *bin) return (arena_run_reg_alloc(bin->runcur, bin_info)); } -void -arena_prof_accum(arena_t *arena, uint64_t accumbytes) -{ - - cassert(config_prof); - - if (config_prof && prof_interval != 0) { - arena->prof_accumbytes += accumbytes; - if (arena->prof_accumbytes >= prof_interval) { - prof_idump(); - arena->prof_accumbytes -= prof_interval; - } - } -} - void arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, uint64_t prof_accumbytes) @@ -1347,11 +1469,8 @@ arena_tcache_fill_small(arena_t *arena, tcache_bin_t *tbin, size_t binind, assert(tbin->ncached == 0); - if (config_prof) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, prof_accumbytes); - malloc_mutex_unlock(&arena->lock); - } + if (config_prof && arena_prof_accum(arena, prof_accumbytes)) + prof_idump(); bin = &arena->bins[binind]; malloc_mutex_lock(&bin->lock); for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> @@ -1396,8 +1515,28 @@ arena_alloc_junk_small(void *ptr, arena_bin_info_t *bin_info, bool zero) } } -void -arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) +#ifdef JEMALLOC_JET +#undef arena_redzone_corruption +#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption_impl) +#endif +static void +arena_redzone_corruption(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + malloc_printf(": Corrupt redzone %zu byte%s %s %p " + "(size %zu), byte=%#x\n", offset, (offset == 1) ? "" : "s", + after ? "after" : "before", ptr, usize, byte); +} +#ifdef JEMALLOC_JET +#undef arena_redzone_corruption +#define arena_redzone_corruption JEMALLOC_N(arena_redzone_corruption) +arena_redzone_corruption_t *arena_redzone_corruption = + JEMALLOC_N(arena_redzone_corruption_impl); +#endif + +static void +arena_redzones_validate(void *ptr, arena_bin_info_t *bin_info, bool reset) { size_t size = bin_info->reg_size; size_t redzone_size = bin_info->redzone_size; @@ -1405,29 +1544,61 @@ arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) bool error = false; for (i = 1; i <= redzone_size; i++) { - unsigned byte; - if ((byte = *(uint8_t *)((uintptr_t)ptr - i)) != 0xa5) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr - i); + if (*byte != 0xa5) { error = true; - malloc_printf(": Corrupt redzone " - "%zu byte%s before %p (size %zu), byte=%#x\n", i, - (i == 1) ? "" : "s", ptr, size, byte); + arena_redzone_corruption(ptr, size, false, i, *byte); + if (reset) + *byte = 0xa5; } } for (i = 0; i < redzone_size; i++) { - unsigned byte; - if ((byte = *(uint8_t *)((uintptr_t)ptr + size + i)) != 0xa5) { + uint8_t *byte = (uint8_t *)((uintptr_t)ptr + size + i); + if (*byte != 0xa5) { error = true; - malloc_printf(": Corrupt redzone " - "%zu byte%s after end of %p (size %zu), byte=%#x\n", - i, (i == 1) ? "" : "s", ptr, size, byte); + arena_redzone_corruption(ptr, size, true, i, *byte); + if (reset) + *byte = 0xa5; } } if (opt_abort && error) abort(); +} +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_small +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small_impl) +#endif +void +arena_dalloc_junk_small(void *ptr, arena_bin_info_t *bin_info) +{ + size_t redzone_size = bin_info->redzone_size; + + arena_redzones_validate(ptr, bin_info, false); memset((void *)((uintptr_t)ptr - redzone_size), 0x5a, bin_info->reg_interval); } +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_small +#define arena_dalloc_junk_small JEMALLOC_N(arena_dalloc_junk_small) +arena_dalloc_junk_small_t *arena_dalloc_junk_small = + JEMALLOC_N(arena_dalloc_junk_small_impl); +#endif + +void +arena_quarantine_junk_small(void *ptr, size_t usize) +{ + size_t binind; + arena_bin_info_t *bin_info; + cassert(config_fill); + assert(opt_junk); + assert(opt_quarantine); + assert(usize <= SMALL_MAXCLASS); + + binind = SMALL_SIZE2BIN(usize); + bin_info = &arena_bin_info[binind]; + arena_redzones_validate(ptr, bin_info, true); +} void * arena_malloc_small(arena_t *arena, size_t size, bool zero) @@ -1459,11 +1630,8 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) bin->stats.nrequests++; } malloc_mutex_unlock(&bin->lock); - if (config_prof && isthreaded == false) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, size); - malloc_mutex_unlock(&arena->lock); - } + if (config_prof && isthreaded == false && arena_prof_accum(arena, size)) + prof_idump(); if (zero == false) { if (config_fill) { @@ -1473,6 +1641,7 @@ arena_malloc_small(arena_t *arena, size_t size, bool zero) } else if (opt_zero) memset(ret, 0, size); } + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } else { if (config_fill && opt_junk) { arena_alloc_junk_small(ret, &arena_bin_info[binind], @@ -1489,11 +1658,12 @@ void * arena_malloc_large(arena_t *arena, size_t size, bool zero) { void *ret; + UNUSED bool idump; /* Large allocation. */ size = PAGE_CEILING(size); malloc_mutex_lock(&arena->lock); - ret = (void *)arena_run_alloc(arena, size, true, BININD_INVALID, zero); + ret = (void *)arena_run_alloc_large(arena, size, zero); if (ret == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); @@ -1507,8 +1677,10 @@ arena_malloc_large(arena_t *arena, size_t size, bool zero) arena->stats.lstats[(size >> LG_PAGE) - 1].curruns++; } if (config_prof) - arena_prof_accum(arena, size); + idump = arena_prof_accum_locked(arena, size); malloc_mutex_unlock(&arena->lock); + if (config_prof && idump) + prof_idump(); if (zero == false) { if (config_fill) { @@ -1537,7 +1709,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) alloc_size = size + alignment - PAGE; malloc_mutex_lock(&arena->lock); - run = arena_run_alloc(arena, alloc_size, true, BININD_INVALID, zero); + run = arena_run_alloc_large(arena, alloc_size, false); if (run == NULL) { malloc_mutex_unlock(&arena->lock); return (NULL); @@ -1557,6 +1729,7 @@ arena_palloc(arena_t *arena, size_t size, size_t alignment, bool zero) arena_run_trim_tail(arena, chunk, ret, size + trailsize, size, false); } + arena_run_init_large(arena, (arena_run_t *)ret, size, zero); if (config_stats) { arena->stats.nmalloc_large++; @@ -1760,21 +1933,38 @@ arena_dalloc_small(arena_t *arena, arena_chunk_t *chunk, void *ptr, arena_dalloc_bin(arena, chunk, ptr, pageind, mapelm); } +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_large +#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large_impl) +#endif +static void +arena_dalloc_junk_large(void *ptr, size_t usize) +{ + + if (config_fill && opt_junk) + memset(ptr, 0x5a, usize); +} +#ifdef JEMALLOC_JET +#undef arena_dalloc_junk_large +#define arena_dalloc_junk_large JEMALLOC_N(arena_dalloc_junk_large) +arena_dalloc_junk_large_t *arena_dalloc_junk_large = + JEMALLOC_N(arena_dalloc_junk_large_impl); +#endif + void arena_dalloc_large_locked(arena_t *arena, arena_chunk_t *chunk, void *ptr) { if (config_fill || config_stats) { size_t pageind = ((uintptr_t)ptr - (uintptr_t)chunk) >> LG_PAGE; - size_t size = arena_mapbits_large_size_get(chunk, pageind); + size_t usize = arena_mapbits_large_size_get(chunk, pageind); - if (config_fill && config_stats && opt_junk) - memset(ptr, 0x5a, size); + arena_dalloc_junk_large(ptr, usize); if (config_stats) { arena->stats.ndalloc_large++; - arena->stats.allocated_large -= size; - arena->stats.lstats[(size >> LG_PAGE) - 1].ndalloc++; - arena->stats.lstats[(size >> LG_PAGE) - 1].curruns--; + arena->stats.allocated_large -= usize; + arena->stats.lstats[(usize >> LG_PAGE) - 1].ndalloc++; + arena->stats.lstats[(usize >> LG_PAGE) - 1].curruns--; } } @@ -1845,9 +2035,8 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, size_t flag_dirty; size_t splitsize = (oldsize + followsize <= size + extra) ? followsize : size + extra - oldsize; - arena_run_split(arena, (arena_run_t *)((uintptr_t)chunk + - ((pageind+npages) << LG_PAGE)), splitsize, true, - BININD_INVALID, zero); + arena_run_split_large(arena, (arena_run_t *)((uintptr_t)chunk + + ((pageind+npages) << LG_PAGE)), splitsize, zero); size = oldsize + splitsize; npages = size >> LG_PAGE; @@ -1886,6 +2075,26 @@ arena_ralloc_large_grow(arena_t *arena, arena_chunk_t *chunk, void *ptr, return (true); } +#ifdef JEMALLOC_JET +#undef arena_ralloc_junk_large +#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large_impl) +#endif +static void +arena_ralloc_junk_large(void *ptr, size_t old_usize, size_t usize) +{ + + if (config_fill && opt_junk) { + memset((void *)((uintptr_t)ptr + usize), 0x5a, + old_usize - usize); + } +} +#ifdef JEMALLOC_JET +#undef arena_ralloc_junk_large +#define arena_ralloc_junk_large JEMALLOC_N(arena_ralloc_junk_large) +arena_ralloc_junk_large_t *arena_ralloc_junk_large = + JEMALLOC_N(arena_ralloc_junk_large_impl); +#endif + /* * Try to resize a large allocation, in order to avoid copying. This will * always fail if growing an object, and the following run is already in use. @@ -1899,10 +2108,6 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, psize = PAGE_CEILING(size + extra); if (psize == oldsize) { /* Same size class. */ - if (config_fill && opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), 0x5a, oldsize - - size); - } return (false); } else { arena_chunk_t *chunk; @@ -1913,10 +2118,7 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, if (psize < oldsize) { /* Fill before shrinking in order avoid a race. */ - if (config_fill && opt_junk) { - memset((void *)((uintptr_t)ptr + size), 0x5a, - oldsize - size); - } + arena_ralloc_junk_large(ptr, oldsize, psize); arena_ralloc_large_shrink(arena, chunk, ptr, oldsize, psize); return (false); @@ -1924,17 +2126,23 @@ arena_ralloc_large(void *ptr, size_t oldsize, size_t size, size_t extra, bool ret = arena_ralloc_large_grow(arena, chunk, ptr, oldsize, PAGE_CEILING(size), psize - PAGE_CEILING(size), zero); - if (config_fill && ret == false && zero == false && - opt_zero) { - memset((void *)((uintptr_t)ptr + oldsize), 0, - size - oldsize); + if (config_fill && ret == false && zero == false) { + if (opt_junk) { + memset((void *)((uintptr_t)ptr + + oldsize), 0xa5, isalloc(ptr, + config_prof) - oldsize); + } else if (opt_zero) { + memset((void *)((uintptr_t)ptr + + oldsize), 0, isalloc(ptr, + config_prof) - oldsize); + } } return (ret); } } } -void * +bool arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, bool zero) { @@ -1949,25 +2157,20 @@ arena_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra, if ((size + extra <= SMALL_MAXCLASS && SMALL_SIZE2BIN(size + extra) == SMALL_SIZE2BIN(oldsize)) || (size <= oldsize && - size + extra >= oldsize)) { - if (config_fill && opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), - 0x5a, oldsize - size); - } - return (ptr); - } + size + extra >= oldsize)) + return (false); } else { assert(size <= arena_maxclass); if (size + extra > SMALL_MAXCLASS) { if (arena_ralloc_large(ptr, oldsize, size, extra, zero) == false) - return (ptr); + return (false); } } } /* Reallocation would require a move. */ - return (NULL); + return (true); } void * @@ -1979,9 +2182,8 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t copysize; /* Try to avoid moving the allocation. */ - ret = arena_ralloc_no_move(ptr, oldsize, size, extra, zero); - if (ret != NULL) - return (ret); + if (arena_ralloc_no_move(ptr, oldsize, size, extra, zero) == false) + return (ptr); /* * size and oldsize are different enough that we need to move the @@ -1992,7 +2194,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t usize = sa2u(size + extra, alignment); if (usize == 0) return (NULL); - ret = ipallocx(usize, alignment, zero, try_tcache_alloc, arena); + ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); } else ret = arena_malloc(arena, size + extra, zero, try_tcache_alloc); @@ -2004,7 +2206,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, size_t usize = sa2u(size, alignment); if (usize == 0) return (NULL); - ret = ipallocx(usize, alignment, zero, try_tcache_alloc, + ret = ipalloct(usize, alignment, zero, try_tcache_alloc, arena); } else ret = arena_malloc(arena, size, zero, try_tcache_alloc); @@ -2022,7 +2224,7 @@ arena_ralloc(arena_t *arena, void *ptr, size_t oldsize, size_t size, copysize = (size < oldsize) ? size : oldsize; VALGRIND_MAKE_MEM_UNDEFINED(ret, copysize); memcpy(ret, ptr, copysize); - iqallocx(ptr, try_tcache_dalloc); + iqalloct(ptr, try_tcache_dalloc); return (ret); } @@ -2277,7 +2479,6 @@ bin_info_run_size_calc(arena_bin_info_t *bin_info, size_t min_run_size) bin_info->reg_interval) - pad_size; } while (try_hdr_size > try_redzone0_offset); } while (try_run_size <= arena_maxclass - && try_run_size <= arena_maxclass && RUN_MAX_OVRHD * (bin_info->reg_interval << 3) > RUN_MAX_OVRHD_RELAX && (try_redzone0_offset << RUN_BFP) > RUN_MAX_OVRHD * try_run_size diff --git a/deps/jemalloc/src/base.c b/deps/jemalloc/src/base.c index b1a5945efaa..4e62e8fa918 100644 --- a/deps/jemalloc/src/base.c +++ b/deps/jemalloc/src/base.c @@ -63,6 +63,7 @@ base_alloc(size_t size) ret = base_next_addr; base_next_addr = (void *)((uintptr_t)base_next_addr + csize); malloc_mutex_unlock(&base_mtx); + VALGRIND_MAKE_MEM_UNDEFINED(ret, csize); return (ret); } @@ -88,6 +89,7 @@ base_node_alloc(void) ret = base_nodes; base_nodes = *(extent_node_t **)ret; malloc_mutex_unlock(&base_mtx); + VALGRIND_MAKE_MEM_UNDEFINED(ret, sizeof(extent_node_t)); } else { malloc_mutex_unlock(&base_mtx); ret = (extent_node_t *)base_alloc(sizeof(extent_node_t)); @@ -100,6 +102,7 @@ void base_node_dealloc(extent_node_t *node) { + VALGRIND_MAKE_MEM_UNDEFINED(node, sizeof(extent_node_t)); malloc_mutex_lock(&base_mtx); *(extent_node_t **)node = base_nodes; base_nodes = node; diff --git a/deps/jemalloc/src/bitmap.c b/deps/jemalloc/src/bitmap.c index b47e2629093..e2bd907d558 100644 --- a/deps/jemalloc/src/bitmap.c +++ b/deps/jemalloc/src/bitmap.c @@ -1,4 +1,4 @@ -#define JEMALLOC_BITMAP_C_ +#define JEMALLOC_BITMAP_C_ #include "jemalloc/internal/jemalloc_internal.h" /******************************************************************************/ diff --git a/deps/jemalloc/src/chunk.c b/deps/jemalloc/src/chunk.c index 1a3bb4f673f..90ab116ae5f 100644 --- a/deps/jemalloc/src/chunk.c +++ b/deps/jemalloc/src/chunk.c @@ -78,6 +78,9 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, assert(node->size >= leadsize + size); trailsize = node->size - leadsize - size; ret = (void *)((uintptr_t)node->addr + leadsize); + zeroed = node->zeroed; + if (zeroed) + *zero = true; /* Remove node from the tree. */ extent_tree_szad_remove(chunks_szad, node); extent_tree_ad_remove(chunks_ad, node); @@ -108,23 +111,26 @@ chunk_recycle(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, size_t size, } node->addr = (void *)((uintptr_t)(ret) + size); node->size = trailsize; + node->zeroed = zeroed; extent_tree_szad_insert(chunks_szad, node); extent_tree_ad_insert(chunks_ad, node); node = NULL; } malloc_mutex_unlock(&chunks_mtx); - zeroed = false; - if (node != NULL) { - if (node->zeroed) { - zeroed = true; - *zero = true; - } + if (node != NULL) base_node_dealloc(node); - } - if (zeroed == false && *zero) { - VALGRIND_MAKE_MEM_UNDEFINED(ret, size); - memset(ret, 0, size); + if (*zero) { + if (zeroed == false) + memset(ret, 0, size); + else if (config_debug) { + size_t i; + size_t *p = (size_t *)(uintptr_t)ret; + + VALGRIND_MAKE_MEM_DEFINED(ret, size); + for (i = 0; i < size / sizeof(size_t); i++) + assert(p[i] == 0); + } } return (ret); } @@ -172,35 +178,32 @@ chunk_alloc(size_t size, size_t alignment, bool base, bool *zero, /* All strategies for allocation failed. */ ret = NULL; label_return: - if (config_ivsalloc && base == false && ret != NULL) { - if (rtree_set(chunks_rtree, (uintptr_t)ret, ret)) { - chunk_dealloc(ret, size, true); - return (NULL); + if (ret != NULL) { + if (config_ivsalloc && base == false) { + if (rtree_set(chunks_rtree, (uintptr_t)ret, 1)) { + chunk_dealloc(ret, size, true); + return (NULL); + } } - } - if ((config_stats || config_prof) && ret != NULL) { - bool gdump; - malloc_mutex_lock(&chunks_mtx); - if (config_stats) - stats_chunks.nchunks += (size / chunksize); - stats_chunks.curchunks += (size / chunksize); - if (stats_chunks.curchunks > stats_chunks.highchunks) { - stats_chunks.highchunks = stats_chunks.curchunks; - if (config_prof) - gdump = true; - } else if (config_prof) - gdump = false; - malloc_mutex_unlock(&chunks_mtx); - if (config_prof && opt_prof && opt_prof_gdump && gdump) - prof_gdump(); - } - if (config_debug && *zero && ret != NULL) { - size_t i; - size_t *p = (size_t *)(uintptr_t)ret; - - VALGRIND_MAKE_MEM_DEFINED(ret, size); - for (i = 0; i < size / sizeof(size_t); i++) - assert(p[i] == 0); + if (config_stats || config_prof) { + bool gdump; + malloc_mutex_lock(&chunks_mtx); + if (config_stats) + stats_chunks.nchunks += (size / chunksize); + stats_chunks.curchunks += (size / chunksize); + if (stats_chunks.curchunks > stats_chunks.highchunks) { + stats_chunks.highchunks = + stats_chunks.curchunks; + if (config_prof) + gdump = true; + } else if (config_prof) + gdump = false; + malloc_mutex_unlock(&chunks_mtx); + if (config_prof && opt_prof && opt_prof_gdump && gdump) + prof_gdump(); + } + if (config_valgrind) + VALGRIND_MAKE_MEM_UNDEFINED(ret, size); } assert(CHUNK_ADDR2BASE(ret) == ret); return (ret); @@ -211,9 +214,10 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, size_t size) { bool unzeroed; - extent_node_t *xnode, *node, *prev, key; + extent_node_t *xnode, *node, *prev, *xprev, key; unzeroed = pages_purge(chunk, size); + VALGRIND_MAKE_MEM_NOACCESS(chunk, size); /* * Allocate a node before acquiring chunks_mtx even though it might not @@ -222,6 +226,8 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, * held. */ xnode = base_node_alloc(); + /* Use xprev to implement conditional deferred deallocation of prev. */ + xprev = NULL; malloc_mutex_lock(&chunks_mtx); key.addr = (void *)((uintptr_t)chunk + size); @@ -238,8 +244,6 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, node->size += size; node->zeroed = (node->zeroed && (unzeroed == false)); extent_tree_szad_insert(chunks_szad, node); - if (xnode != NULL) - base_node_dealloc(xnode); } else { /* Coalescing forward failed, so insert a new node. */ if (xnode == NULL) { @@ -249,10 +253,10 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, * already been purged, so this is only a virtual * memory leak. */ - malloc_mutex_unlock(&chunks_mtx); - return; + goto label_return; } node = xnode; + xnode = NULL; /* Prevent deallocation below. */ node->addr = chunk; node->size = size; node->zeroed = (unzeroed == false); @@ -278,9 +282,19 @@ chunk_record(extent_tree_t *chunks_szad, extent_tree_t *chunks_ad, void *chunk, node->zeroed = (node->zeroed && prev->zeroed); extent_tree_szad_insert(chunks_szad, node); - base_node_dealloc(prev); + xprev = prev; } + +label_return: malloc_mutex_unlock(&chunks_mtx); + /* + * Deallocate xnode and/or xprev after unlocking chunks_mtx in order to + * avoid potential deadlock. + */ + if (xnode != NULL) + base_node_dealloc(xnode); + if (xprev != NULL) + base_node_dealloc(xprev); } void @@ -307,7 +321,7 @@ chunk_dealloc(void *chunk, size_t size, bool unmap) assert((size & chunksize_mask) == 0); if (config_ivsalloc) - rtree_set(chunks_rtree, (uintptr_t)chunk, NULL); + rtree_set(chunks_rtree, (uintptr_t)chunk, 0); if (config_stats || config_prof) { malloc_mutex_lock(&chunks_mtx); assert(stats_chunks.curchunks >= (size / chunksize)); @@ -342,7 +356,7 @@ chunk_boot(void) extent_tree_ad_new(&chunks_ad_dss); if (config_ivsalloc) { chunks_rtree = rtree_new((ZU(1) << (LG_SIZEOF_PTR+3)) - - opt_lg_chunk); + opt_lg_chunk, base_alloc, NULL); if (chunks_rtree == NULL) return (true); } @@ -354,7 +368,7 @@ void chunk_prefork(void) { - malloc_mutex_lock(&chunks_mtx); + malloc_mutex_prefork(&chunks_mtx); if (config_ivsalloc) rtree_prefork(chunks_rtree); chunk_dss_prefork(); diff --git a/deps/jemalloc/src/chunk_dss.c b/deps/jemalloc/src/chunk_dss.c index 24781cc52dc..510bb8bee85 100644 --- a/deps/jemalloc/src/chunk_dss.c +++ b/deps/jemalloc/src/chunk_dss.c @@ -28,16 +28,17 @@ static void *dss_max; /******************************************************************************/ -#ifndef JEMALLOC_HAVE_SBRK static void * -sbrk(intptr_t increment) +chunk_dss_sbrk(intptr_t increment) { +#ifdef JEMALLOC_HAVE_SBRK + return (sbrk(increment)); +#else not_implemented(); - return (NULL); -} #endif +} dss_prec_t chunk_dss_prec_get(void) @@ -93,7 +94,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero) */ do { /* Get the current end of the DSS. */ - dss_max = sbrk(0); + dss_max = chunk_dss_sbrk(0); /* * Calculate how much padding is necessary to * chunk-align the end of the DSS. @@ -117,7 +118,7 @@ chunk_alloc_dss(size_t size, size_t alignment, bool *zero) return (NULL); } incr = gap_size + cpad_size + size; - dss_prev = sbrk(incr); + dss_prev = chunk_dss_sbrk(incr); if (dss_prev == dss_max) { /* Success. */ dss_max = dss_next; @@ -163,7 +164,7 @@ chunk_dss_boot(void) if (malloc_mutex_init(&dss_mtx)) return (true); - dss_base = sbrk(0); + dss_base = chunk_dss_sbrk(0); dss_prev = dss_base; dss_max = dss_base; diff --git a/deps/jemalloc/src/chunk_mmap.c b/deps/jemalloc/src/chunk_mmap.c index 8a42e75915f..2056d793f05 100644 --- a/deps/jemalloc/src/chunk_mmap.c +++ b/deps/jemalloc/src/chunk_mmap.c @@ -43,7 +43,7 @@ pages_map(void *addr, size_t size) if (munmap(ret, size) == -1) { char buf[BUFERROR_BUF]; - buferror(buf, sizeof(buf)); + buferror(get_errno(), buf, sizeof(buf)); malloc_printf(": Error in " #ifdef _WIN32 "VirtualFree" diff --git a/deps/jemalloc/src/ckh.c b/deps/jemalloc/src/ckh.c index 742a950bea2..04c52966193 100644 --- a/deps/jemalloc/src/ckh.c +++ b/deps/jemalloc/src/ckh.c @@ -49,7 +49,7 @@ static void ckh_shrink(ckh_t *ckh); * Search bucket for key and return the cell number if found; SIZE_T_MAX * otherwise. */ -JEMALLOC_INLINE size_t +JEMALLOC_INLINE_C size_t ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) { ckhc_t *cell; @@ -67,28 +67,28 @@ ckh_bucket_search(ckh_t *ckh, size_t bucket, const void *key) /* * Search table for key and return cell number if found; SIZE_T_MAX otherwise. */ -JEMALLOC_INLINE size_t +JEMALLOC_INLINE_C size_t ckh_isearch(ckh_t *ckh, const void *key) { - size_t hash1, hash2, bucket, cell; + size_t hashes[2], bucket, cell; assert(ckh != NULL); - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + ckh->hash(key, hashes); /* Search primary bucket. */ - bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); cell = ckh_bucket_search(ckh, bucket, key); if (cell != SIZE_T_MAX) return (cell); /* Search secondary bucket. */ - bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); cell = ckh_bucket_search(ckh, bucket, key); return (cell); } -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, const void *data) { @@ -120,13 +120,13 @@ ckh_try_bucket_insert(ckh_t *ckh, size_t bucket, const void *key, * eviction/relocation procedure until either success or detection of an * eviction/relocation bucket cycle. */ -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, void const **argdata) { const void *key, *data, *tkey, *tdata; ckhc_t *cell; - size_t hash1, hash2, bucket, tbucket; + size_t hashes[2], bucket, tbucket; unsigned i; bucket = argbucket; @@ -155,10 +155,11 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, #endif /* Find the alternate bucket for the evicted item. */ - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); - tbucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + ckh->hash(key, hashes); + tbucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (tbucket == bucket) { - tbucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + tbucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) + - 1); /* * It may be that (tbucket == bucket) still, if the * item's hashes both indicate this bucket. However, @@ -189,22 +190,22 @@ ckh_evict_reloc_insert(ckh_t *ckh, size_t argbucket, void const **argkey, } } -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) { - size_t hash1, hash2, bucket; + size_t hashes[2], bucket; const void *key = *argkey; const void *data = *argdata; - ckh->hash(key, ckh->lg_curbuckets, &hash1, &hash2); + ckh->hash(key, hashes); /* Try to insert in primary bucket. */ - bucket = hash1 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[0] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) return (false); /* Try to insert in secondary bucket. */ - bucket = hash2 & ((ZU(1) << ckh->lg_curbuckets) - 1); + bucket = hashes[1] & ((ZU(1) << ckh->lg_curbuckets) - 1); if (ckh_try_bucket_insert(ckh, bucket, key, data) == false) return (false); @@ -218,7 +219,7 @@ ckh_try_insert(ckh_t *ckh, void const**argkey, void const**argdata) * Try to rebuild the hash table from scratch by inserting all items from the * old table into the new. */ -JEMALLOC_INLINE bool +JEMALLOC_INLINE_C bool ckh_rebuild(ckh_t *ckh, ckhc_t *aTab) { size_t count, i, nins; @@ -417,9 +418,8 @@ ckh_delete(ckh_t *ckh) #endif idalloc(ckh->tab); -#ifdef JEMALLOC_DEBUG - memset(ckh, 0x5a, sizeof(ckh_t)); -#endif + if (config_debug) + memset(ckh, 0x5a, sizeof(ckh_t)); } size_t @@ -526,31 +526,10 @@ ckh_search(ckh_t *ckh, const void *searchkey, void **key, void **data) } void -ckh_string_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) +ckh_string_hash(const void *key, size_t r_hash[2]) { - size_t ret1, ret2; - uint64_t h; - - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - - h = hash(key, strlen((const char *)key), UINT64_C(0x94122f335b332aea)); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - ret1 = h; - ret2 = hash(key, strlen((const char *)key), - UINT64_C(0x8432a476666bbc13)); - } - *hash1 = ret1; - *hash2 = ret2; + hash(key, strlen((const char *)key), 0x94122f33U, r_hash); } bool @@ -564,41 +543,16 @@ ckh_string_keycomp(const void *k1, const void *k2) } void -ckh_pointer_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2) +ckh_pointer_hash(const void *key, size_t r_hash[2]) { - size_t ret1, ret2; - uint64_t h; union { const void *v; - uint64_t i; + size_t i; } u; - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - assert(sizeof(u.v) == sizeof(u.i)); -#if (LG_SIZEOF_PTR != LG_SIZEOF_INT) - u.i = 0; -#endif u.v = key; - h = hash(&u.i, sizeof(u.i), UINT64_C(0xd983396e68886082)); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - assert(SIZEOF_PTR == 8); - ret1 = h; - ret2 = hash(&u.i, sizeof(u.i), UINT64_C(0x5e2be9aff8709a5d)); - } - - *hash1 = ret1; - *hash2 = ret2; + hash(&u.i, sizeof(u.i), 0xd983396eU, r_hash); } bool diff --git a/deps/jemalloc/src/ctl.c b/deps/jemalloc/src/ctl.c index 6e01b1e27e1..cc2c5aef570 100644 --- a/deps/jemalloc/src/ctl.c +++ b/deps/jemalloc/src/ctl.c @@ -546,43 +546,30 @@ ctl_arena_refresh(arena_t *arena, unsigned i) static bool ctl_grow(void) { - size_t astats_size; ctl_arena_stats_t *astats; arena_t **tarenas; - /* Extend arena stats and arenas arrays. */ - astats_size = (ctl_stats.narenas + 2) * sizeof(ctl_arena_stats_t); - if (ctl_stats.narenas == narenas_auto) { - /* ctl_stats.arenas and arenas came from base_alloc(). */ - astats = (ctl_arena_stats_t *)imalloc(astats_size); - if (astats == NULL) - return (true); - memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * - sizeof(ctl_arena_stats_t)); - - tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * - sizeof(arena_t *)); - if (tarenas == NULL) { - idalloc(astats); - return (true); - } - memcpy(tarenas, arenas, ctl_stats.narenas * sizeof(arena_t *)); - } else { - astats = (ctl_arena_stats_t *)iralloc(ctl_stats.arenas, - astats_size, 0, 0, false, false); - if (astats == NULL) - return (true); - - tarenas = (arena_t **)iralloc(arenas, (ctl_stats.narenas + 1) * - sizeof(arena_t *), 0, 0, false, false); - if (tarenas == NULL) - return (true); + /* Allocate extended arena stats and arenas arrays. */ + astats = (ctl_arena_stats_t *)imalloc((ctl_stats.narenas + 2) * + sizeof(ctl_arena_stats_t)); + if (astats == NULL) + return (true); + tarenas = (arena_t **)imalloc((ctl_stats.narenas + 1) * + sizeof(arena_t *)); + if (tarenas == NULL) { + idalloc(astats); + return (true); } - /* Initialize the new astats and arenas elements. */ + + /* Initialize the new astats element. */ + memcpy(astats, ctl_stats.arenas, (ctl_stats.narenas + 1) * + sizeof(ctl_arena_stats_t)); memset(&astats[ctl_stats.narenas + 1], 0, sizeof(ctl_arena_stats_t)); - if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) + if (ctl_arena_init(&astats[ctl_stats.narenas + 1])) { + idalloc(tarenas); + idalloc(astats); return (true); - tarenas[ctl_stats.narenas] = NULL; + } /* Swap merged stats to their new location. */ { ctl_arena_stats_t tstats; @@ -593,13 +580,34 @@ ctl_grow(void) memcpy(&astats[ctl_stats.narenas + 1], &tstats, sizeof(ctl_arena_stats_t)); } + /* Initialize the new arenas element. */ + tarenas[ctl_stats.narenas] = NULL; + { + arena_t **arenas_old = arenas; + /* + * Swap extended arenas array into place. Although ctl_mtx + * protects this function from other threads extending the + * array, it does not protect from other threads mutating it + * (i.e. initializing arenas and setting array elements to + * point to them). Therefore, array copying must happen under + * the protection of arenas_lock. + */ + malloc_mutex_lock(&arenas_lock); + arenas = tarenas; + memcpy(arenas, arenas_old, ctl_stats.narenas * + sizeof(arena_t *)); + narenas_total++; + arenas_extend(narenas_total - 1); + malloc_mutex_unlock(&arenas_lock); + /* + * Deallocate arenas_old only if it came from imalloc() (not + * base_alloc()). + */ + if (ctl_stats.narenas != narenas_auto) + idalloc(arenas_old); + } ctl_stats.arenas = astats; ctl_stats.narenas++; - malloc_mutex_lock(&arenas_lock); - arenas = tarenas; - narenas_total++; - arenas_extend(narenas_total - 1); - malloc_mutex_unlock(&arenas_lock); return (false); } @@ -921,7 +929,7 @@ void ctl_prefork(void) { - malloc_mutex_lock(&ctl_mtx); + malloc_mutex_prefork(&ctl_mtx); } void @@ -960,11 +968,11 @@ ctl_postfork_child(void) if (*oldlenp != sizeof(t)) { \ size_t copylen = (sizeof(t) <= *oldlenp) \ ? sizeof(t) : *oldlenp; \ - memcpy(oldp, (void *)&v, copylen); \ + memcpy(oldp, (void *)&(v), copylen); \ ret = EINVAL; \ goto label_return; \ } else \ - *(t *)oldp = v; \ + *(t *)oldp = (v); \ } \ } while (0) @@ -974,7 +982,7 @@ ctl_postfork_child(void) ret = EINVAL; \ goto label_return; \ } \ - v = *(t *)newp; \ + (v) = *(t *)newp; \ } \ } while (0) @@ -995,7 +1003,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ if (l) \ malloc_mutex_lock(&ctl_mtx); \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ @@ -1017,7 +1025,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ return (ENOENT); \ malloc_mutex_lock(&ctl_mtx); \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ @@ -1036,7 +1044,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ \ malloc_mutex_lock(&ctl_mtx); \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ @@ -1060,7 +1068,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ if ((c) == false) \ return (ENOENT); \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ @@ -1077,7 +1085,7 @@ n##_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, \ t oldval; \ \ READONLY(); \ - oldval = v; \ + oldval = (v); \ READ(oldval, t); \ \ ret = 0; \ @@ -1102,6 +1110,8 @@ label_return: \ return (ret); \ } +/******************************************************************************/ + CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int @@ -1109,7 +1119,7 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; - uint64_t newval; + UNUSED uint64_t newval; malloc_mutex_lock(&ctl_mtx); WRITE(newval, uint64_t); @@ -1123,49 +1133,52 @@ epoch_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, return (ret); } -static int -thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - int ret; - bool oldval; - - if (config_tcache == false) - return (ENOENT); - - oldval = tcache_enabled_get(); - if (newp != NULL) { - if (newlen != sizeof(bool)) { - ret = EINVAL; - goto label_return; - } - tcache_enabled_set(*(bool *)newp); - } - READ(oldval, bool); - - ret = 0; -label_return: - return (ret); -} - -static int -thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) -{ - int ret; +/******************************************************************************/ - if (config_tcache == false) - return (ENOENT); +CTL_RO_BOOL_CONFIG_GEN(config_debug) +CTL_RO_BOOL_CONFIG_GEN(config_dss) +CTL_RO_BOOL_CONFIG_GEN(config_fill) +CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) +CTL_RO_BOOL_CONFIG_GEN(config_mremap) +CTL_RO_BOOL_CONFIG_GEN(config_munmap) +CTL_RO_BOOL_CONFIG_GEN(config_prof) +CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) +CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind) +CTL_RO_BOOL_CONFIG_GEN(config_stats) +CTL_RO_BOOL_CONFIG_GEN(config_tcache) +CTL_RO_BOOL_CONFIG_GEN(config_tls) +CTL_RO_BOOL_CONFIG_GEN(config_utrace) +CTL_RO_BOOL_CONFIG_GEN(config_valgrind) +CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) - READONLY(); - WRITEONLY(); +/******************************************************************************/ - tcache_flush(); +CTL_RO_NL_GEN(opt_abort, opt_abort, bool) +CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) +CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) +CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) +CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) +CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) +CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool) +CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) +CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) +CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) +CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) +CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool) +CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) +CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) +CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) +CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) +CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */ +CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) +CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) +CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) +CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) +CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) - ret = 0; -label_return: - return (ret); -} +/******************************************************************************/ static int thread_arena_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, @@ -1227,50 +1240,49 @@ CTL_RO_NL_CGEN(config_stats, thread_deallocated, CTL_RO_NL_CGEN(config_stats, thread_deallocatedp, &thread_allocated_tsd_get()->deallocated, uint64_t *) -/******************************************************************************/ +static int +thread_tcache_enabled_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + bool oldval; -CTL_RO_BOOL_CONFIG_GEN(config_debug) -CTL_RO_BOOL_CONFIG_GEN(config_dss) -CTL_RO_BOOL_CONFIG_GEN(config_fill) -CTL_RO_BOOL_CONFIG_GEN(config_lazy_lock) -CTL_RO_BOOL_CONFIG_GEN(config_mremap) -CTL_RO_BOOL_CONFIG_GEN(config_munmap) -CTL_RO_BOOL_CONFIG_GEN(config_prof) -CTL_RO_BOOL_CONFIG_GEN(config_prof_libgcc) -CTL_RO_BOOL_CONFIG_GEN(config_prof_libunwind) -CTL_RO_BOOL_CONFIG_GEN(config_stats) -CTL_RO_BOOL_CONFIG_GEN(config_tcache) -CTL_RO_BOOL_CONFIG_GEN(config_tls) -CTL_RO_BOOL_CONFIG_GEN(config_utrace) -CTL_RO_BOOL_CONFIG_GEN(config_valgrind) -CTL_RO_BOOL_CONFIG_GEN(config_xmalloc) + if (config_tcache == false) + return (ENOENT); -/******************************************************************************/ + oldval = tcache_enabled_get(); + if (newp != NULL) { + if (newlen != sizeof(bool)) { + ret = EINVAL; + goto label_return; + } + tcache_enabled_set(*(bool *)newp); + } + READ(oldval, bool); -CTL_RO_NL_GEN(opt_abort, opt_abort, bool) -CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) -CTL_RO_NL_GEN(opt_lg_chunk, opt_lg_chunk, size_t) -CTL_RO_NL_GEN(opt_narenas, opt_narenas, size_t) -CTL_RO_NL_GEN(opt_lg_dirty_mult, opt_lg_dirty_mult, ssize_t) -CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) -CTL_RO_NL_CGEN(config_fill, opt_junk, opt_junk, bool) -CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) -CTL_RO_NL_CGEN(config_fill, opt_quarantine, opt_quarantine, size_t) -CTL_RO_NL_CGEN(config_fill, opt_redzone, opt_redzone, bool) -CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) -CTL_RO_NL_CGEN(config_valgrind, opt_valgrind, opt_valgrind, bool) -CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) -CTL_RO_NL_CGEN(config_tcache, opt_tcache, opt_tcache, bool) -CTL_RO_NL_CGEN(config_tcache, opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) -CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) -CTL_RO_CGEN(config_prof, opt_prof_active, opt_prof_active, bool) /* Mutable. */ -CTL_RO_NL_CGEN(config_prof, opt_lg_prof_sample, opt_lg_prof_sample, size_t) -CTL_RO_NL_CGEN(config_prof, opt_lg_prof_interval, opt_lg_prof_interval, ssize_t) -CTL_RO_NL_CGEN(config_prof, opt_prof_gdump, opt_prof_gdump, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_final, opt_prof_final, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) -CTL_RO_NL_CGEN(config_prof, opt_prof_accum, opt_prof_accum, bool) + ret = 0; +label_return: + return (ret); +} + +static int +thread_tcache_flush_ctl(const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) +{ + int ret; + + if (config_tcache == false) + return (ENOENT); + + READONLY(); + WRITEONLY(); + + tcache_flush(); + + ret = 0; +label_return: + return (ret); +} /******************************************************************************/ @@ -1382,31 +1394,8 @@ arena_i_index(const size_t *mib, size_t miblen, size_t i) return (ret); } - /******************************************************************************/ -CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) -CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) -CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) -static const ctl_named_node_t * -arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) -{ - - if (i > NBINS) - return (NULL); - return (super_arenas_bin_i_node); -} - -CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) -static const ctl_named_node_t * -arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) -{ - - if (i > nlclasses) - return (NULL); - return (super_arenas_lrun_i_node); -} - static int arenas_narenas_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) @@ -1460,7 +1449,28 @@ CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_CGEN(config_tcache, arenas_tcache_max, tcache_maxclass, size_t) CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) CTL_RO_NL_CGEN(config_tcache, arenas_nhbins, nhbins, unsigned) +CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) +CTL_RO_NL_GEN(arenas_bin_i_run_size, arena_bin_info[mib[2]].run_size, size_t) +static const ctl_named_node_t * +arenas_bin_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > NBINS) + return (NULL); + return (super_arenas_bin_i_node); +} + CTL_RO_NL_GEN(arenas_nlruns, nlclasses, size_t) +CTL_RO_NL_GEN(arenas_lrun_i_size, ((mib[2]+1) << LG_PAGE), size_t) +static const ctl_named_node_t * +arenas_lrun_i_index(const size_t *mib, size_t miblen, size_t i) +{ + + if (i > nlclasses) + return (NULL); + return (super_arenas_lrun_i_node); +} static int arenas_purge_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, @@ -1492,6 +1502,7 @@ arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; + unsigned narenas; malloc_mutex_lock(&ctl_mtx); READONLY(); @@ -1499,7 +1510,8 @@ arenas_extend_ctl(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, ret = EAGAIN; goto label_return; } - READ(ctl_stats.narenas - 1, unsigned); + narenas = ctl_stats.narenas - 1; + READ(narenas, unsigned); ret = 0; label_return: @@ -1565,6 +1577,11 @@ CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) /******************************************************************************/ +CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) +CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) +CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) +CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) + CTL_RO_CGEN(config_stats, stats_chunks_current, ctl_stats.chunks.current, size_t) CTL_RO_CGEN(config_stats, stats_chunks_total, ctl_stats.chunks.total, uint64_t) @@ -1572,6 +1589,20 @@ CTL_RO_CGEN(config_stats, stats_chunks_high, ctl_stats.chunks.high, size_t) CTL_RO_CGEN(config_stats, stats_huge_allocated, huge_allocated, size_t) CTL_RO_CGEN(config_stats, stats_huge_nmalloc, huge_nmalloc, uint64_t) CTL_RO_CGEN(config_stats, stats_huge_ndalloc, huge_ndalloc, uint64_t) + +CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) +CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) +CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) +CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, + ctl_stats.arenas[mib[2]].astats.mapped, size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_npurge, + ctl_stats.arenas[mib[2]].astats.npurge, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, + ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_purged, + ctl_stats.arenas[mib[2]].astats.purged, uint64_t) + CTL_RO_CGEN(config_stats, stats_arenas_i_small_allocated, ctl_stats.arenas[mib[2]].allocated_small, size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_small_nmalloc, @@ -1635,19 +1666,6 @@ stats_arenas_i_lruns_j_index(const size_t *mib, size_t miblen, size_t j) return (super_stats_arenas_i_lruns_j_node); } -CTL_RO_GEN(stats_arenas_i_nthreads, ctl_stats.arenas[mib[2]].nthreads, unsigned) -CTL_RO_GEN(stats_arenas_i_dss, ctl_stats.arenas[mib[2]].dss, const char *) -CTL_RO_GEN(stats_arenas_i_pactive, ctl_stats.arenas[mib[2]].pactive, size_t) -CTL_RO_GEN(stats_arenas_i_pdirty, ctl_stats.arenas[mib[2]].pdirty, size_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, - ctl_stats.arenas[mib[2]].astats.mapped, size_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_npurge, - ctl_stats.arenas[mib[2]].astats.npurge, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_nmadvise, - ctl_stats.arenas[mib[2]].astats.nmadvise, uint64_t) -CTL_RO_CGEN(config_stats, stats_arenas_i_purged, - ctl_stats.arenas[mib[2]].astats.purged, uint64_t) - static const ctl_named_node_t * stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) { @@ -1664,8 +1682,3 @@ stats_arenas_i_index(const size_t *mib, size_t miblen, size_t i) malloc_mutex_unlock(&ctl_mtx); return (ret); } - -CTL_RO_CGEN(config_stats, stats_cactive, &stats_cactive, size_t *) -CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats.allocated, size_t) -CTL_RO_CGEN(config_stats, stats_active, ctl_stats.active, size_t) -CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats.mapped, size_t) diff --git a/deps/jemalloc/src/huge.c b/deps/jemalloc/src/huge.c index aa08d43d362..d72f2135702 100644 --- a/deps/jemalloc/src/huge.c +++ b/deps/jemalloc/src/huge.c @@ -16,14 +16,14 @@ malloc_mutex_t huge_mtx; static extent_tree_t huge; void * -huge_malloc(size_t size, bool zero) +huge_malloc(size_t size, bool zero, dss_prec_t dss_prec) { - return (huge_palloc(size, chunksize, zero)); + return (huge_palloc(size, chunksize, zero, dss_prec)); } void * -huge_palloc(size_t size, size_t alignment, bool zero) +huge_palloc(size_t size, size_t alignment, bool zero, dss_prec_t dss_prec) { void *ret; size_t csize; @@ -48,8 +48,7 @@ huge_palloc(size_t size, size_t alignment, bool zero) * it is possible to make correct junk/zero fill decisions below. */ is_zeroed = zero; - ret = chunk_alloc(csize, alignment, false, &is_zeroed, - chunk_dss_prec_get()); + ret = chunk_alloc(csize, alignment, false, &is_zeroed, dss_prec); if (ret == NULL) { base_node_dealloc(node); return (NULL); @@ -78,7 +77,7 @@ huge_palloc(size_t size, size_t alignment, bool zero) return (ret); } -void * +bool huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) { @@ -89,28 +88,23 @@ huge_ralloc_no_move(void *ptr, size_t oldsize, size_t size, size_t extra) && CHUNK_CEILING(oldsize) >= CHUNK_CEILING(size) && CHUNK_CEILING(oldsize) <= CHUNK_CEILING(size+extra)) { assert(CHUNK_CEILING(oldsize) == oldsize); - if (config_fill && opt_junk && size < oldsize) { - memset((void *)((uintptr_t)ptr + size), 0x5a, - oldsize - size); - } - return (ptr); + return (false); } /* Reallocation would require a move. */ - return (NULL); + return (true); } void * huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero, bool try_tcache_dalloc) + size_t alignment, bool zero, bool try_tcache_dalloc, dss_prec_t dss_prec) { void *ret; size_t copysize; /* Try to avoid moving the allocation. */ - ret = huge_ralloc_no_move(ptr, oldsize, size, extra); - if (ret != NULL) - return (ret); + if (huge_ralloc_no_move(ptr, oldsize, size, extra) == false) + return (ptr); /* * size and oldsize are different enough that we need to use a @@ -118,18 +112,18 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, * space and copying. */ if (alignment > chunksize) - ret = huge_palloc(size + extra, alignment, zero); + ret = huge_palloc(size + extra, alignment, zero, dss_prec); else - ret = huge_malloc(size + extra, zero); + ret = huge_malloc(size + extra, zero, dss_prec); if (ret == NULL) { if (extra == 0) return (NULL); /* Try again, this time without extra. */ if (alignment > chunksize) - ret = huge_palloc(size, alignment, zero); + ret = huge_palloc(size, alignment, zero, dss_prec); else - ret = huge_malloc(size, zero); + ret = huge_malloc(size, zero, dss_prec); if (ret == NULL) return (NULL); @@ -169,23 +163,56 @@ huge_ralloc(void *ptr, size_t oldsize, size_t size, size_t extra, */ char buf[BUFERROR_BUF]; - buferror(buf, sizeof(buf)); + buferror(get_errno(), buf, sizeof(buf)); malloc_printf(": Error in mremap(): %s\n", buf); if (opt_abort) abort(); memcpy(ret, ptr, copysize); chunk_dealloc_mmap(ptr, oldsize); + } else if (config_fill && zero == false && opt_junk && oldsize + < newsize) { + /* + * mremap(2) clobbers the original mapping, so + * junk/zero filling is not preserved. There is no + * need to zero fill here, since any trailing + * uninititialized memory is demand-zeroed by the + * kernel, but junk filling must be redone. + */ + memset(ret + oldsize, 0xa5, newsize - oldsize); } } else #endif { memcpy(ret, ptr, copysize); - iqallocx(ptr, try_tcache_dalloc); + iqalloct(ptr, try_tcache_dalloc); } return (ret); } +#ifdef JEMALLOC_JET +#undef huge_dalloc_junk +#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk_impl) +#endif +static void +huge_dalloc_junk(void *ptr, size_t usize) +{ + + if (config_fill && config_dss && opt_junk) { + /* + * Only bother junk filling if the chunk isn't about to be + * unmapped. + */ + if (config_munmap == false || (config_dss && chunk_in_dss(ptr))) + memset(ptr, 0x5a, usize); + } +} +#ifdef JEMALLOC_JET +#undef huge_dalloc_junk +#define huge_dalloc_junk JEMALLOC_N(huge_dalloc_junk) +huge_dalloc_junk_t *huge_dalloc_junk = JEMALLOC_N(huge_dalloc_junk_impl); +#endif + void huge_dalloc(void *ptr, bool unmap) { @@ -208,8 +235,8 @@ huge_dalloc(void *ptr, bool unmap) malloc_mutex_unlock(&huge_mtx); - if (unmap && config_fill && config_dss && opt_junk) - memset(node->addr, 0x5a, node->size); + if (unmap) + huge_dalloc_junk(node->addr, node->size); chunk_dealloc(node->addr, node->size, unmap); @@ -236,6 +263,13 @@ huge_salloc(const void *ptr) return (ret); } +dss_prec_t +huge_dss_prec_get(arena_t *arena) +{ + + return (arena_dss_prec_get(choose_arena(arena))); +} + prof_ctx_t * huge_prof_ctx_get(const void *ptr) { diff --git a/deps/jemalloc/src/jemalloc.c b/deps/jemalloc/src/jemalloc.c index 8a667b62e2d..204778bc89d 100644 --- a/deps/jemalloc/src/jemalloc.c +++ b/deps/jemalloc/src/jemalloc.c @@ -10,17 +10,20 @@ malloc_tsd_data(, thread_allocated, thread_allocated_t, /* Runtime configuration options. */ const char *je_malloc_conf; +bool opt_abort = #ifdef JEMALLOC_DEBUG -bool opt_abort = true; -# ifdef JEMALLOC_FILL -bool opt_junk = true; -# else -bool opt_junk = false; -# endif + true +#else + false +#endif + ; +bool opt_junk = +#if (defined(JEMALLOC_DEBUG) && defined(JEMALLOC_FILL)) + true #else -bool opt_abort = false; -bool opt_junk = false; + false #endif + ; size_t opt_quarantine = ZU(0); bool opt_redzone = false; bool opt_utrace = false; @@ -83,11 +86,13 @@ typedef struct { #ifdef JEMALLOC_UTRACE # define UTRACE(a, b, c) do { \ if (opt_utrace) { \ + int utrace_serrno = errno; \ malloc_utrace_t ut; \ ut.p = (a); \ ut.s = (b); \ ut.r = (c); \ utrace(&ut, sizeof(ut)); \ + errno = utrace_serrno; \ } \ } while (0) #else @@ -95,18 +100,12 @@ typedef struct { #endif /******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static void stats_print_atexit(void); -static unsigned malloc_ncpus(void); -static bool malloc_conf_next(char const **opts_p, char const **k_p, - size_t *klen_p, char const **v_p, size_t *vlen_p); -static void malloc_conf_error(const char *msg, const char *k, size_t klen, - const char *v, size_t vlen); -static void malloc_conf_init(void); +/* + * Function prototypes for static functions that are referenced prior to + * definition. + */ + static bool malloc_init_hard(void); -static int imemalign(void **memptr, size_t alignment, size_t size, - size_t min_alignment); /******************************************************************************/ /* @@ -247,7 +246,6 @@ stats_print_atexit(void) static unsigned malloc_ncpus(void) { - unsigned ret; long result; #ifdef _WIN32 @@ -257,14 +255,7 @@ malloc_ncpus(void) #else result = sysconf(_SC_NPROCESSORS_ONLN); #endif - if (result == -1) { - /* Error. */ - ret = 1; - } else { - ret = (unsigned)result; - } - - return (ret); + return ((result == -1) ? 1 : (unsigned)result); } void @@ -277,12 +268,30 @@ arenas_cleanup(void *arg) malloc_mutex_unlock(&arenas_lock); } -static inline bool +JEMALLOC_ALWAYS_INLINE_C void +malloc_thread_init(void) +{ + + /* + * TSD initialization can't be safely done as a side effect of + * deallocation, because it is possible for a thread to do nothing but + * deallocate its TLS data via free(), in which case writing to TLS + * would cause write-after-free memory corruption. The quarantine + * facility *only* gets used as a side effect of deallocation, so make + * a best effort attempt at initializing its TSD by hooking all + * allocation events. + */ + if (config_fill && opt_quarantine) + quarantine_alloc_hook(); +} + +JEMALLOC_ALWAYS_INLINE_C bool malloc_init(void) { - if (malloc_initialized == false) - return (malloc_init_hard()); + if (malloc_initialized == false && malloc_init_hard()) + return (true); + malloc_thread_init(); return (false); } @@ -413,8 +422,9 @@ malloc_conf_init(void) } break; case 1: { + int linklen = 0; #ifndef _WIN32 - int linklen; + int saved_errno = errno; const char *linkname = # ifdef JEMALLOC_PREFIX "/etc/"JEMALLOC_PREFIX"malloc.conf" @@ -423,21 +433,20 @@ malloc_conf_init(void) # endif ; - if ((linklen = readlink(linkname, buf, - sizeof(buf) - 1)) != -1) { - /* - * Use the contents of the "/etc/malloc.conf" - * symbolic link's name. - */ - buf[linklen] = '\0'; - opts = buf; - } else -#endif - { + /* + * Try to use the contents of the "/etc/malloc.conf" + * symbolic link's name. + */ + linklen = readlink(linkname, buf, sizeof(buf) - 1); + if (linklen == -1) { /* No configuration specified. */ - buf[0] = '\0'; - opts = buf; + linklen = 0; + /* restore errno */ + set_errno(saved_errno); } +#endif + buf[linklen] = '\0'; + opts = buf; break; } case 2: { const char *envname = @@ -461,15 +470,14 @@ malloc_conf_init(void) } break; } default: - /* NOTREACHED */ - assert(false); + not_reached(); buf[0] = '\0'; opts = buf; } while (*opts != '\0' && malloc_conf_next(&opts, &k, &klen, &v, &vlen) == false) { -#define CONF_HANDLE_BOOL_HIT(o, n, hit) \ +#define CONF_HANDLE_BOOL(o, n) \ if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ if (strncmp("true", v, vlen) == 0 && \ @@ -483,16 +491,9 @@ malloc_conf_init(void) "Invalid conf value", \ k, klen, v, vlen); \ } \ - hit = true; \ - } else \ - hit = false; -#define CONF_HANDLE_BOOL(o, n) { \ - bool hit; \ - CONF_HANDLE_BOOL_HIT(o, n, hit); \ - if (hit) \ continue; \ -} -#define CONF_HANDLE_SIZE_T(o, n, min, max) \ + } +#define CONF_HANDLE_SIZE_T(o, n, min, max, clip) \ if (sizeof(n)-1 == klen && strncmp(n, k, \ klen) == 0) { \ uintmax_t um; \ @@ -505,12 +506,23 @@ malloc_conf_init(void) malloc_conf_error( \ "Invalid conf value", \ k, klen, v, vlen); \ - } else if (um < min || um > max) { \ - malloc_conf_error( \ - "Out-of-range conf value", \ - k, klen, v, vlen); \ - } else \ - o = um; \ + } else if (clip) { \ + if (min != 0 && um < min) \ + o = min; \ + else if (um > max) \ + o = max; \ + else \ + o = um; \ + } else { \ + if ((min != 0 && um < min) || \ + um > max) { \ + malloc_conf_error( \ + "Out-of-range " \ + "conf value", \ + k, klen, v, vlen); \ + } else \ + o = um; \ + } \ continue; \ } #define CONF_HANDLE_SSIZE_T(o, n, min, max) \ @@ -555,7 +567,8 @@ malloc_conf_init(void) * config_fill. */ CONF_HANDLE_SIZE_T(opt_lg_chunk, "lg_chunk", LG_PAGE + - (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1) + (config_fill ? 2 : 1), (sizeof(size_t) << 3) - 1, + true) if (strncmp("dss", k, klen) == 0) { int i; bool match = false; @@ -581,14 +594,14 @@ malloc_conf_init(void) continue; } CONF_HANDLE_SIZE_T(opt_narenas, "narenas", 1, - SIZE_T_MAX) + SIZE_T_MAX, false) CONF_HANDLE_SSIZE_T(opt_lg_dirty_mult, "lg_dirty_mult", -1, (sizeof(size_t) << 3) - 1) CONF_HANDLE_BOOL(opt_stats_print, "stats_print") if (config_fill) { CONF_HANDLE_BOOL(opt_junk, "junk") CONF_HANDLE_SIZE_T(opt_quarantine, "quarantine", - 0, SIZE_T_MAX) + 0, SIZE_T_MAX, false) CONF_HANDLE_BOOL(opt_redzone, "redzone") CONF_HANDLE_BOOL(opt_zero, "zero") } @@ -668,17 +681,6 @@ malloc_init_hard(void) malloc_conf_init(); -#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ - && !defined(_WIN32)) - /* Register fork handlers. */ - if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, - jemalloc_postfork_child) != 0) { - malloc_write(": Error in pthread_atfork()\n"); - if (opt_abort) - abort(); - } -#endif - if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -718,8 +720,10 @@ malloc_init_hard(void) return (true); } - if (malloc_mutex_init(&arenas_lock)) + if (malloc_mutex_init(&arenas_lock)) { + malloc_mutex_unlock(&init_lock); return (true); + } /* * Create enough scaffolding to allow recursive allocation in @@ -765,9 +769,25 @@ malloc_init_hard(void) return (true); } - /* Get number of CPUs. */ malloc_mutex_unlock(&init_lock); + /**********************************************************************/ + /* Recursive allocation may follow. */ + ncpus = malloc_ncpus(); + +#if (!defined(JEMALLOC_MUTEX_INIT_CB) && !defined(JEMALLOC_ZONE) \ + && !defined(_WIN32)) + /* LinuxThreads's pthread_atfork() allocates. */ + if (pthread_atfork(jemalloc_prefork, jemalloc_postfork_parent, + jemalloc_postfork_child) != 0) { + malloc_write(": Error in pthread_atfork()\n"); + if (opt_abort) + abort(); + } +#endif + + /* Done recursively allocating. */ + /**********************************************************************/ malloc_mutex_lock(&init_lock); if (mutex_boot()) { @@ -814,6 +834,7 @@ malloc_init_hard(void) malloc_initialized = true; malloc_mutex_unlock(&init_lock); + return (false); } @@ -825,42 +846,88 @@ malloc_init_hard(void) * Begin malloc(3)-compatible functions. */ +static void * +imalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = imalloc(SMALL_MAXCLASS+1); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = imalloc(usize); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +imalloc_prof(size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if ((uintptr_t)cnt != (uintptr_t)1U) + p = imalloc_prof_sample(usize, cnt); + else + p = imalloc(usize); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); + + return (p); +} + +/* + * MALLOC_BODY() is a macro rather than a function because its contents are in + * the fast path, but inlining would cause reliability issues when determining + * how many frames to discard from heap profiling backtraces. + */ +#define MALLOC_BODY(ret, size, usize) do { \ + if (malloc_init()) \ + ret = NULL; \ + else { \ + if (config_prof && opt_prof) { \ + prof_thr_cnt_t *cnt; \ + \ + usize = s2u(size); \ + /* \ + * Call PROF_ALLOC_PREP() here rather than in \ + * imalloc_prof() so that imalloc_prof() can be \ + * inlined without introducing uncertainty \ + * about the number of backtrace frames to \ + * ignore. imalloc_prof() is in the fast path \ + * when heap profiling is enabled, so inlining \ + * is critical to performance. (For \ + * consistency all callers of PROF_ALLOC_PREP() \ + * are structured similarly, even though e.g. \ + * realloc() isn't called enough for inlining \ + * to be critical.) \ + */ \ + PROF_ALLOC_PREP(1, usize, cnt); \ + ret = imalloc_prof(usize, cnt); \ + } else { \ + if (config_stats || (config_valgrind && \ + opt_valgrind)) \ + usize = s2u(size); \ + ret = imalloc(size); \ + } \ + } \ +} while (0) + void * je_malloc(size_t size) { void *ret; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); - - if (malloc_init()) { - ret = NULL; - goto label_oom; - } if (size == 0) size = 1; - if (config_prof && opt_prof) { - usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - ret = NULL; - goto label_oom; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - SMALL_MAXCLASS) { - ret = imalloc(SMALL_MAXCLASS+1); - if (ret != NULL) - arena_prof_promoted(ret, usize); - } else - ret = imalloc(size); - } else { - if (config_stats || (config_valgrind && opt_valgrind)) - usize = s2u(size); - ret = imalloc(size); - } + MALLOC_BODY(ret, size, usize); -label_oom: if (ret == NULL) { if (config_xmalloc && opt_xmalloc) { malloc_write(": Error in malloc(): " @@ -869,8 +936,6 @@ je_malloc(size_t size) } set_errno(ENOMEM); } - if (config_prof && opt_prof && ret != NULL) - prof_malloc(ret, usize, cnt); if (config_stats && ret != NULL) { assert(usize == isalloc(ret, config_prof)); thread_allocated_tsd_get()->allocated += usize; @@ -880,28 +945,63 @@ je_malloc(size_t size) return (ret); } +static void * +imemalign_prof_sample(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + assert(sa2u(SMALL_MAXCLASS+1, alignment) != 0); + p = ipalloc(sa2u(SMALL_MAXCLASS+1, alignment), alignment, + false); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = ipalloc(usize, alignment, false); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +imemalign_prof(size_t alignment, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if ((uintptr_t)cnt != (uintptr_t)1U) + p = imemalign_prof_sample(alignment, usize, cnt); + else + p = ipalloc(usize, alignment, false); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); + + return (p); +} + JEMALLOC_ATTR(nonnull(1)) #ifdef JEMALLOC_PROF /* * Avoid any uncertainty as to how many backtrace frames to ignore in * PROF_ALLOC_PREP(). */ -JEMALLOC_ATTR(noinline) +JEMALLOC_NOINLINE #endif static int -imemalign(void **memptr, size_t alignment, size_t size, - size_t min_alignment) +imemalign(void **memptr, size_t alignment, size_t size, size_t min_alignment) { int ret; size_t usize; void *result; - prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); assert(min_alignment != 0); - if (malloc_init()) + if (malloc_init()) { result = NULL; - else { + goto label_oom; + } else { if (size == 0) size = 1; @@ -921,57 +1021,38 @@ imemalign(void **memptr, size_t alignment, size_t size, usize = sa2u(size, alignment); if (usize == 0) { result = NULL; - ret = ENOMEM; - goto label_return; + goto label_oom; } if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + PROF_ALLOC_PREP(2, usize, cnt); - if (cnt == NULL) { - result = NULL; - ret = EINVAL; - } else { - if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= SMALL_MAXCLASS) { - assert(sa2u(SMALL_MAXCLASS+1, - alignment) != 0); - result = ipalloc(sa2u(SMALL_MAXCLASS+1, - alignment), alignment, false); - if (result != NULL) { - arena_prof_promoted(result, - usize); - } - } else { - result = ipalloc(usize, alignment, - false); - } - } + result = imemalign_prof(alignment, usize, cnt); } else result = ipalloc(usize, alignment, false); - } - - if (result == NULL) { - if (config_xmalloc && opt_xmalloc) { - malloc_write(": Error allocating aligned " - "memory: out of memory\n"); - abort(); - } - ret = ENOMEM; - goto label_return; + if (result == NULL) + goto label_oom; } *memptr = result; ret = 0; - label_return: if (config_stats && result != NULL) { assert(usize == isalloc(result, config_prof)); thread_allocated_tsd_get()->allocated += usize; } - if (config_prof && opt_prof && result != NULL) - prof_malloc(result, usize, cnt); UTRACE(0, size, result); return (ret); +label_oom: + assert(result == NULL); + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error allocating aligned memory: " + "out of memory\n"); + abort(); + } + ret = ENOMEM; + goto label_return; } int @@ -998,13 +1079,46 @@ je_aligned_alloc(size_t alignment, size_t size) return (ret); } +static void * +icalloc_prof_sample(size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = icalloc(SMALL_MAXCLASS+1); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = icalloc(usize); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +icalloc_prof(size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if ((uintptr_t)cnt != (uintptr_t)1U) + p = icalloc_prof_sample(usize, cnt); + else + p = icalloc(usize); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); + + return (p); +} + void * je_calloc(size_t num, size_t size) { void *ret; size_t num_size; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); if (malloc_init()) { num_size = 0; @@ -1033,19 +1147,11 @@ je_calloc(size_t num, size_t size) } if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + usize = s2u(num_size); PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - ret = NULL; - goto label_return; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize - <= SMALL_MAXCLASS) { - ret = icalloc(SMALL_MAXCLASS+1); - if (ret != NULL) - arena_prof_promoted(ret, usize); - } else - ret = icalloc(num_size); + ret = icalloc_prof(usize, cnt); } else { if (config_stats || (config_valgrind && opt_valgrind)) usize = s2u(num_size); @@ -1061,9 +1167,6 @@ je_calloc(size_t num, size_t size) } set_errno(ENOMEM); } - - if (config_prof && opt_prof && ret != NULL) - prof_malloc(ret, usize, cnt); if (config_stats && ret != NULL) { assert(usize == isalloc(ret, config_prof)); thread_allocated_tsd_get()->allocated += usize; @@ -1073,150 +1176,126 @@ je_calloc(size_t num, size_t size) return (ret); } +static void * +irealloc_prof_sample(void *oldptr, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = iralloc(oldptr, SMALL_MAXCLASS+1, 0, 0, false); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = iralloc(oldptr, usize, 0, 0, false); + + return (p); +} + +JEMALLOC_ALWAYS_INLINE_C void * +irealloc_prof(void *oldptr, size_t old_usize, size_t usize, prof_thr_cnt_t *cnt) +{ + void *p; + prof_ctx_t *old_ctx; + + old_ctx = prof_ctx_get(oldptr); + if ((uintptr_t)cnt != (uintptr_t)1U) + p = irealloc_prof_sample(oldptr, usize, cnt); + else + p = iralloc(oldptr, usize, 0, 0, false); + if (p == NULL) + return (NULL); + prof_realloc(p, usize, cnt, old_usize, old_ctx); + + return (p); +} + +JEMALLOC_INLINE_C void +ifree(void *ptr) +{ + size_t usize; + UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); + + assert(ptr != NULL); + assert(malloc_initialized || IS_INITIALIZER); + + if (config_prof && opt_prof) { + usize = isalloc(ptr, config_prof); + prof_free(ptr, usize); + } else if (config_stats || config_valgrind) + usize = isalloc(ptr, config_prof); + if (config_stats) + thread_allocated_tsd_get()->deallocated += usize; + if (config_valgrind && opt_valgrind) + rzsize = p2rz(ptr); + iqalloc(ptr); + JEMALLOC_VALGRIND_FREE(ptr, rzsize); +} + void * je_realloc(void *ptr, size_t size) { void *ret; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - size_t old_size = 0; - size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); - prof_thr_cnt_t *cnt JEMALLOC_CC_SILENCE_INIT(NULL); - prof_ctx_t *old_ctx JEMALLOC_CC_SILENCE_INIT(NULL); + size_t old_usize = 0; + UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); if (size == 0) { if (ptr != NULL) { - /* realloc(ptr, 0) is equivalent to free(p). */ - if (config_prof) { - old_size = isalloc(ptr, true); - if (config_valgrind && opt_valgrind) - old_rzsize = p2rz(ptr); - } else if (config_stats) { - old_size = isalloc(ptr, false); - if (config_valgrind && opt_valgrind) - old_rzsize = u2rz(old_size); - } else if (config_valgrind && opt_valgrind) { - old_size = isalloc(ptr, false); - old_rzsize = u2rz(old_size); - } - if (config_prof && opt_prof) { - old_ctx = prof_ctx_get(ptr); - cnt = NULL; - } - iqalloc(ptr); - ret = NULL; - goto label_return; - } else - size = 1; + /* realloc(ptr, 0) is equivalent to free(ptr). */ + UTRACE(ptr, 0, 0); + ifree(ptr); + return (NULL); + } + size = 1; } if (ptr != NULL) { assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); + + if ((config_prof && opt_prof) || config_stats || + (config_valgrind && opt_valgrind)) + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && opt_valgrind) + old_rzsize = config_prof ? p2rz(ptr) : u2rz(old_usize); - if (config_prof) { - old_size = isalloc(ptr, true); - if (config_valgrind && opt_valgrind) - old_rzsize = p2rz(ptr); - } else if (config_stats) { - old_size = isalloc(ptr, false); - if (config_valgrind && opt_valgrind) - old_rzsize = u2rz(old_size); - } else if (config_valgrind && opt_valgrind) { - old_size = isalloc(ptr, false); - old_rzsize = u2rz(old_size); - } if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + usize = s2u(size); - old_ctx = prof_ctx_get(ptr); PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) { - old_ctx = NULL; - ret = NULL; - goto label_oom; - } - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && - usize <= SMALL_MAXCLASS) { - ret = iralloc(ptr, SMALL_MAXCLASS+1, 0, 0, - false, false); - if (ret != NULL) - arena_prof_promoted(ret, usize); - else - old_ctx = NULL; - } else { - ret = iralloc(ptr, size, 0, 0, false, false); - if (ret == NULL) - old_ctx = NULL; - } + ret = irealloc_prof(ptr, old_usize, usize, cnt); } else { if (config_stats || (config_valgrind && opt_valgrind)) usize = s2u(size); - ret = iralloc(ptr, size, 0, 0, false, false); - } - -label_oom: - if (ret == NULL) { - if (config_xmalloc && opt_xmalloc) { - malloc_write(": Error in realloc(): " - "out of memory\n"); - abort(); - } - set_errno(ENOMEM); + ret = iralloc(ptr, size, 0, 0, false); } } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - if (config_prof && opt_prof) - old_ctx = NULL; - if (malloc_init()) { - if (config_prof && opt_prof) - cnt = NULL; - ret = NULL; - } else { - if (config_prof && opt_prof) { - usize = s2u(size); - PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) - ret = NULL; - else { - if (prof_promote && (uintptr_t)cnt != - (uintptr_t)1U && usize <= - SMALL_MAXCLASS) { - ret = imalloc(SMALL_MAXCLASS+1); - if (ret != NULL) { - arena_prof_promoted(ret, - usize); - } - } else - ret = imalloc(size); - } - } else { - if (config_stats || (config_valgrind && - opt_valgrind)) - usize = s2u(size); - ret = imalloc(size); - } - } + MALLOC_BODY(ret, size, usize); + } - if (ret == NULL) { - if (config_xmalloc && opt_xmalloc) { - malloc_write(": Error in realloc(): " - "out of memory\n"); - abort(); - } - set_errno(ENOMEM); + if (ret == NULL) { + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error in realloc(): " + "out of memory\n"); + abort(); } + set_errno(ENOMEM); } - -label_return: - if (config_prof && opt_prof) - prof_realloc(ret, usize, cnt, old_size, old_ctx); if (config_stats && ret != NULL) { thread_allocated_t *ta; assert(usize == isalloc(ret, config_prof)); ta = thread_allocated_tsd_get(); ta->allocated += usize; - ta->deallocated += old_size; + ta->deallocated += old_usize; } UTRACE(ptr, size, ret); - JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_size, old_rzsize, false); + JEMALLOC_VALGRIND_REALLOC(ret, usize, ptr, old_usize, old_rzsize, + false); return (ret); } @@ -1225,24 +1304,8 @@ je_free(void *ptr) { UTRACE(ptr, 0, 0); - if (ptr != NULL) { - size_t usize; - size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); - - assert(malloc_initialized || IS_INITIALIZER); - - if (config_prof && opt_prof) { - usize = isalloc(ptr, config_prof); - prof_free(ptr, usize); - } else if (config_stats || config_valgrind) - usize = isalloc(ptr, config_prof); - if (config_stats) - thread_allocated_tsd_get()->deallocated += usize; - if (config_valgrind && opt_valgrind) - rzsize = p2rz(ptr); - iqalloc(ptr); - JEMALLOC_VALGRIND_FREE(ptr, rzsize); - } + if (ptr != NULL) + ifree(ptr); } /* @@ -1308,206 +1371,344 @@ JEMALLOC_EXPORT void *(* __memalign_hook)(size_t alignment, size_t size) = * Begin non-standard functions. */ -size_t -je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) +JEMALLOC_ALWAYS_INLINE_C void * +imallocx(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena) { - size_t ret; - assert(malloc_initialized || IS_INITIALIZER); + assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, + alignment))); - if (config_ivsalloc) - ret = ivsalloc(ptr, config_prof); + if (alignment != 0) + return (ipalloct(usize, alignment, zero, try_tcache, arena)); + else if (zero) + return (icalloct(usize, try_tcache, arena)); else - ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; - - return (ret); + return (imalloct(usize, try_tcache, arena)); } -void -je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, - const char *opts) +static void * +imallocx_prof_sample(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena, prof_thr_cnt_t *cnt) { + void *p; - stats_print(write_cb, cbopaque, opts); + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + size_t usize_promoted = (alignment == 0) ? + s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, alignment); + assert(usize_promoted != 0); + p = imallocx(usize_promoted, alignment, zero, try_tcache, + arena); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else + p = imallocx(usize, alignment, zero, try_tcache, arena); + + return (p); } -int -je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, - size_t newlen) +JEMALLOC_ALWAYS_INLINE_C void * +imallocx_prof(size_t usize, size_t alignment, bool zero, bool try_tcache, + arena_t *arena, prof_thr_cnt_t *cnt) { + void *p; - if (malloc_init()) - return (EAGAIN); + if ((uintptr_t)cnt != (uintptr_t)1U) { + p = imallocx_prof_sample(usize, alignment, zero, try_tcache, + arena, cnt); + } else + p = imallocx(usize, alignment, zero, try_tcache, arena); + if (p == NULL) + return (NULL); + prof_malloc(p, usize, cnt); - return (ctl_byname(name, oldp, oldlenp, newp, newlen)); + return (p); } -int -je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) +void * +je_mallocx(size_t size, int flags) { + void *p; + size_t usize; + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) + & (SIZE_T_MAX-1)); + bool zero = flags & MALLOCX_ZERO; + unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + arena_t *arena; + bool try_tcache; + + assert(size != 0); if (malloc_init()) - return (EAGAIN); + goto label_oom; - return (ctl_nametomib(name, mibp, miblenp)); + if (arena_ind != UINT_MAX) { + arena = arenas[arena_ind]; + try_tcache = false; + } else { + arena = NULL; + try_tcache = true; + } + + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + assert(usize != 0); + + if (config_prof && opt_prof) { + prof_thr_cnt_t *cnt; + + PROF_ALLOC_PREP(1, usize, cnt); + p = imallocx_prof(usize, alignment, zero, try_tcache, arena, + cnt); + } else + p = imallocx(usize, alignment, zero, try_tcache, arena); + if (p == NULL) + goto label_oom; + + if (config_stats) { + assert(usize == isalloc(p, config_prof)); + thread_allocated_tsd_get()->allocated += usize; + } + UTRACE(0, size, p); + JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); + return (p); +label_oom: + if (config_xmalloc && opt_xmalloc) { + malloc_write(": Error in mallocx(): out of memory\n"); + abort(); + } + UTRACE(0, size, 0); + return (NULL); } -int -je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, - void *newp, size_t newlen) +static void * +irallocx_prof_sample(void *oldptr, size_t size, size_t alignment, size_t usize, + bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, arena_t *arena, + prof_thr_cnt_t *cnt) { + void *p; - if (malloc_init()) - return (EAGAIN); + if (cnt == NULL) + return (NULL); + if (prof_promote && usize <= SMALL_MAXCLASS) { + p = iralloct(oldptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= + size) ? 0 : size - (SMALL_MAXCLASS+1), alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); + if (p == NULL) + return (NULL); + arena_prof_promoted(p, usize); + } else { + p = iralloct(oldptr, size, 0, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); + } - return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); + return (p); } -/* - * End non-standard functions. - */ -/******************************************************************************/ -/* - * Begin experimental functions. - */ -#ifdef JEMALLOC_EXPERIMENTAL - -JEMALLOC_INLINE void * -iallocm(size_t usize, size_t alignment, bool zero, bool try_tcache, - arena_t *arena) +JEMALLOC_ALWAYS_INLINE_C void * +irallocx_prof(void *oldptr, size_t old_usize, size_t size, size_t alignment, + size_t *usize, bool zero, bool try_tcache_alloc, bool try_tcache_dalloc, + arena_t *arena, prof_thr_cnt_t *cnt) { + void *p; + prof_ctx_t *old_ctx; - assert(usize == ((alignment == 0) ? s2u(usize) : sa2u(usize, - alignment))); + old_ctx = prof_ctx_get(oldptr); + if ((uintptr_t)cnt != (uintptr_t)1U) + p = irallocx_prof_sample(oldptr, size, alignment, *usize, zero, + try_tcache_alloc, try_tcache_dalloc, arena, cnt); + else { + p = iralloct(oldptr, size, 0, alignment, zero, + try_tcache_alloc, try_tcache_dalloc, arena); + } + if (p == NULL) + return (NULL); - if (alignment != 0) - return (ipallocx(usize, alignment, zero, try_tcache, arena)); - else if (zero) - return (icallocx(usize, try_tcache, arena)); - else - return (imallocx(usize, try_tcache, arena)); + if (p == oldptr && alignment != 0) { + /* + * The allocation did not move, so it is possible that the size + * class is smaller than would guarantee the requested + * alignment, and that the alignment constraint was + * serendipitously satisfied. Additionally, old_usize may not + * be the same as the current usize because of in-place large + * reallocation. Therefore, query the actual value of usize. + */ + *usize = isalloc(p, config_prof); + } + prof_realloc(p, *usize, cnt, old_usize, old_ctx); + + return (p); } -int -je_allocm(void **ptr, size_t *rsize, size_t size, int flags) +void * +je_rallocx(void *ptr, size_t size, int flags) { void *p; - size_t usize; - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + size_t usize, old_usize; + UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); - bool zero = flags & ALLOCM_ZERO; + bool zero = flags & MALLOCX_ZERO; unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; + bool try_tcache_alloc, try_tcache_dalloc; arena_t *arena; - bool try_tcache; assert(ptr != NULL); assert(size != 0); - - if (malloc_init()) - goto label_oom; + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); if (arena_ind != UINT_MAX) { + arena_chunk_t *chunk; + try_tcache_alloc = false; + chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(ptr); + try_tcache_dalloc = (chunk == ptr || chunk->arena != + arenas[arena_ind]); arena = arenas[arena_ind]; - try_tcache = false; } else { + try_tcache_alloc = true; + try_tcache_dalloc = true; arena = NULL; - try_tcache = true; } - usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); - if (usize == 0) - goto label_oom; + if ((config_prof && opt_prof) || config_stats || + (config_valgrind && opt_valgrind)) + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_usize); if (config_prof && opt_prof) { prof_thr_cnt_t *cnt; + usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); + assert(usize != 0); PROF_ALLOC_PREP(1, usize, cnt); - if (cnt == NULL) + p = irallocx_prof(ptr, old_usize, size, alignment, &usize, zero, + try_tcache_alloc, try_tcache_dalloc, arena, cnt); + if (p == NULL) goto label_oom; - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U && usize <= - SMALL_MAXCLASS) { - size_t usize_promoted = (alignment == 0) ? - s2u(SMALL_MAXCLASS+1) : sa2u(SMALL_MAXCLASS+1, - alignment); - assert(usize_promoted != 0); - p = iallocm(usize_promoted, alignment, zero, - try_tcache, arena); - if (p == NULL) - goto label_oom; - arena_prof_promoted(p, usize); - } else { - p = iallocm(usize, alignment, zero, try_tcache, arena); - if (p == NULL) - goto label_oom; - } - prof_malloc(p, usize, cnt); } else { - p = iallocm(usize, alignment, zero, try_tcache, arena); + p = iralloct(ptr, size, 0, alignment, zero, try_tcache_alloc, + try_tcache_dalloc, arena); if (p == NULL) goto label_oom; + if (config_stats || (config_valgrind && opt_valgrind)) + usize = isalloc(p, config_prof); } - if (rsize != NULL) - *rsize = usize; - *ptr = p; if (config_stats) { - assert(usize == isalloc(p, config_prof)); - thread_allocated_tsd_get()->allocated += usize; + thread_allocated_t *ta; + ta = thread_allocated_tsd_get(); + ta->allocated += usize; + ta->deallocated += old_usize; } - UTRACE(0, size, p); - JEMALLOC_VALGRIND_MALLOC(true, p, usize, zero); - return (ALLOCM_SUCCESS); + UTRACE(ptr, size, p); + JEMALLOC_VALGRIND_REALLOC(p, usize, ptr, old_usize, old_rzsize, zero); + return (p); label_oom: if (config_xmalloc && opt_xmalloc) { - malloc_write(": Error in allocm(): " - "out of memory\n"); + malloc_write(": Error in rallocx(): out of memory\n"); abort(); } - *ptr = NULL; - UTRACE(0, size, 0); - return (ALLOCM_ERR_OOM); + UTRACE(ptr, size, 0); + return (NULL); } -int -je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) +JEMALLOC_ALWAYS_INLINE_C size_t +ixallocx_helper(void *ptr, size_t old_usize, size_t size, size_t extra, + size_t alignment, bool zero, arena_t *arena) { - void *p, *q; size_t usize; - size_t old_size; - size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + + if (ixalloc(ptr, size, extra, alignment, zero)) + return (old_usize); + usize = isalloc(ptr, config_prof); + + return (usize); +} + +static size_t +ixallocx_prof_sample(void *ptr, size_t old_usize, size_t size, size_t extra, + size_t alignment, size_t max_usize, bool zero, arena_t *arena, + prof_thr_cnt_t *cnt) +{ + size_t usize; + + if (cnt == NULL) + return (old_usize); + /* Use minimum usize to determine whether promotion may happen. */ + if (prof_promote && ((alignment == 0) ? s2u(size) : sa2u(size, + alignment)) <= SMALL_MAXCLASS) { + if (ixalloc(ptr, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= + size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), + alignment, zero)) + return (old_usize); + usize = isalloc(ptr, config_prof); + if (max_usize < PAGE) + arena_prof_promoted(ptr, usize); + } else { + usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, + zero, arena); + } + + return (usize); +} + +JEMALLOC_ALWAYS_INLINE_C size_t +ixallocx_prof(void *ptr, size_t old_usize, size_t size, size_t extra, + size_t alignment, size_t max_usize, bool zero, arena_t *arena, + prof_thr_cnt_t *cnt) +{ + size_t usize; + prof_ctx_t *old_ctx; + + old_ctx = prof_ctx_get(ptr); + if ((uintptr_t)cnt != (uintptr_t)1U) { + usize = ixallocx_prof_sample(ptr, old_usize, size, extra, + alignment, zero, max_usize, arena, cnt); + } else { + usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, + zero, arena); + } + if (usize == old_usize) + return (usize); + prof_realloc(ptr, usize, cnt, old_usize, old_ctx); + + return (usize); +} + +size_t +je_xallocx(void *ptr, size_t size, size_t extra, int flags) +{ + size_t usize, old_usize; + UNUSED size_t old_rzsize JEMALLOC_CC_SILENCE_INIT(0); + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); - bool zero = flags & ALLOCM_ZERO; - bool no_move = flags & ALLOCM_NO_MOVE; + bool zero = flags & MALLOCX_ZERO; unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; - bool try_tcache_alloc, try_tcache_dalloc; arena_t *arena; assert(ptr != NULL); - assert(*ptr != NULL); assert(size != 0); assert(SIZE_T_MAX - size >= extra); assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); - if (arena_ind != UINT_MAX) { - arena_chunk_t *chunk; - try_tcache_alloc = true; - chunk = (arena_chunk_t *)CHUNK_ADDR2BASE(*ptr); - try_tcache_dalloc = (chunk == *ptr || chunk->arena != - arenas[arena_ind]); + if (arena_ind != UINT_MAX) arena = arenas[arena_ind]; - } else { - try_tcache_alloc = true; - try_tcache_dalloc = true; + else arena = NULL; - } - p = *ptr; + old_usize = isalloc(ptr, config_prof); + if (config_valgrind && opt_valgrind) + old_rzsize = u2rz(old_usize); + if (config_prof && opt_prof) { prof_thr_cnt_t *cnt; - /* - * usize isn't knowable before iralloc() returns when extra is + * usize isn't knowable before ixalloc() returns when extra is * non-zero. Therefore, compute its maximum possible value and * use that in PROF_ALLOC_PREP() to decide whether to capture a * backtrace. prof_realloc() will use the actual usize to @@ -1515,111 +1716,51 @@ je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) */ size_t max_usize = (alignment == 0) ? s2u(size+extra) : sa2u(size+extra, alignment); - prof_ctx_t *old_ctx = prof_ctx_get(p); - old_size = isalloc(p, true); - if (config_valgrind && opt_valgrind) - old_rzsize = p2rz(p); PROF_ALLOC_PREP(1, max_usize, cnt); - if (cnt == NULL) - goto label_oom; - /* - * Use minimum usize to determine whether promotion may happen. - */ - if (prof_promote && (uintptr_t)cnt != (uintptr_t)1U - && ((alignment == 0) ? s2u(size) : sa2u(size, alignment)) - <= SMALL_MAXCLASS) { - q = irallocx(p, SMALL_MAXCLASS+1, (SMALL_MAXCLASS+1 >= - size+extra) ? 0 : size+extra - (SMALL_MAXCLASS+1), - alignment, zero, no_move, try_tcache_alloc, - try_tcache_dalloc, arena); - if (q == NULL) - goto label_err; - if (max_usize < PAGE) { - usize = max_usize; - arena_prof_promoted(q, usize); - } else - usize = isalloc(q, config_prof); - } else { - q = irallocx(p, size, extra, alignment, zero, no_move, - try_tcache_alloc, try_tcache_dalloc, arena); - if (q == NULL) - goto label_err; - usize = isalloc(q, config_prof); - } - prof_realloc(q, usize, cnt, old_size, old_ctx); - if (rsize != NULL) - *rsize = usize; + usize = ixallocx_prof(ptr, old_usize, size, extra, alignment, + max_usize, zero, arena, cnt); } else { - if (config_stats) { - old_size = isalloc(p, false); - if (config_valgrind && opt_valgrind) - old_rzsize = u2rz(old_size); - } else if (config_valgrind && opt_valgrind) { - old_size = isalloc(p, false); - old_rzsize = u2rz(old_size); - } - q = irallocx(p, size, extra, alignment, zero, no_move, - try_tcache_alloc, try_tcache_dalloc, arena); - if (q == NULL) - goto label_err; - if (config_stats) - usize = isalloc(q, config_prof); - if (rsize != NULL) { - if (config_stats == false) - usize = isalloc(q, config_prof); - *rsize = usize; - } + usize = ixallocx_helper(ptr, old_usize, size, extra, alignment, + zero, arena); } + if (usize == old_usize) + goto label_not_resized; - *ptr = q; if (config_stats) { thread_allocated_t *ta; ta = thread_allocated_tsd_get(); ta->allocated += usize; - ta->deallocated += old_size; - } - UTRACE(p, size, q); - JEMALLOC_VALGRIND_REALLOC(q, usize, p, old_size, old_rzsize, zero); - return (ALLOCM_SUCCESS); -label_err: - if (no_move) { - UTRACE(p, size, q); - return (ALLOCM_ERR_NOT_MOVED); + ta->deallocated += old_usize; } -label_oom: - if (config_xmalloc && opt_xmalloc) { - malloc_write(": Error in rallocm(): " - "out of memory\n"); - abort(); - } - UTRACE(p, size, 0); - return (ALLOCM_ERR_OOM); + JEMALLOC_VALGRIND_REALLOC(ptr, usize, ptr, old_usize, old_rzsize, zero); +label_not_resized: + UTRACE(ptr, size, ptr); + return (usize); } -int -je_sallocm(const void *ptr, size_t *rsize, int flags) +size_t +je_sallocx(const void *ptr, int flags) { - size_t sz; + size_t usize; assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); if (config_ivsalloc) - sz = ivsalloc(ptr, config_prof); + usize = ivsalloc(ptr, config_prof); else { assert(ptr != NULL); - sz = isalloc(ptr, config_prof); + usize = isalloc(ptr, config_prof); } - assert(rsize != NULL); - *rsize = sz; - return (ALLOCM_SUCCESS); + return (usize); } -int -je_dallocm(void *ptr, int flags) +void +je_dallocx(void *ptr, int flags) { size_t usize; - size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); + UNUSED size_t rzsize JEMALLOC_CC_SILENCE_INIT(0); unsigned arena_ind = ((unsigned)(flags >> 8)) - 1; bool try_tcache; @@ -1645,28 +1786,162 @@ je_dallocm(void *ptr, int flags) thread_allocated_tsd_get()->deallocated += usize; if (config_valgrind && opt_valgrind) rzsize = p2rz(ptr); - iqallocx(ptr, try_tcache); + iqalloct(ptr, try_tcache); JEMALLOC_VALGRIND_FREE(ptr, rzsize); - - return (ALLOCM_SUCCESS); } -int -je_nallocm(size_t *rsize, size_t size, int flags) +size_t +je_nallocx(size_t size, int flags) { size_t usize; - size_t alignment = (ZU(1) << (flags & ALLOCM_LG_ALIGN_MASK) + size_t alignment = (ZU(1) << (flags & MALLOCX_LG_ALIGN_MASK) & (SIZE_T_MAX-1)); assert(size != 0); if (malloc_init()) - return (ALLOCM_ERR_OOM); + return (0); usize = (alignment == 0) ? s2u(size) : sa2u(size, alignment); - if (usize == 0) + assert(usize != 0); + return (usize); +} + +int +je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_byname(name, oldp, oldlenp, newp, newlen)); +} + +int +je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_nametomib(name, mibp, miblenp)); +} + +int +je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) +{ + + if (malloc_init()) + return (EAGAIN); + + return (ctl_bymib(mib, miblen, oldp, oldlenp, newp, newlen)); +} + +void +je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, + const char *opts) +{ + + stats_print(write_cb, cbopaque, opts); +} + +size_t +je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) +{ + size_t ret; + + assert(malloc_initialized || IS_INITIALIZER); + malloc_thread_init(); + + if (config_ivsalloc) + ret = ivsalloc(ptr, config_prof); + else + ret = (ptr != NULL) ? isalloc(ptr, config_prof) : 0; + + return (ret); +} + +/* + * End non-standard functions. + */ +/******************************************************************************/ +/* + * Begin experimental functions. + */ +#ifdef JEMALLOC_EXPERIMENTAL + +int +je_allocm(void **ptr, size_t *rsize, size_t size, int flags) +{ + void *p; + + assert(ptr != NULL); + + p = je_mallocx(size, flags); + if (p == NULL) return (ALLOCM_ERR_OOM); + if (rsize != NULL) + *rsize = isalloc(p, config_prof); + *ptr = p; + return (ALLOCM_SUCCESS); +} + +int +je_rallocm(void **ptr, size_t *rsize, size_t size, size_t extra, int flags) +{ + int ret; + bool no_move = flags & ALLOCM_NO_MOVE; + assert(ptr != NULL); + assert(*ptr != NULL); + assert(size != 0); + assert(SIZE_T_MAX - size >= extra); + + if (no_move) { + size_t usize = je_xallocx(*ptr, size, extra, flags); + ret = (usize >= size) ? ALLOCM_SUCCESS : ALLOCM_ERR_NOT_MOVED; + if (rsize != NULL) + *rsize = usize; + } else { + void *p = je_rallocx(*ptr, size+extra, flags); + if (p != NULL) { + *ptr = p; + ret = ALLOCM_SUCCESS; + } else + ret = ALLOCM_ERR_OOM; + if (rsize != NULL) + *rsize = isalloc(*ptr, config_prof); + } + return (ret); +} + +int +je_sallocm(const void *ptr, size_t *rsize, int flags) +{ + + assert(rsize != NULL); + *rsize = je_sallocx(ptr, flags); + return (ALLOCM_SUCCESS); +} + +int +je_dallocm(void *ptr, int flags) +{ + + je_dallocx(ptr, flags); + return (ALLOCM_SUCCESS); +} + +int +je_nallocm(size_t *rsize, size_t size, int flags) +{ + size_t usize; + + usize = je_nallocx(size, flags); + if (usize == 0) + return (ALLOCM_ERR_OOM); if (rsize != NULL) *rsize = usize; return (ALLOCM_SUCCESS); @@ -1721,12 +1996,12 @@ _malloc_prefork(void) /* Acquire all mutexes in a safe order. */ ctl_prefork(); + prof_prefork(); malloc_mutex_prefork(&arenas_lock); for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) arena_prefork(arenas[i]); } - prof_prefork(); chunk_prefork(); base_prefork(); huge_prefork(); @@ -1752,12 +2027,12 @@ _malloc_postfork(void) huge_postfork_parent(); base_postfork_parent(); chunk_postfork_parent(); - prof_postfork_parent(); for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) arena_postfork_parent(arenas[i]); } malloc_mutex_postfork_parent(&arenas_lock); + prof_postfork_parent(); ctl_postfork_parent(); } @@ -1772,12 +2047,12 @@ jemalloc_postfork_child(void) huge_postfork_child(); base_postfork_child(); chunk_postfork_child(); - prof_postfork_child(); for (i = 0; i < narenas_total; i++) { if (arenas[i] != NULL) arena_postfork_child(arenas[i]); } malloc_mutex_postfork_child(&arenas_lock); + prof_postfork_child(); ctl_postfork_child(); } @@ -1801,7 +2076,7 @@ a0alloc(size_t size, bool zero) if (size <= arena_maxclass) return (arena_malloc(arenas[0], size, zero, false)); else - return (huge_malloc(size, zero)); + return (huge_malloc(size, zero, huge_dss_prec_get(arenas[0]))); } void * diff --git a/deps/jemalloc/src/mutex.c b/deps/jemalloc/src/mutex.c index 55e18c23713..788eca38703 100644 --- a/deps/jemalloc/src/mutex.c +++ b/deps/jemalloc/src/mutex.c @@ -6,7 +6,7 @@ #endif #ifndef _CRT_SPINCOUNT -#define _CRT_SPINCOUNT 4000 +#define _CRT_SPINCOUNT 4000 #endif /******************************************************************************/ diff --git a/deps/jemalloc/src/prof.c b/deps/jemalloc/src/prof.c index 04964ef7ca3..7722b7b4373 100644 --- a/deps/jemalloc/src/prof.c +++ b/deps/jemalloc/src/prof.c @@ -24,9 +24,14 @@ bool opt_prof_gdump = false; bool opt_prof_final = true; bool opt_prof_leak = false; bool opt_prof_accum = false; -char opt_prof_prefix[PATH_MAX + 1]; +char opt_prof_prefix[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; -uint64_t prof_interval; +uint64_t prof_interval = 0; bool prof_promote; /* @@ -54,47 +59,23 @@ static uint64_t prof_dump_useq; /* * This buffer is rather large for stack allocation, so use a single buffer for - * all profile dumps. The buffer is implicitly protected by bt2ctx_mtx, since - * it must be locked anyway during dumping. + * all profile dumps. */ -static char prof_dump_buf[PROF_DUMP_BUFSIZE]; +static malloc_mutex_t prof_dump_mtx; +static char prof_dump_buf[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PROF_DUMP_BUFSIZE +#else + 1 +#endif +]; static unsigned prof_dump_buf_end; static int prof_dump_fd; /* Do not dump any profiles until bootstrapping is complete. */ static bool prof_booted = false; -/******************************************************************************/ -/* Function prototypes for non-inline static functions. */ - -static prof_bt_t *bt_dup(prof_bt_t *bt); -static void bt_destroy(prof_bt_t *bt); -#ifdef JEMALLOC_PROF_LIBGCC -static _Unwind_Reason_Code prof_unwind_init_callback( - struct _Unwind_Context *context, void *arg); -static _Unwind_Reason_Code prof_unwind_callback( - struct _Unwind_Context *context, void *arg); -#endif -static bool prof_flush(bool propagate_err); -static bool prof_write(bool propagate_err, const char *s); -static bool prof_printf(bool propagate_err, const char *format, ...) - JEMALLOC_ATTR(format(printf, 2, 3)); -static void prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, - size_t *leak_nctx); -static void prof_ctx_destroy(prof_ctx_t *ctx); -static void prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt); -static bool prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, - prof_bt_t *bt); -static bool prof_dump_maps(bool propagate_err); -static bool prof_dump(bool propagate_err, const char *filename, - bool leakcheck); -static void prof_dump_filename(char *filename, char v, int64_t vseq); -static void prof_fdump(void); -static void prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, - size_t *hash2); -static bool prof_bt_keycomp(const void *k1, const void *k2); -static malloc_mutex_t *prof_ctx_mutex_choose(void); - /******************************************************************************/ void @@ -424,10 +405,169 @@ prof_backtrace(prof_bt_t *bt, unsigned nignore) { cassert(config_prof); - assert(false); + not_reached(); } #endif +static malloc_mutex_t * +prof_ctx_mutex_choose(void) +{ + unsigned nctxs = atomic_add_u(&cum_ctxs, 1); + + return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); +} + +static void +prof_ctx_init(prof_ctx_t *ctx, prof_bt_t *bt) +{ + + ctx->bt = bt; + ctx->lock = prof_ctx_mutex_choose(); + /* + * Set nlimbo to 1, in order to avoid a race condition with + * prof_ctx_merge()/prof_ctx_destroy(). + */ + ctx->nlimbo = 1; + ql_elm_new(ctx, dump_link); + memset(&ctx->cnt_merged, 0, sizeof(prof_cnt_t)); + ql_new(&ctx->cnts_ql); +} + +static void +prof_ctx_destroy(prof_ctx_t *ctx) +{ + prof_tdata_t *prof_tdata; + + cassert(config_prof); + + /* + * Check that ctx is still unused by any thread cache before destroying + * it. prof_lookup() increments ctx->nlimbo in order to avoid a race + * condition with this function, as does prof_ctx_merge() in order to + * avoid a race between the main body of prof_ctx_merge() and entry + * into this function. + */ + prof_tdata = prof_tdata_get(false); + assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX); + prof_enter(prof_tdata); + malloc_mutex_lock(ctx->lock); + if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 0 && + ctx->nlimbo == 1) { + assert(ctx->cnt_merged.curbytes == 0); + assert(ctx->cnt_merged.accumobjs == 0); + assert(ctx->cnt_merged.accumbytes == 0); + /* Remove ctx from bt2ctx. */ + if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) + not_reached(); + prof_leave(prof_tdata); + /* Destroy ctx. */ + malloc_mutex_unlock(ctx->lock); + bt_destroy(ctx->bt); + idalloc(ctx); + } else { + /* + * Compensate for increment in prof_ctx_merge() or + * prof_lookup(). + */ + ctx->nlimbo--; + malloc_mutex_unlock(ctx->lock); + prof_leave(prof_tdata); + } +} + +static void +prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) +{ + bool destroy; + + cassert(config_prof); + + /* Merge cnt stats and detach from ctx. */ + malloc_mutex_lock(ctx->lock); + ctx->cnt_merged.curobjs += cnt->cnts.curobjs; + ctx->cnt_merged.curbytes += cnt->cnts.curbytes; + ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; + ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; + ql_remove(&ctx->cnts_ql, cnt, cnts_link); + if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && + ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { + /* + * Increment ctx->nlimbo in order to keep another thread from + * winning the race to destroy ctx while this one has ctx->lock + * dropped. Without this, it would be possible for another + * thread to: + * + * 1) Sample an allocation associated with ctx. + * 2) Deallocate the sampled object. + * 3) Successfully prof_ctx_destroy(ctx). + * + * The result would be that ctx no longer exists by the time + * this thread accesses it in prof_ctx_destroy(). + */ + ctx->nlimbo++; + destroy = true; + } else + destroy = false; + malloc_mutex_unlock(ctx->lock); + if (destroy) + prof_ctx_destroy(ctx); +} + +static bool +prof_lookup_global(prof_bt_t *bt, prof_tdata_t *prof_tdata, void **p_btkey, + prof_ctx_t **p_ctx, bool *p_new_ctx) +{ + union { + prof_ctx_t *p; + void *v; + } ctx; + union { + prof_bt_t *p; + void *v; + } btkey; + bool new_ctx; + + prof_enter(prof_tdata); + if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { + /* bt has never been seen before. Insert it. */ + ctx.v = imalloc(sizeof(prof_ctx_t)); + if (ctx.v == NULL) { + prof_leave(prof_tdata); + return (true); + } + btkey.p = bt_dup(bt); + if (btkey.v == NULL) { + prof_leave(prof_tdata); + idalloc(ctx.v); + return (true); + } + prof_ctx_init(ctx.p, btkey.p); + if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { + /* OOM. */ + prof_leave(prof_tdata); + idalloc(btkey.v); + idalloc(ctx.v); + return (true); + } + new_ctx = true; + } else { + /* + * Increment nlimbo, in order to avoid a race condition with + * prof_ctx_merge()/prof_ctx_destroy(). + */ + malloc_mutex_lock(ctx.p->lock); + ctx.p->nlimbo++; + malloc_mutex_unlock(ctx.p->lock); + new_ctx = false; + } + prof_leave(prof_tdata); + + *p_btkey = btkey.v; + *p_ctx = ctx.p; + *p_new_ctx = new_ctx; + return (false); +} + prof_thr_cnt_t * prof_lookup(prof_bt_t *bt) { @@ -439,67 +579,21 @@ prof_lookup(prof_bt_t *bt) cassert(config_prof); - prof_tdata = prof_tdata_get(); + prof_tdata = prof_tdata_get(false); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (NULL); if (ckh_search(&prof_tdata->bt2cnt, bt, NULL, &ret.v)) { - union { - prof_bt_t *p; - void *v; - } btkey; - union { - prof_ctx_t *p; - void *v; - } ctx; + void *btkey; + prof_ctx_t *ctx; bool new_ctx; /* * This thread's cache lacks bt. Look for it in the global * cache. */ - prof_enter(prof_tdata); - if (ckh_search(&bt2ctx, bt, &btkey.v, &ctx.v)) { - /* bt has never been seen before. Insert it. */ - ctx.v = imalloc(sizeof(prof_ctx_t)); - if (ctx.v == NULL) { - prof_leave(prof_tdata); - return (NULL); - } - btkey.p = bt_dup(bt); - if (btkey.v == NULL) { - prof_leave(prof_tdata); - idalloc(ctx.v); - return (NULL); - } - ctx.p->bt = btkey.p; - ctx.p->lock = prof_ctx_mutex_choose(); - /* - * Set nlimbo to 1, in order to avoid a race condition - * with prof_ctx_merge()/prof_ctx_destroy(). - */ - ctx.p->nlimbo = 1; - memset(&ctx.p->cnt_merged, 0, sizeof(prof_cnt_t)); - ql_new(&ctx.p->cnts_ql); - if (ckh_insert(&bt2ctx, btkey.v, ctx.v)) { - /* OOM. */ - prof_leave(prof_tdata); - idalloc(btkey.v); - idalloc(ctx.v); - return (NULL); - } - new_ctx = true; - } else { - /* - * Increment nlimbo, in order to avoid a race condition - * with prof_ctx_merge()/prof_ctx_destroy(). - */ - malloc_mutex_lock(ctx.p->lock); - ctx.p->nlimbo++; - malloc_mutex_unlock(ctx.p->lock); - new_ctx = false; - } - prof_leave(prof_tdata); + if (prof_lookup_global(bt, prof_tdata, &btkey, &ctx, &new_ctx)) + return (NULL); /* Link a prof_thd_cnt_t into ctx for this thread. */ if (ckh_count(&prof_tdata->bt2cnt) == PROF_TCMAX) { @@ -512,7 +606,7 @@ prof_lookup(prof_bt_t *bt) assert(ret.v != NULL); if (ckh_remove(&prof_tdata->bt2cnt, ret.p->ctx->bt, NULL, NULL)) - assert(false); + not_reached(); ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); prof_ctx_merge(ret.p->ctx, ret.p); /* ret can now be re-used. */ @@ -522,27 +616,27 @@ prof_lookup(prof_bt_t *bt) ret.v = imalloc(sizeof(prof_thr_cnt_t)); if (ret.p == NULL) { if (new_ctx) - prof_ctx_destroy(ctx.p); + prof_ctx_destroy(ctx); return (NULL); } ql_elm_new(ret.p, cnts_link); ql_elm_new(ret.p, lru_link); } /* Finish initializing ret. */ - ret.p->ctx = ctx.p; + ret.p->ctx = ctx; ret.p->epoch = 0; memset(&ret.p->cnts, 0, sizeof(prof_cnt_t)); - if (ckh_insert(&prof_tdata->bt2cnt, btkey.v, ret.v)) { + if (ckh_insert(&prof_tdata->bt2cnt, btkey, ret.v)) { if (new_ctx) - prof_ctx_destroy(ctx.p); + prof_ctx_destroy(ctx); idalloc(ret.v); return (NULL); } ql_head_insert(&prof_tdata->lru_ql, ret.p, lru_link); - malloc_mutex_lock(ctx.p->lock); - ql_tail_insert(&ctx.p->cnts_ql, ret.p, cnts_link); - ctx.p->nlimbo--; - malloc_mutex_unlock(ctx.p->lock); + malloc_mutex_lock(ctx->lock); + ql_tail_insert(&ctx->cnts_ql, ret.p, cnts_link); + ctx->nlimbo--; + malloc_mutex_unlock(ctx->lock); } else { /* Move ret to the front of the LRU. */ ql_remove(&prof_tdata->lru_ql, ret.p, lru_link); @@ -552,8 +646,52 @@ prof_lookup(prof_bt_t *bt) return (ret.p); } +#ifdef JEMALLOC_JET +size_t +prof_bt_count(void) +{ + size_t bt_count; + prof_tdata_t *prof_tdata; + + prof_tdata = prof_tdata_get(false); + if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) + return (0); + + prof_enter(prof_tdata); + bt_count = ckh_count(&bt2ctx); + prof_leave(prof_tdata); + + return (bt_count); +} +#endif + +#ifdef JEMALLOC_JET +#undef prof_dump_open +#define prof_dump_open JEMALLOC_N(prof_dump_open_impl) +#endif +static int +prof_dump_open(bool propagate_err, const char *filename) +{ + int fd; + + fd = creat(filename, 0644); + if (fd == -1 && propagate_err == false) { + malloc_printf(": creat(\"%s\"), 0644) failed\n", + filename); + if (opt_abort) + abort(); + } + + return (fd); +} +#ifdef JEMALLOC_JET +#undef prof_dump_open +#define prof_dump_open JEMALLOC_N(prof_dump_open) +prof_dump_open_t *prof_dump_open = JEMALLOC_N(prof_dump_open_impl); +#endif + static bool -prof_flush(bool propagate_err) +prof_dump_flush(bool propagate_err) { bool ret = false; ssize_t err; @@ -576,7 +714,20 @@ prof_flush(bool propagate_err) } static bool -prof_write(bool propagate_err, const char *s) +prof_dump_close(bool propagate_err) +{ + bool ret; + + assert(prof_dump_fd != -1); + ret = prof_dump_flush(propagate_err); + close(prof_dump_fd); + prof_dump_fd = -1; + + return (ret); +} + +static bool +prof_dump_write(bool propagate_err, const char *s) { unsigned i, slen, n; @@ -587,7 +738,7 @@ prof_write(bool propagate_err, const char *s) while (i < slen) { /* Flush the buffer if it is full. */ if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) - if (prof_flush(propagate_err) && propagate_err) + if (prof_dump_flush(propagate_err) && propagate_err) return (true); if (prof_dump_buf_end + slen <= PROF_DUMP_BUFSIZE) { @@ -607,7 +758,7 @@ prof_write(bool propagate_err, const char *s) JEMALLOC_ATTR(format(printf, 2, 3)) static bool -prof_printf(bool propagate_err, const char *format, ...) +prof_dump_printf(bool propagate_err, const char *format, ...) { bool ret; va_list ap; @@ -616,13 +767,14 @@ prof_printf(bool propagate_err, const char *format, ...) va_start(ap, format); malloc_vsnprintf(buf, sizeof(buf), format, ap); va_end(ap); - ret = prof_write(propagate_err, buf); + ret = prof_dump_write(propagate_err, buf); return (ret); } static void -prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) +prof_dump_ctx_prep(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx, + prof_ctx_list_t *ctx_ql) { prof_thr_cnt_t *thr_cnt; prof_cnt_t tcnt; @@ -631,6 +783,14 @@ prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) malloc_mutex_lock(ctx->lock); + /* + * Increment nlimbo so that ctx won't go away before dump. + * Additionally, link ctx into the dump list so that it is included in + * prof_dump()'s second pass. + */ + ctx->nlimbo++; + ql_tail_insert(ctx_ql, ctx, dump_link); + memcpy(&ctx->cnt_summed, &ctx->cnt_merged, sizeof(prof_cnt_t)); ql_foreach(thr_cnt, &ctx->cnts_ql, cnts_link) { volatile unsigned *epoch = &thr_cnt->epoch; @@ -671,89 +831,52 @@ prof_ctx_sum(prof_ctx_t *ctx, prof_cnt_t *cnt_all, size_t *leak_nctx) malloc_mutex_unlock(ctx->lock); } -static void -prof_ctx_destroy(prof_ctx_t *ctx) +static bool +prof_dump_header(bool propagate_err, const prof_cnt_t *cnt_all) { - prof_tdata_t *prof_tdata; - - cassert(config_prof); - /* - * Check that ctx is still unused by any thread cache before destroying - * it. prof_lookup() increments ctx->nlimbo in order to avoid a race - * condition with this function, as does prof_ctx_merge() in order to - * avoid a race between the main body of prof_ctx_merge() and entry - * into this function. - */ - prof_tdata = *prof_tdata_tsd_get(); - assert((uintptr_t)prof_tdata > (uintptr_t)PROF_TDATA_STATE_MAX); - prof_enter(prof_tdata); - malloc_mutex_lock(ctx->lock); - if (ql_first(&ctx->cnts_ql) == NULL && ctx->cnt_merged.curobjs == 0 && - ctx->nlimbo == 1) { - assert(ctx->cnt_merged.curbytes == 0); - assert(ctx->cnt_merged.accumobjs == 0); - assert(ctx->cnt_merged.accumbytes == 0); - /* Remove ctx from bt2ctx. */ - if (ckh_remove(&bt2ctx, ctx->bt, NULL, NULL)) - assert(false); - prof_leave(prof_tdata); - /* Destroy ctx. */ - malloc_mutex_unlock(ctx->lock); - bt_destroy(ctx->bt); - idalloc(ctx); + if (opt_lg_prof_sample == 0) { + if (prof_dump_printf(propagate_err, + "heap profile: %"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @ heapprofile\n", + cnt_all->curobjs, cnt_all->curbytes, + cnt_all->accumobjs, cnt_all->accumbytes)) + return (true); } else { - /* - * Compensate for increment in prof_ctx_merge() or - * prof_lookup(). - */ - ctx->nlimbo--; - malloc_mutex_unlock(ctx->lock); - prof_leave(prof_tdata); + if (prof_dump_printf(propagate_err, + "heap profile: %"PRId64": %"PRId64 + " [%"PRIu64": %"PRIu64"] @ heap_v2/%"PRIu64"\n", + cnt_all->curobjs, cnt_all->curbytes, + cnt_all->accumobjs, cnt_all->accumbytes, + ((uint64_t)1U << opt_lg_prof_sample))) + return (true); } + + return (false); } static void -prof_ctx_merge(prof_ctx_t *ctx, prof_thr_cnt_t *cnt) +prof_dump_ctx_cleanup_locked(prof_ctx_t *ctx, prof_ctx_list_t *ctx_ql) { - bool destroy; - cassert(config_prof); + ctx->nlimbo--; + ql_remove(ctx_ql, ctx, dump_link); +} + +static void +prof_dump_ctx_cleanup(prof_ctx_t *ctx, prof_ctx_list_t *ctx_ql) +{ - /* Merge cnt stats and detach from ctx. */ malloc_mutex_lock(ctx->lock); - ctx->cnt_merged.curobjs += cnt->cnts.curobjs; - ctx->cnt_merged.curbytes += cnt->cnts.curbytes; - ctx->cnt_merged.accumobjs += cnt->cnts.accumobjs; - ctx->cnt_merged.accumbytes += cnt->cnts.accumbytes; - ql_remove(&ctx->cnts_ql, cnt, cnts_link); - if (opt_prof_accum == false && ql_first(&ctx->cnts_ql) == NULL && - ctx->cnt_merged.curobjs == 0 && ctx->nlimbo == 0) { - /* - * Increment ctx->nlimbo in order to keep another thread from - * winning the race to destroy ctx while this one has ctx->lock - * dropped. Without this, it would be possible for another - * thread to: - * - * 1) Sample an allocation associated with ctx. - * 2) Deallocate the sampled object. - * 3) Successfully prof_ctx_destroy(ctx). - * - * The result would be that ctx no longer exists by the time - * this thread accesses it in prof_ctx_destroy(). - */ - ctx->nlimbo++; - destroy = true; - } else - destroy = false; + prof_dump_ctx_cleanup_locked(ctx, ctx_ql); malloc_mutex_unlock(ctx->lock); - if (destroy) - prof_ctx_destroy(ctx); } static bool -prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, prof_bt_t *bt) +prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, const prof_bt_t *bt, + prof_ctx_list_t *ctx_ql) { + bool ret; unsigned i; cassert(config_prof); @@ -765,66 +888,109 @@ prof_dump_ctx(bool propagate_err, prof_ctx_t *ctx, prof_bt_t *bt) * filled in. Avoid dumping any ctx that is an artifact of either * implementation detail. */ + malloc_mutex_lock(ctx->lock); if ((opt_prof_accum == false && ctx->cnt_summed.curobjs == 0) || (opt_prof_accum && ctx->cnt_summed.accumobjs == 0)) { assert(ctx->cnt_summed.curobjs == 0); assert(ctx->cnt_summed.curbytes == 0); assert(ctx->cnt_summed.accumobjs == 0); assert(ctx->cnt_summed.accumbytes == 0); - return (false); + ret = false; + goto label_return; } - if (prof_printf(propagate_err, "%"PRId64": %"PRId64 + if (prof_dump_printf(propagate_err, "%"PRId64": %"PRId64 " [%"PRIu64": %"PRIu64"] @", ctx->cnt_summed.curobjs, ctx->cnt_summed.curbytes, - ctx->cnt_summed.accumobjs, ctx->cnt_summed.accumbytes)) - return (true); + ctx->cnt_summed.accumobjs, ctx->cnt_summed.accumbytes)) { + ret = true; + goto label_return; + } for (i = 0; i < bt->len; i++) { - if (prof_printf(propagate_err, " %#"PRIxPTR, - (uintptr_t)bt->vec[i])) - return (true); + if (prof_dump_printf(propagate_err, " %#"PRIxPTR, + (uintptr_t)bt->vec[i])) { + ret = true; + goto label_return; + } } - if (prof_write(propagate_err, "\n")) - return (true); + if (prof_dump_write(propagate_err, "\n")) { + ret = true; + goto label_return; + } - return (false); + ret = false; +label_return: + prof_dump_ctx_cleanup_locked(ctx, ctx_ql); + malloc_mutex_unlock(ctx->lock); + return (ret); } static bool prof_dump_maps(bool propagate_err) { + bool ret; int mfd; char filename[PATH_MAX + 1]; cassert(config_prof); - +#ifdef __FreeBSD__ + malloc_snprintf(filename, sizeof(filename), "/proc/curproc/map"); +#else malloc_snprintf(filename, sizeof(filename), "/proc/%d/maps", (int)getpid()); +#endif mfd = open(filename, O_RDONLY); if (mfd != -1) { ssize_t nread; - if (prof_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && - propagate_err) - return (true); + if (prof_dump_write(propagate_err, "\nMAPPED_LIBRARIES:\n") && + propagate_err) { + ret = true; + goto label_return; + } nread = 0; do { prof_dump_buf_end += nread; if (prof_dump_buf_end == PROF_DUMP_BUFSIZE) { /* Make space in prof_dump_buf before read(). */ - if (prof_flush(propagate_err) && propagate_err) - return (true); + if (prof_dump_flush(propagate_err) && + propagate_err) { + ret = true; + goto label_return; + } } nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE - prof_dump_buf_end); } while (nread > 0); + } else { + ret = true; + goto label_return; + } + + ret = false; +label_return: + if (mfd != -1) close(mfd); - } else - return (true); + return (ret); +} - return (false); +static void +prof_leakcheck(const prof_cnt_t *cnt_all, size_t leak_nctx, + const char *filename) +{ + + if (cnt_all->curbytes != 0) { + malloc_printf(": Leak summary: %"PRId64" byte%s, %" + PRId64" object%s, %zu context%s\n", + cnt_all->curbytes, (cnt_all->curbytes != 1) ? "s" : "", + cnt_all->curobjs, (cnt_all->curobjs != 1) ? "s" : "", + leak_nctx, (leak_nctx != 1) ? "s" : ""); + malloc_printf( + ": Run pprof on \"%s\" for leak detail\n", + filename); + } } static bool @@ -833,99 +999,75 @@ prof_dump(bool propagate_err, const char *filename, bool leakcheck) prof_tdata_t *prof_tdata; prof_cnt_t cnt_all; size_t tabind; - union { - prof_bt_t *p; - void *v; - } bt; union { prof_ctx_t *p; void *v; } ctx; size_t leak_nctx; + prof_ctx_list_t ctx_ql; cassert(config_prof); - prof_tdata = prof_tdata_get(); + prof_tdata = prof_tdata_get(false); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return (true); - prof_enter(prof_tdata); - prof_dump_fd = creat(filename, 0644); - if (prof_dump_fd == -1) { - if (propagate_err == false) { - malloc_printf( - ": creat(\"%s\"), 0644) failed\n", - filename); - if (opt_abort) - abort(); - } - goto label_error; - } + + malloc_mutex_lock(&prof_dump_mtx); /* Merge per thread profile stats, and sum them in cnt_all. */ memset(&cnt_all, 0, sizeof(prof_cnt_t)); leak_nctx = 0; + ql_new(&ctx_ql); + prof_enter(prof_tdata); for (tabind = 0; ckh_iter(&bt2ctx, &tabind, NULL, &ctx.v) == false;) - prof_ctx_sum(ctx.p, &cnt_all, &leak_nctx); + prof_dump_ctx_prep(ctx.p, &cnt_all, &leak_nctx, &ctx_ql); + prof_leave(prof_tdata); + + /* Create dump file. */ + if ((prof_dump_fd = prof_dump_open(propagate_err, filename)) == -1) + goto label_open_close_error; /* Dump profile header. */ - if (opt_lg_prof_sample == 0) { - if (prof_printf(propagate_err, - "heap profile: %"PRId64": %"PRId64 - " [%"PRIu64": %"PRIu64"] @ heapprofile\n", - cnt_all.curobjs, cnt_all.curbytes, - cnt_all.accumobjs, cnt_all.accumbytes)) - goto label_error; - } else { - if (prof_printf(propagate_err, - "heap profile: %"PRId64": %"PRId64 - " [%"PRIu64": %"PRIu64"] @ heap_v2/%"PRIu64"\n", - cnt_all.curobjs, cnt_all.curbytes, - cnt_all.accumobjs, cnt_all.accumbytes, - ((uint64_t)1U << opt_lg_prof_sample))) - goto label_error; - } + if (prof_dump_header(propagate_err, &cnt_all)) + goto label_write_error; - /* Dump per ctx profile stats. */ - for (tabind = 0; ckh_iter(&bt2ctx, &tabind, &bt.v, &ctx.v) - == false;) { - if (prof_dump_ctx(propagate_err, ctx.p, bt.p)) - goto label_error; + /* Dump per ctx profile stats. */ + while ((ctx.p = ql_first(&ctx_ql)) != NULL) { + if (prof_dump_ctx(propagate_err, ctx.p, ctx.p->bt, &ctx_ql)) + goto label_write_error; } /* Dump /proc//maps if possible. */ if (prof_dump_maps(propagate_err)) - goto label_error; + goto label_write_error; - if (prof_flush(propagate_err)) - goto label_error; - close(prof_dump_fd); - prof_leave(prof_tdata); + if (prof_dump_close(propagate_err)) + goto label_open_close_error; - if (leakcheck && cnt_all.curbytes != 0) { - malloc_printf(": Leak summary: %"PRId64" byte%s, %" - PRId64" object%s, %zu context%s\n", - cnt_all.curbytes, (cnt_all.curbytes != 1) ? "s" : "", - cnt_all.curobjs, (cnt_all.curobjs != 1) ? "s" : "", - leak_nctx, (leak_nctx != 1) ? "s" : ""); - malloc_printf( - ": Run pprof on \"%s\" for leak detail\n", - filename); - } + malloc_mutex_unlock(&prof_dump_mtx); + + if (leakcheck) + prof_leakcheck(&cnt_all, leak_nctx, filename); return (false); -label_error: - prof_leave(prof_tdata); +label_write_error: + prof_dump_close(propagate_err); +label_open_close_error: + while ((ctx.p = ql_first(&ctx_ql)) != NULL) + prof_dump_ctx_cleanup(ctx.p, &ctx_ql); + malloc_mutex_unlock(&prof_dump_mtx); return (true); } #define DUMP_FILENAME_BUFSIZE (PATH_MAX + 1) +#define VSEQ_INVALID UINT64_C(0xffffffffffffffff) static void prof_dump_filename(char *filename, char v, int64_t vseq) { cassert(config_prof); - if (vseq != UINT64_C(0xffffffffffffffff)) { + if (vseq != VSEQ_INVALID) { /* "...v.heap" */ malloc_snprintf(filename, DUMP_FILENAME_BUFSIZE, "%s.%d.%"PRIu64".%c%"PRId64".heap", @@ -951,7 +1093,7 @@ prof_fdump(void) if (opt_prof_final && opt_prof_prefix[0] != '\0') { malloc_mutex_lock(&prof_dump_seq_mtx); - prof_dump_filename(filename, 'f', UINT64_C(0xffffffffffffffff)); + prof_dump_filename(filename, 'f', VSEQ_INVALID); malloc_mutex_unlock(&prof_dump_seq_mtx); prof_dump(false, filename, opt_prof_leak); } @@ -967,11 +1109,7 @@ prof_idump(void) if (prof_booted == false) return; - /* - * Don't call prof_tdata_get() here, because it could cause recursive - * allocation. - */ - prof_tdata = *prof_tdata_tsd_get(); + prof_tdata = prof_tdata_get(false); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return; if (prof_tdata->enq) { @@ -1021,11 +1159,7 @@ prof_gdump(void) if (prof_booted == false) return; - /* - * Don't call prof_tdata_get() here, because it could cause recursive - * allocation. - */ - prof_tdata = *prof_tdata_tsd_get(); + prof_tdata = prof_tdata_get(false); if ((uintptr_t)prof_tdata <= (uintptr_t)PROF_TDATA_STATE_MAX) return; if (prof_tdata->enq) { @@ -1043,34 +1177,13 @@ prof_gdump(void) } static void -prof_bt_hash(const void *key, unsigned minbits, size_t *hash1, size_t *hash2) +prof_bt_hash(const void *key, size_t r_hash[2]) { - size_t ret1, ret2; - uint64_t h; prof_bt_t *bt = (prof_bt_t *)key; cassert(config_prof); - assert(minbits <= 32 || (SIZEOF_PTR == 8 && minbits <= 64)); - assert(hash1 != NULL); - assert(hash2 != NULL); - h = hash(bt->vec, bt->len * sizeof(void *), - UINT64_C(0x94122f335b332aea)); - if (minbits <= 32) { - /* - * Avoid doing multiple hashes, since a single hash provides - * enough bits. - */ - ret1 = h & ZU(0xffffffffU); - ret2 = h >> 32; - } else { - ret1 = h; - ret2 = hash(bt->vec, bt->len * sizeof(void *), - UINT64_C(0x8432a476666bbc13)); - } - - *hash1 = ret1; - *hash2 = ret2; + hash(bt->vec, bt->len * sizeof(void *), 0x94122f33U, r_hash); } static bool @@ -1086,14 +1199,6 @@ prof_bt_keycomp(const void *k1, const void *k2) return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } -static malloc_mutex_t * -prof_ctx_mutex_choose(void) -{ - unsigned nctxs = atomic_add_u(&cum_ctxs, 1); - - return (&ctx_locks[(nctxs - 1) % PROF_NCTX_LOCKS]); -} - prof_tdata_t * prof_tdata_init(void) { @@ -1206,13 +1311,11 @@ prof_boot1(void) */ opt_prof = true; opt_prof_gdump = false; - prof_interval = 0; } else if (opt_prof) { if (opt_lg_prof_interval >= 0) { prof_interval = (((uint64_t)1U) << opt_lg_prof_interval); - } else - prof_interval = 0; + } } prof_promote = (opt_prof && opt_lg_prof_sample > LG_PAGE); @@ -1240,6 +1343,8 @@ prof_boot2(void) if (malloc_mutex_init(&prof_dump_seq_mtx)) return (true); + if (malloc_mutex_init(&prof_dump_mtx)) + return (true); if (atexit(prof_fdump) != 0) { malloc_write(": Error in atexit()\n"); @@ -1277,10 +1382,10 @@ prof_prefork(void) if (opt_prof) { unsigned i; - malloc_mutex_lock(&bt2ctx_mtx); - malloc_mutex_lock(&prof_dump_seq_mtx); + malloc_mutex_prefork(&bt2ctx_mtx); + malloc_mutex_prefork(&prof_dump_seq_mtx); for (i = 0; i < PROF_NCTX_LOCKS; i++) - malloc_mutex_lock(&ctx_locks[i]); + malloc_mutex_prefork(&ctx_locks[i]); } } diff --git a/deps/jemalloc/src/quarantine.c b/deps/jemalloc/src/quarantine.c index 9005ab3ba06..5431511640a 100644 --- a/deps/jemalloc/src/quarantine.c +++ b/deps/jemalloc/src/quarantine.c @@ -1,3 +1,4 @@ +#define JEMALLOC_QUARANTINE_C_ #include "jemalloc/internal/jemalloc_internal.h" /* @@ -11,39 +12,18 @@ /******************************************************************************/ /* Data. */ -typedef struct quarantine_obj_s quarantine_obj_t; -typedef struct quarantine_s quarantine_t; - -struct quarantine_obj_s { - void *ptr; - size_t usize; -}; - -struct quarantine_s { - size_t curbytes; - size_t curobjs; - size_t first; -#define LG_MAXOBJS_INIT 10 - size_t lg_maxobjs; - quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */ -}; - -static void quarantine_cleanup(void *arg); - -malloc_tsd_data(static, quarantine, quarantine_t *, NULL) -malloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL, - quarantine_cleanup) +malloc_tsd_data(, quarantine, quarantine_t *, NULL) /******************************************************************************/ /* Function prototypes for non-inline static functions. */ -static quarantine_t *quarantine_init(size_t lg_maxobjs); static quarantine_t *quarantine_grow(quarantine_t *quarantine); +static void quarantine_drain_one(quarantine_t *quarantine); static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); /******************************************************************************/ -static quarantine_t * +quarantine_t * quarantine_init(size_t lg_maxobjs) { quarantine_t *quarantine; @@ -68,8 +48,10 @@ quarantine_grow(quarantine_t *quarantine) quarantine_t *ret; ret = quarantine_init(quarantine->lg_maxobjs + 1); - if (ret == NULL) + if (ret == NULL) { + quarantine_drain_one(quarantine); return (quarantine); + } ret->curbytes = quarantine->curbytes; ret->curobjs = quarantine->curobjs; @@ -89,23 +71,29 @@ quarantine_grow(quarantine_t *quarantine) memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * sizeof(quarantine_obj_t)); } + idalloc(quarantine); return (ret); } +static void +quarantine_drain_one(quarantine_t *quarantine) +{ + quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; + assert(obj->usize == isalloc(obj->ptr, config_prof)); + idalloc(obj->ptr); + quarantine->curbytes -= obj->usize; + quarantine->curobjs--; + quarantine->first = (quarantine->first + 1) & ((ZU(1) << + quarantine->lg_maxobjs) - 1); +} + static void quarantine_drain(quarantine_t *quarantine, size_t upper_bound) { - while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) { - quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; - assert(obj->usize == isalloc(obj->ptr, config_prof)); - idalloc(obj->ptr); - quarantine->curbytes -= obj->usize; - quarantine->curobjs--; - quarantine->first = (quarantine->first + 1) & ((ZU(1) << - quarantine->lg_maxobjs) - 1); - } + while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) + quarantine_drain_one(quarantine); } void @@ -119,24 +107,16 @@ quarantine(void *ptr) quarantine = *quarantine_tsd_get(); if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { - if (quarantine == NULL) { - if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) == - NULL) { - idalloc(ptr); - return; - } - } else { - if (quarantine == QUARANTINE_STATE_PURGATORY) { - /* - * Make a note that quarantine() was called - * after quarantine_cleanup() was called. - */ - quarantine = QUARANTINE_STATE_REINCARNATED; - quarantine_tsd_set(&quarantine); - } - idalloc(ptr); - return; + if (quarantine == QUARANTINE_STATE_PURGATORY) { + /* + * Make a note that quarantine() was called after + * quarantine_cleanup() was called. + */ + quarantine = QUARANTINE_STATE_REINCARNATED; + quarantine_tsd_set(&quarantine); } + idalloc(ptr); + return; } /* * Drain one or more objects if the quarantine size limit would be @@ -161,15 +141,24 @@ quarantine(void *ptr) obj->usize = usize; quarantine->curbytes += usize; quarantine->curobjs++; - if (opt_junk) - memset(ptr, 0x5a, usize); + if (config_fill && opt_junk) { + /* + * Only do redzone validation if Valgrind isn't in + * operation. + */ + if ((config_valgrind == false || opt_valgrind == false) + && usize <= SMALL_MAXCLASS) + arena_quarantine_junk_small(ptr, usize); + else + memset(ptr, 0x5a, usize); + } } else { assert(quarantine->curbytes == 0); idalloc(ptr); } } -static void +void quarantine_cleanup(void *arg) { quarantine_t *quarantine = *(quarantine_t **)arg; diff --git a/deps/jemalloc/src/rtree.c b/deps/jemalloc/src/rtree.c index 90c6935a0ed..205957ac4e1 100644 --- a/deps/jemalloc/src/rtree.c +++ b/deps/jemalloc/src/rtree.c @@ -2,42 +2,55 @@ #include "jemalloc/internal/jemalloc_internal.h" rtree_t * -rtree_new(unsigned bits) +rtree_new(unsigned bits, rtree_alloc_t *alloc, rtree_dalloc_t *dalloc) { rtree_t *ret; - unsigned bits_per_level, height, i; + unsigned bits_per_level, bits_in_leaf, height, i; + + assert(bits > 0 && bits <= (sizeof(uintptr_t) << 3)); bits_per_level = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(void *)))) - 1; - height = bits / bits_per_level; - if (height * bits_per_level != bits) - height++; - assert(height * bits_per_level >= bits); + bits_in_leaf = ffs(pow2_ceil((RTREE_NODESIZE / sizeof(uint8_t)))) - 1; + if (bits > bits_in_leaf) { + height = 1 + (bits - bits_in_leaf) / bits_per_level; + if ((height-1) * bits_per_level + bits_in_leaf != bits) + height++; + } else { + height = 1; + } + assert((height-1) * bits_per_level + bits_in_leaf >= bits); - ret = (rtree_t*)base_alloc(offsetof(rtree_t, level2bits) + + ret = (rtree_t*)alloc(offsetof(rtree_t, level2bits) + (sizeof(unsigned) * height)); if (ret == NULL) return (NULL); memset(ret, 0, offsetof(rtree_t, level2bits) + (sizeof(unsigned) * height)); + ret->alloc = alloc; + ret->dalloc = dalloc; if (malloc_mutex_init(&ret->mutex)) { - /* Leak the rtree. */ + if (dalloc != NULL) + dalloc(ret); return (NULL); } ret->height = height; - if (bits_per_level * height > bits) - ret->level2bits[0] = bits % bits_per_level; - else - ret->level2bits[0] = bits_per_level; - for (i = 1; i < height; i++) - ret->level2bits[i] = bits_per_level; - - ret->root = (void**)base_alloc(sizeof(void *) << ret->level2bits[0]); + if (height > 1) { + if ((height-1) * bits_per_level + bits_in_leaf > bits) { + ret->level2bits[0] = (bits - bits_in_leaf) % + bits_per_level; + } else + ret->level2bits[0] = bits_per_level; + for (i = 1; i < height-1; i++) + ret->level2bits[i] = bits_per_level; + ret->level2bits[height-1] = bits_in_leaf; + } else + ret->level2bits[0] = bits; + + ret->root = (void**)alloc(sizeof(void *) << ret->level2bits[0]); if (ret->root == NULL) { - /* - * We leak the rtree here, since there's no generic base - * deallocation. - */ + if (dalloc != NULL) + dalloc(ret); return (NULL); } memset(ret->root, 0, sizeof(void *) << ret->level2bits[0]); @@ -45,6 +58,31 @@ rtree_new(unsigned bits) return (ret); } +static void +rtree_delete_subtree(rtree_t *rtree, void **node, unsigned level) +{ + + if (level < rtree->height - 1) { + size_t nchildren, i; + + nchildren = ZU(1) << rtree->level2bits[level]; + for (i = 0; i < nchildren; i++) { + void **child = (void **)node[i]; + if (child != NULL) + rtree_delete_subtree(rtree, child, level + 1); + } + } + rtree->dalloc(node); +} + +void +rtree_delete(rtree_t *rtree) +{ + + rtree_delete_subtree(rtree, rtree->root, 0); + rtree->dalloc(rtree); +} + void rtree_prefork(rtree_t *rtree) { diff --git a/deps/jemalloc/src/stats.c b/deps/jemalloc/src/stats.c index 43f87af6700..bef2ab33cd4 100644 --- a/deps/jemalloc/src/stats.c +++ b/deps/jemalloc/src/stats.c @@ -345,25 +345,25 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, malloc_cprintf(write_cb, cbopaque, "Assertions %s\n", bv ? "enabled" : "disabled"); -#define OPT_WRITE_BOOL(n) \ +#define OPT_WRITE_BOOL(n) \ if ((err = je_mallctl("opt."#n, &bv, &bsz, NULL, 0)) \ == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %s\n", bv ? "true" : "false"); \ } -#define OPT_WRITE_SIZE_T(n) \ +#define OPT_WRITE_SIZE_T(n) \ if ((err = je_mallctl("opt."#n, &sv, &ssz, NULL, 0)) \ == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zu\n", sv); \ } -#define OPT_WRITE_SSIZE_T(n) \ +#define OPT_WRITE_SSIZE_T(n) \ if ((err = je_mallctl("opt."#n, &ssv, &sssz, NULL, 0)) \ == 0) { \ malloc_cprintf(write_cb, cbopaque, \ " opt."#n": %zd\n", ssv); \ } -#define OPT_WRITE_CHAR_P(n) \ +#define OPT_WRITE_CHAR_P(n) \ if ((err = je_mallctl("opt."#n, &cpv, &cpsz, NULL, 0)) \ == 0) { \ malloc_cprintf(write_cb, cbopaque, \ diff --git a/deps/jemalloc/src/tcache.c b/deps/jemalloc/src/tcache.c index 47e14f30b1a..6de92960b2d 100644 --- a/deps/jemalloc/src/tcache.c +++ b/deps/jemalloc/src/tcache.c @@ -97,9 +97,8 @@ tcache_bin_flush_small(tcache_bin_t *tbin, size_t binind, unsigned rem, arena_bin_t *bin = &arena->bins[binind]; if (config_prof && arena == tcache->arena) { - malloc_mutex_lock(&arena->lock); - arena_prof_accum(arena, tcache->prof_accumbytes); - malloc_mutex_unlock(&arena->lock); + if (arena_prof_accum(arena, tcache->prof_accumbytes)) + prof_idump(); tcache->prof_accumbytes = 0; } @@ -176,11 +175,14 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, arena_chunk_t *chunk = (arena_chunk_t *)CHUNK_ADDR2BASE( tbin->avail[0]); arena_t *arena = chunk->arena; + UNUSED bool idump; + if (config_prof) + idump = false; malloc_mutex_lock(&arena->lock); if ((config_prof || config_stats) && arena == tcache->arena) { if (config_prof) { - arena_prof_accum(arena, + idump = arena_prof_accum_locked(arena, tcache->prof_accumbytes); tcache->prof_accumbytes = 0; } @@ -212,6 +214,8 @@ tcache_bin_flush_large(tcache_bin_t *tbin, size_t binind, unsigned rem, } } malloc_mutex_unlock(&arena->lock); + if (config_prof && idump) + prof_idump(); } if (config_stats && merged_stats == false) { /* @@ -256,8 +260,8 @@ tcache_arena_dissociate(tcache_t *tcache) /* Unlink from list of extant tcaches. */ malloc_mutex_lock(&tcache->arena->lock); ql_remove(&tcache->arena->tcache_ql, tcache, link); - malloc_mutex_unlock(&tcache->arena->lock); tcache_stats_merge(tcache, tcache->arena); + malloc_mutex_unlock(&tcache->arena->lock); } } @@ -288,7 +292,7 @@ tcache_create(arena_t *arena) else if (size <= tcache_maxclass) tcache = (tcache_t *)arena_malloc_large(arena, size, true); else - tcache = (tcache_t *)icallocx(size, false, arena); + tcache = (tcache_t *)icalloct(size, false, arena); if (tcache == NULL) return (NULL); @@ -343,11 +347,9 @@ tcache_destroy(tcache_t *tcache) } } - if (config_prof && tcache->prof_accumbytes > 0) { - malloc_mutex_lock(&tcache->arena->lock); - arena_prof_accum(tcache->arena, tcache->prof_accumbytes); - malloc_mutex_unlock(&tcache->arena->lock); - } + if (config_prof && tcache->prof_accumbytes > 0 && + arena_prof_accum(tcache->arena, tcache->prof_accumbytes)) + prof_idump(); tcache_size = arena_salloc(tcache, false); if (tcache_size <= SMALL_MAXCLASS) { @@ -364,7 +366,7 @@ tcache_destroy(tcache_t *tcache) arena_dalloc_large(arena, chunk, tcache); } else - idallocx(tcache, false); + idalloct(tcache, false); } void @@ -397,11 +399,14 @@ tcache_thread_cleanup(void *arg) } } +/* Caller must own arena->lock. */ void tcache_stats_merge(tcache_t *tcache, arena_t *arena) { unsigned i; + cassert(config_stats); + /* Merge and reset tcache stats. */ for (i = 0; i < NBINS; i++) { arena_bin_t *bin = &arena->bins[i]; diff --git a/deps/jemalloc/src/tsd.c b/deps/jemalloc/src/tsd.c index 961a546329c..700caabfe47 100644 --- a/deps/jemalloc/src/tsd.c +++ b/deps/jemalloc/src/tsd.c @@ -21,7 +21,7 @@ void malloc_tsd_dalloc(void *wrapper) { - idalloc(wrapper); + idalloct(wrapper, false); } void @@ -105,3 +105,37 @@ JEMALLOC_SECTION(".CRT$XLY") JEMALLOC_ATTR(used) static const BOOL (WINAPI *tls_callback)(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) = _tls_callback; #endif + +#if (!defined(JEMALLOC_MALLOC_THREAD_CLEANUP) && !defined(JEMALLOC_TLS) && \ + !defined(_WIN32)) +void * +tsd_init_check_recursion(tsd_init_head_t *head, tsd_init_block_t *block) +{ + pthread_t self = pthread_self(); + tsd_init_block_t *iter; + + /* Check whether this thread has already inserted into the list. */ + malloc_mutex_lock(&head->lock); + ql_foreach(iter, &head->blocks, link) { + if (iter->thread == self) { + malloc_mutex_unlock(&head->lock); + return (iter->data); + } + } + /* Insert block into list. */ + ql_elm_new(block, link); + block->thread = self; + ql_tail_insert(&head->blocks, block, link); + malloc_mutex_unlock(&head->lock); + return (NULL); +} + +void +tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) +{ + + malloc_mutex_lock(&head->lock); + ql_remove(&head->blocks, block, link); + malloc_mutex_unlock(&head->lock); +} +#endif diff --git a/deps/jemalloc/src/util.c b/deps/jemalloc/src/util.c index b3a01143698..93a19fd16f7 100644 --- a/deps/jemalloc/src/util.c +++ b/deps/jemalloc/src/util.c @@ -77,7 +77,7 @@ malloc_write(const char *s) * provide a wrapper. */ int -buferror(char *buf, size_t buflen) +buferror(int err, char *buf, size_t buflen) { #ifdef _WIN32 @@ -85,34 +85,36 @@ buferror(char *buf, size_t buflen) (LPSTR)buf, buflen, NULL); return (0); #elif defined(_GNU_SOURCE) - char *b = strerror_r(errno, buf, buflen); + char *b = strerror_r(err, buf, buflen); if (b != buf) { strncpy(buf, b, buflen); buf[buflen-1] = '\0'; } return (0); #else - return (strerror_r(errno, buf, buflen)); + return (strerror_r(err, buf, buflen)); #endif } uintmax_t -malloc_strtoumax(const char *nptr, char **endptr, int base) +malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { uintmax_t ret, digit; int b; bool neg; const char *p, *ns; + p = nptr; if (base < 0 || base == 1 || base > 36) { + ns = p; set_errno(EINVAL); - return (UINTMAX_MAX); + ret = UINTMAX_MAX; + goto label_return; } b = base; /* Swallow leading whitespace and get sign, if any. */ neg = false; - p = nptr; while (true) { switch (*p) { case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': @@ -146,7 +148,7 @@ malloc_strtoumax(const char *nptr, char **endptr, int base) if (b == 8) p++; break; - case 'x': + case 'X': case 'x': switch (p[2]) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': @@ -164,7 +166,9 @@ malloc_strtoumax(const char *nptr, char **endptr, int base) } break; default: - break; + p++; + ret = 0; + goto label_return; } } if (b == 0) @@ -181,13 +185,22 @@ malloc_strtoumax(const char *nptr, char **endptr, int base) if (ret < pret) { /* Overflow. */ set_errno(ERANGE); - return (UINTMAX_MAX); + ret = UINTMAX_MAX; + goto label_return; } p++; } if (neg) ret = -ret; + if (p == ns) { + /* No conversion performed. */ + set_errno(EINVAL); + ret = UINTMAX_MAX; + goto label_return; + } + +label_return: if (endptr != NULL) { if (p == ns) { /* No characters were converted. */ @@ -195,7 +208,6 @@ malloc_strtoumax(const char *nptr, char **endptr, int base) } else *endptr = (char *)p; } - return (ret); } @@ -331,7 +343,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) APPEND_C(' '); \ } \ } while (0) -#define GET_ARG_NUMERIC(val, len) do { \ +#define GET_ARG_NUMERIC(val, len) do { \ switch (len) { \ case '?': \ val = va_arg(ap, int); \ @@ -354,6 +366,9 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) case 'j': \ val = va_arg(ap, intmax_t); \ break; \ + case 'j' | 0x80: \ + val = va_arg(ap, uintmax_t); \ + break; \ case 't': \ val = va_arg(ap, ptrdiff_t); \ break; \ @@ -385,11 +400,6 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) unsigned char len = '?'; f++; - if (*f == '%') { - /* %% */ - APPEND_C(*f); - break; - } /* Flags. */ while (true) { switch (*f) { @@ -419,6 +429,10 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) case '*': width = va_arg(ap, int); f++; + if (width < 0) { + left_justify = true; + width = -width; + } break; case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': { @@ -428,19 +442,16 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) assert(uwidth != UINTMAX_MAX || get_errno() != ERANGE); width = (int)uwidth; - if (*f == '.') { - f++; - goto label_precision; - } else - goto label_length; break; - } case '.': - f++; - goto label_precision; - default: goto label_length; + } default: + break; } + /* Width/precision separator. */ + if (*f == '.') + f++; + else + goto label_length; /* Precision. */ - label_precision: switch (*f) { case '*': prec = va_arg(ap, int); @@ -469,16 +480,8 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) } else len = 'l'; break; - case 'j': - len = 'j'; - f++; - break; - case 't': - len = 't'; - f++; - break; - case 'z': - len = 'z'; + case 'q': case 'j': case 't': case 'z': + len = *f; f++; break; default: break; @@ -487,6 +490,11 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) switch (*f) { char *s; size_t slen; + case '%': + /* %% */ + APPEND_C(*f); + f++; + break; case 'd': case 'i': { intmax_t val JEMALLOC_CC_SILENCE_INIT(0); char buf[D2S_BUFSIZE]; @@ -540,7 +548,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) assert(len == '?' || len == 'l'); assert_not_implemented(len != 'l'); s = va_arg(ap, char *); - slen = (prec == -1) ? strlen(s) : prec; + slen = (prec < 0) ? strlen(s) : prec; APPEND_PADDED_S(s, slen, width, left_justify); f++; break; @@ -553,8 +561,7 @@ malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) APPEND_PADDED_S(s, slen, width, left_justify); f++; break; - } - default: not_implemented(); + } default: not_reached(); } break; } default: { diff --git a/deps/jemalloc/src/zone.c b/deps/jemalloc/src/zone.c index c62c183f65e..e0302ef4edc 100644 --- a/deps/jemalloc/src/zone.c +++ b/deps/jemalloc/src/zone.c @@ -137,7 +137,7 @@ zone_destroy(malloc_zone_t *zone) { /* This function should never be called. */ - assert(false); + not_reached(); return (NULL); } diff --git a/deps/jemalloc/test/ALLOCM_ARENA.c b/deps/jemalloc/test/ALLOCM_ARENA.c deleted file mode 100644 index 15856908ffc..00000000000 --- a/deps/jemalloc/test/ALLOCM_ARENA.c +++ /dev/null @@ -1,66 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define NTHREADS 10 - -void * -je_thread_start(void *arg) -{ - unsigned thread_ind = (unsigned)(uintptr_t)arg; - unsigned arena_ind; - int r; - void *p; - size_t rsz, sz; - - sz = sizeof(arena_ind); - if (mallctl("arenas.extend", &arena_ind, &sz, NULL, 0) - != 0) { - malloc_printf("Error in arenas.extend\n"); - abort(); - } - - if (thread_ind % 4 != 3) { - size_t mib[3]; - size_t miblen = sizeof(mib) / sizeof(size_t); - const char *dss_precs[] = {"disabled", "primary", "secondary"}; - const char *dss = dss_precs[thread_ind % 4]; - if (mallctlnametomib("arena.0.dss", mib, &miblen) != 0) { - malloc_printf("Error in mallctlnametomib()\n"); - abort(); - } - mib[1] = arena_ind; - if (mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss, - sizeof(const char *))) { - malloc_printf("Error in mallctlbymib()\n"); - abort(); - } - } - - r = allocm(&p, &rsz, 1, ALLOCM_ARENA(arena_ind)); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected allocm() error\n"); - abort(); - } - - return (NULL); -} - -int -main(void) -{ - je_thread_t threads[NTHREADS]; - unsigned i; - - malloc_printf("Test begin\n"); - - for (i = 0; i < NTHREADS; i++) { - je_thread_create(&threads[i], je_thread_start, - (void *)(uintptr_t)i); - } - - for (i = 0; i < NTHREADS; i++) - je_thread_join(threads[i], NULL); - - malloc_printf("Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/ALLOCM_ARENA.exp b/deps/jemalloc/test/ALLOCM_ARENA.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/ALLOCM_ARENA.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/aligned_alloc.exp b/deps/jemalloc/test/aligned_alloc.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc/test/aligned_alloc.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc/test/allocated.c b/deps/jemalloc/test/allocated.c deleted file mode 100644 index 9884905d810..00000000000 --- a/deps/jemalloc/test/allocated.c +++ /dev/null @@ -1,118 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -void * -je_thread_start(void *arg) -{ - int err; - void *p; - uint64_t a0, a1, d0, d1; - uint64_t *ap0, *ap1, *dp0, *dp1; - size_t sz, usize; - - sz = sizeof(a0); - if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto label_return; - } - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - sz = sizeof(ap0); - if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto label_return; - } - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - assert(*ap0 == a0); - - sz = sizeof(d0); - if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto label_return; - } - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - sz = sizeof(dp0); - if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_STATS - assert(false); -#endif - goto label_return; - } - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - exit(1); - } - assert(*dp0 == d0); - - p = malloc(1); - if (p == NULL) { - malloc_printf("%s(): Error in malloc()\n", __func__); - exit(1); - } - - sz = sizeof(a1); - mallctl("thread.allocated", &a1, &sz, NULL, 0); - sz = sizeof(ap1); - mallctl("thread.allocatedp", &ap1, &sz, NULL, 0); - assert(*ap1 == a1); - assert(ap0 == ap1); - - usize = malloc_usable_size(p); - assert(a0 + usize <= a1); - - free(p); - - sz = sizeof(d1); - mallctl("thread.deallocated", &d1, &sz, NULL, 0); - sz = sizeof(dp1); - mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0); - assert(*dp1 == d1); - assert(dp0 == dp1); - - assert(d0 + usize <= d1); - -label_return: - return (NULL); -} - -int -main(void) -{ - int ret = 0; - je_thread_t thread; - - malloc_printf("Test begin\n"); - - je_thread_start(NULL); - - je_thread_create(&thread, je_thread_start, NULL); - je_thread_join(thread, (void *)&ret); - - je_thread_start(NULL); - - je_thread_create(&thread, je_thread_start, NULL); - je_thread_join(thread, (void *)&ret); - - je_thread_start(NULL); - - malloc_printf("Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/allocated.exp b/deps/jemalloc/test/allocated.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/allocated.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/allocm.c b/deps/jemalloc/test/allocm.c deleted file mode 100644 index 80be673b8fd..00000000000 --- a/deps/jemalloc/test/allocm.c +++ /dev/null @@ -1,194 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ -#define MAXALIGN ((size_t)0x2000000LU) -#define NITER 4 - -int -main(void) -{ - int r; - void *p; - size_t nsz, rsz, sz, alignment, total; - unsigned i; - void *ps[NITER]; - - malloc_printf("Test begin\n"); - - sz = 42; - nsz = 0; - r = nallocm(&nsz, sz, 0); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected nallocm() error\n"); - abort(); - } - rsz = 0; - r = allocm(&p, &rsz, sz, 0); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected allocm() error\n"); - abort(); - } - if (rsz < sz) - malloc_printf("Real size smaller than expected\n"); - if (nsz != rsz) - malloc_printf("nallocm()/allocm() rsize mismatch\n"); - if (dallocm(p, 0) != ALLOCM_SUCCESS) - malloc_printf("Unexpected dallocm() error\n"); - - r = allocm(&p, NULL, sz, 0); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected allocm() error\n"); - abort(); - } - if (dallocm(p, 0) != ALLOCM_SUCCESS) - malloc_printf("Unexpected dallocm() error\n"); - - nsz = 0; - r = nallocm(&nsz, sz, ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected nallocm() error\n"); - abort(); - } - rsz = 0; - r = allocm(&p, &rsz, sz, ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected allocm() error\n"); - abort(); - } - if (nsz != rsz) - malloc_printf("nallocm()/allocm() rsize mismatch\n"); - if (dallocm(p, 0) != ALLOCM_SUCCESS) - malloc_printf("Unexpected dallocm() error\n"); - -#if LG_SIZEOF_PTR == 3 - alignment = UINT64_C(0x8000000000000000); - sz = UINT64_C(0x8000000000000000); -#else - alignment = 0x80000000LU; - sz = 0x80000000LU; -#endif - nsz = 0; - r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - malloc_printf( - "Expected error for nallocm(&nsz, %zu, %#x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - rsz = 0; - r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - malloc_printf( - "Expected error for allocm(&p, %zu, %#x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - if (nsz != rsz) - malloc_printf("nallocm()/allocm() rsize mismatch\n"); - -#if LG_SIZEOF_PTR == 3 - alignment = UINT64_C(0x4000000000000000); - sz = UINT64_C(0x8400000000000001); -#else - alignment = 0x40000000LU; - sz = 0x84000001LU; -#endif - nsz = 0; - r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected nallocm() error\n"); - rsz = 0; - r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - malloc_printf( - "Expected error for allocm(&p, %zu, %#x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - - alignment = 0x10LU; -#if LG_SIZEOF_PTR == 3 - sz = UINT64_C(0xfffffffffffffff0); -#else - sz = 0xfffffff0LU; -#endif - nsz = 0; - r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - malloc_printf( - "Expected error for nallocm(&nsz, %zu, %#x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - rsz = 0; - r = allocm(&p, &rsz, sz, ALLOCM_ALIGN(alignment)); - if (r == ALLOCM_SUCCESS) { - malloc_printf( - "Expected error for allocm(&p, %zu, %#x)\n", - sz, ALLOCM_ALIGN(alignment)); - } - if (nsz != rsz) - malloc_printf("nallocm()/allocm() rsize mismatch\n"); - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - malloc_printf("Alignment: %zu\n", alignment); - for (sz = 1; - sz < 3 * alignment && sz < (1U << 31); - sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - nsz = 0; - r = nallocm(&nsz, sz, - ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - malloc_printf( - "nallocm() error for size %zu" - " (%#zx): %d\n", - sz, sz, r); - exit(1); - } - rsz = 0; - r = allocm(&ps[i], &rsz, sz, - ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); - if (r != ALLOCM_SUCCESS) { - malloc_printf( - "allocm() error for size %zu" - " (%#zx): %d\n", - sz, sz, r); - exit(1); - } - if (rsz < sz) { - malloc_printf( - "Real size smaller than" - " expected\n"); - } - if (nsz != rsz) { - malloc_printf( - "nallocm()/allocm() rsize" - " mismatch\n"); - } - if ((uintptr_t)p & (alignment-1)) { - malloc_printf( - "%p inadequately aligned for" - " alignment: %zu\n", p, alignment); - } - sallocm(ps[i], &rsz, 0); - total += rsz; - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - dallocm(ps[i], 0); - ps[i] = NULL; - } - } - } - } - - malloc_printf("Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/allocm.exp b/deps/jemalloc/test/allocm.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc/test/allocm.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc/test/bitmap.exp b/deps/jemalloc/test/bitmap.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/bitmap.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/include/test/SFMT-alti.h b/deps/jemalloc/test/include/test/SFMT-alti.h new file mode 100644 index 00000000000..0005df6b484 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-alti.h @@ -0,0 +1,186 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT-alti.h + * + * @brief SIMD oriented Fast Mersenne Twister(SFMT) + * pseudorandom number generator + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * Copyright (C) 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software. + * see LICENSE.txt + */ + +#ifndef SFMT_ALTI_H +#define SFMT_ALTI_H + +/** + * This function represents the recursion formula in AltiVec and BIG ENDIAN. + * @param a a 128-bit part of the interal state array + * @param b a 128-bit part of the interal state array + * @param c a 128-bit part of the interal state array + * @param d a 128-bit part of the interal state array + * @return output + */ +JEMALLOC_ALWAYS_INLINE +vector unsigned int vec_recursion(vector unsigned int a, + vector unsigned int b, + vector unsigned int c, + vector unsigned int d) { + + const vector unsigned int sl1 = ALTI_SL1; + const vector unsigned int sr1 = ALTI_SR1; +#ifdef ONLY64 + const vector unsigned int mask = ALTI_MSK64; + const vector unsigned char perm_sl = ALTI_SL2_PERM64; + const vector unsigned char perm_sr = ALTI_SR2_PERM64; +#else + const vector unsigned int mask = ALTI_MSK; + const vector unsigned char perm_sl = ALTI_SL2_PERM; + const vector unsigned char perm_sr = ALTI_SR2_PERM; +#endif + vector unsigned int v, w, x, y, z; + x = vec_perm(a, (vector unsigned int)perm_sl, perm_sl); + v = a; + y = vec_sr(b, sr1); + z = vec_perm(c, (vector unsigned int)perm_sr, perm_sr); + w = vec_sl(d, sl1); + z = vec_xor(z, w); + y = vec_and(y, mask); + v = vec_xor(v, x); + z = vec_xor(z, y); + z = vec_xor(z, v); + return z; +} + +/** + * This function fills the internal state array with pseudorandom + * integers. + */ +JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) { + int i; + vector unsigned int r, r1, r2; + + r1 = ctx->sfmt[N - 2].s; + r2 = ctx->sfmt[N - 1].s; + for (i = 0; i < N - POS1; i++) { + r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2); + ctx->sfmt[i].s = r; + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1 - N].s, r1, r2); + ctx->sfmt[i].s = r; + r1 = r2; + r2 = r; + } +} + +/** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pesudorandom numbers to be generated. + */ +JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) { + int i, j; + vector unsigned int r, r1, r2; + + r1 = ctx->sfmt[N - 2].s; + r2 = ctx->sfmt[N - 1].s; + for (i = 0; i < N - POS1; i++) { + r = vec_recursion(ctx->sfmt[i].s, ctx->sfmt[i + POS1].s, r1, r2); + array[i].s = r; + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = vec_recursion(ctx->sfmt[i].s, array[i + POS1 - N].s, r1, r2); + array[i].s = r; + r1 = r2; + r2 = r; + } + /* main loop */ + for (; i < size - N; i++) { + r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2); + array[i].s = r; + r1 = r2; + r2 = r; + } + for (j = 0; j < 2 * N - size; j++) { + ctx->sfmt[j].s = array[j + size - N].s; + } + for (; i < size; i++) { + r = vec_recursion(array[i - N].s, array[i + POS1 - N].s, r1, r2); + array[i].s = r; + ctx->sfmt[j++].s = r; + r1 = r2; + r2 = r; + } +} + +#ifndef ONLY64 +#if defined(__APPLE__) +#define ALTI_SWAP (vector unsigned char) \ + (4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11) +#else +#define ALTI_SWAP {4, 5, 6, 7, 0, 1, 2, 3, 12, 13, 14, 15, 8, 9, 10, 11} +#endif +/** + * This function swaps high and low 32-bit of 64-bit integers in user + * specified array. + * + * @param array an 128-bit array to be swaped. + * @param size size of 128-bit array. + */ +JEMALLOC_INLINE void swap(w128_t *array, int size) { + int i; + const vector unsigned char perm = ALTI_SWAP; + + for (i = 0; i < size; i++) { + array[i].s = vec_perm(array[i].s, (vector unsigned int)perm, perm); + } +} +#endif + +#endif diff --git a/deps/jemalloc/test/include/test/SFMT-params.h b/deps/jemalloc/test/include/test/SFMT-params.h new file mode 100644 index 00000000000..ade6622206d --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params.h @@ -0,0 +1,132 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS_H +#define SFMT_PARAMS_H + +#if !defined(MEXP) +#ifdef __GNUC__ + #warning "MEXP is not defined. I assume MEXP is 19937." +#endif + #define MEXP 19937 +#endif +/*----------------- + BASIC DEFINITIONS + -----------------*/ +/** Mersenne Exponent. The period of the sequence + * is a multiple of 2^MEXP-1. + * #define MEXP 19937 */ +/** SFMT generator has an internal state array of 128-bit integers, + * and N is its size. */ +#define N (MEXP / 128 + 1) +/** N32 is the size of internal state array when regarded as an array + * of 32-bit integers.*/ +#define N32 (N * 4) +/** N64 is the size of internal state array when regarded as an array + * of 64-bit integers.*/ +#define N64 (N * 2) + +/*---------------------- + the parameters of SFMT + following definitions are in paramsXXXX.h file. + ----------------------*/ +/** the pick up position of the array. +#define POS1 122 +*/ + +/** the parameter of shift left as four 32-bit registers. +#define SL1 18 + */ + +/** the parameter of shift left as one 128-bit register. + * The 128-bit integer is shifted by (SL2 * 8) bits. +#define SL2 1 +*/ + +/** the parameter of shift right as four 32-bit registers. +#define SR1 11 +*/ + +/** the parameter of shift right as one 128-bit register. + * The 128-bit integer is shifted by (SL2 * 8) bits. +#define SR2 1 +*/ + +/** A bitmask, used in the recursion. These parameters are introduced + * to break symmetry of SIMD. +#define MSK1 0xdfffffefU +#define MSK2 0xddfecb7fU +#define MSK3 0xbffaffffU +#define MSK4 0xbffffff6U +*/ + +/** These definitions are part of a 128-bit period certification vector. +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0xc98e126aU +*/ + +#if MEXP == 607 + #include "test/SFMT-params607.h" +#elif MEXP == 1279 + #include "test/SFMT-params1279.h" +#elif MEXP == 2281 + #include "test/SFMT-params2281.h" +#elif MEXP == 4253 + #include "test/SFMT-params4253.h" +#elif MEXP == 11213 + #include "test/SFMT-params11213.h" +#elif MEXP == 19937 + #include "test/SFMT-params19937.h" +#elif MEXP == 44497 + #include "test/SFMT-params44497.h" +#elif MEXP == 86243 + #include "test/SFMT-params86243.h" +#elif MEXP == 132049 + #include "test/SFMT-params132049.h" +#elif MEXP == 216091 + #include "test/SFMT-params216091.h" +#else +#ifdef __GNUC__ + #error "MEXP is not valid." + #undef MEXP +#else + #undef MEXP +#endif + +#endif + +#endif /* SFMT_PARAMS_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params11213.h b/deps/jemalloc/test/include/test/SFMT-params11213.h new file mode 100644 index 00000000000..2994bd21da2 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params11213.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS11213_H +#define SFMT_PARAMS11213_H + +#define POS1 68 +#define SL1 14 +#define SL2 3 +#define SR1 7 +#define SR2 3 +#define MSK1 0xeffff7fbU +#define MSK2 0xffffffefU +#define MSK3 0xdfdfbfffU +#define MSK4 0x7fffdbfdU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0xe8148000U +#define PARITY4 0xd0c7afa3U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12} + #define ALTI_SR2_PERM64 {13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12} +#endif /* For OSX */ +#define IDSTR "SFMT-11213:68-14-3-7-3:effff7fb-ffffffef-dfdfbfff-7fffdbfd" + +#endif /* SFMT_PARAMS11213_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params1279.h b/deps/jemalloc/test/include/test/SFMT-params1279.h new file mode 100644 index 00000000000..d7959f98089 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params1279.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS1279_H +#define SFMT_PARAMS1279_H + +#define POS1 7 +#define SL1 14 +#define SL2 3 +#define SR1 5 +#define SR2 1 +#define MSK1 0xf7fefffdU +#define MSK2 0x7fefcfffU +#define MSK3 0xaff3ef3fU +#define MSK4 0xb5ffff7fU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x20000000U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-1279:7-14-3-5-1:f7fefffd-7fefcfff-aff3ef3f-b5ffff7f" + +#endif /* SFMT_PARAMS1279_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params132049.h b/deps/jemalloc/test/include/test/SFMT-params132049.h new file mode 100644 index 00000000000..a1dcec39267 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params132049.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS132049_H +#define SFMT_PARAMS132049_H + +#define POS1 110 +#define SL1 19 +#define SL2 1 +#define SR1 21 +#define SR2 1 +#define MSK1 0xffffbb5fU +#define MSK2 0xfb6ebf95U +#define MSK3 0xfffefffaU +#define MSK4 0xcff77fffU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0xcb520000U +#define PARITY4 0xc7e91c7dU + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-132049:110-19-1-21-1:ffffbb5f-fb6ebf95-fffefffa-cff77fff" + +#endif /* SFMT_PARAMS132049_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params19937.h b/deps/jemalloc/test/include/test/SFMT-params19937.h new file mode 100644 index 00000000000..fb92b4c9b09 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params19937.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS19937_H +#define SFMT_PARAMS19937_H + +#define POS1 122 +#define SL1 18 +#define SL2 1 +#define SR1 11 +#define SR2 1 +#define MSK1 0xdfffffefU +#define MSK2 0xddfecb7fU +#define MSK3 0xbffaffffU +#define MSK4 0xbffffff6U +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x13c9e684U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-19937:122-18-1-11-1:dfffffef-ddfecb7f-bffaffff-bffffff6" + +#endif /* SFMT_PARAMS19937_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params216091.h b/deps/jemalloc/test/include/test/SFMT-params216091.h new file mode 100644 index 00000000000..125ce282048 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params216091.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS216091_H +#define SFMT_PARAMS216091_H + +#define POS1 627 +#define SL1 11 +#define SL2 3 +#define SR1 10 +#define SR2 1 +#define MSK1 0xbff7bff7U +#define MSK2 0xbfffffffU +#define MSK3 0xbffffa7fU +#define MSK4 0xffddfbfbU +#define PARITY1 0xf8000001U +#define PARITY2 0x89e80709U +#define PARITY3 0x3bd2b64bU +#define PARITY4 0x0c64b1e4U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-216091:627-11-3-10-1:bff7bff7-bfffffff-bffffa7f-ffddfbfb" + +#endif /* SFMT_PARAMS216091_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params2281.h b/deps/jemalloc/test/include/test/SFMT-params2281.h new file mode 100644 index 00000000000..0ef85c40701 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params2281.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS2281_H +#define SFMT_PARAMS2281_H + +#define POS1 12 +#define SL1 19 +#define SL2 1 +#define SR1 5 +#define SR2 1 +#define MSK1 0xbff7ffbfU +#define MSK2 0xfdfffffeU +#define MSK3 0xf7ffef7fU +#define MSK4 0xf2f7cbbfU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x41dfa600U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-2281:12-19-1-5-1:bff7ffbf-fdfffffe-f7ffef7f-f2f7cbbf" + +#endif /* SFMT_PARAMS2281_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params4253.h b/deps/jemalloc/test/include/test/SFMT-params4253.h new file mode 100644 index 00000000000..9f07bc67e1a --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params4253.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS4253_H +#define SFMT_PARAMS4253_H + +#define POS1 17 +#define SL1 20 +#define SL2 1 +#define SR1 7 +#define SR2 1 +#define MSK1 0x9f7bffffU +#define MSK2 0x9fffff5fU +#define MSK3 0x3efffffbU +#define MSK4 0xfffff7bbU +#define PARITY1 0xa8000001U +#define PARITY2 0xaf5390a3U +#define PARITY3 0xb740b3f8U +#define PARITY4 0x6c11486dU + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {1,2,3,23,5,6,7,0,9,10,11,4,13,14,15,8} + #define ALTI_SL2_PERM64 {1,2,3,4,5,6,7,31,9,10,11,12,13,14,15,0} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-4253:17-20-1-7-1:9f7bffff-9fffff5f-3efffffb-fffff7bb" + +#endif /* SFMT_PARAMS4253_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params44497.h b/deps/jemalloc/test/include/test/SFMT-params44497.h new file mode 100644 index 00000000000..85598fed519 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params44497.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS44497_H +#define SFMT_PARAMS44497_H + +#define POS1 330 +#define SL1 5 +#define SL2 3 +#define SR1 9 +#define SR2 3 +#define MSK1 0xeffffffbU +#define MSK2 0xdfbebfffU +#define MSK3 0xbfbf7befU +#define MSK4 0x9ffd7bffU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0xa3ac4000U +#define PARITY4 0xecc1327aU + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12} + #define ALTI_SR2_PERM64 {13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12} +#endif /* For OSX */ +#define IDSTR "SFMT-44497:330-5-3-9-3:effffffb-dfbebfff-bfbf7bef-9ffd7bff" + +#endif /* SFMT_PARAMS44497_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params607.h b/deps/jemalloc/test/include/test/SFMT-params607.h new file mode 100644 index 00000000000..bc76485f8b6 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params607.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS607_H +#define SFMT_PARAMS607_H + +#define POS1 2 +#define SL1 15 +#define SL2 3 +#define SR1 13 +#define SR2 3 +#define MSK1 0xfdff37ffU +#define MSK2 0xef7f3f7dU +#define MSK3 0xff777b7dU +#define MSK4 0x7ff7fb2fU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0x5986f054U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2) + #define ALTI_SR2_PERM \ + (vector unsigned char)(5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {3,21,21,21,7,0,1,2,11,4,5,6,15,8,9,10} + #define ALTI_SL2_PERM64 {3,4,5,6,7,29,29,29,11,12,13,14,15,0,1,2} + #define ALTI_SR2_PERM {5,6,7,0,9,10,11,4,13,14,15,8,19,19,19,12} + #define ALTI_SR2_PERM64 {13,14,15,0,1,2,3,4,19,19,19,8,9,10,11,12} +#endif /* For OSX */ +#define IDSTR "SFMT-607:2-15-3-13-3:fdff37ff-ef7f3f7d-ff777b7d-7ff7fb2f" + +#endif /* SFMT_PARAMS607_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-params86243.h b/deps/jemalloc/test/include/test/SFMT-params86243.h new file mode 100644 index 00000000000..5e4d783c5d7 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-params86243.h @@ -0,0 +1,81 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#ifndef SFMT_PARAMS86243_H +#define SFMT_PARAMS86243_H + +#define POS1 366 +#define SL1 6 +#define SL2 7 +#define SR1 19 +#define SR2 1 +#define MSK1 0xfdbffbffU +#define MSK2 0xbff7ff3fU +#define MSK3 0xfd77efffU +#define MSK4 0xbf9ff3ffU +#define PARITY1 0x00000001U +#define PARITY2 0x00000000U +#define PARITY3 0x00000000U +#define PARITY4 0xe9528d85U + + +/* PARAMETERS FOR ALTIVEC */ +#if defined(__APPLE__) /* For OSX */ + #define ALTI_SL1 (vector unsigned int)(SL1, SL1, SL1, SL1) + #define ALTI_SR1 (vector unsigned int)(SR1, SR1, SR1, SR1) + #define ALTI_MSK (vector unsigned int)(MSK1, MSK2, MSK3, MSK4) + #define ALTI_MSK64 \ + (vector unsigned int)(MSK2, MSK1, MSK4, MSK3) + #define ALTI_SL2_PERM \ + (vector unsigned char)(25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6) + #define ALTI_SL2_PERM64 \ + (vector unsigned char)(7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6) + #define ALTI_SR2_PERM \ + (vector unsigned char)(7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14) + #define ALTI_SR2_PERM64 \ + (vector unsigned char)(15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14) +#else /* For OTHER OSs(Linux?) */ + #define ALTI_SL1 {SL1, SL1, SL1, SL1} + #define ALTI_SR1 {SR1, SR1, SR1, SR1} + #define ALTI_MSK {MSK1, MSK2, MSK3, MSK4} + #define ALTI_MSK64 {MSK2, MSK1, MSK4, MSK3} + #define ALTI_SL2_PERM {25,25,25,25,3,25,25,25,7,0,1,2,11,4,5,6} + #define ALTI_SL2_PERM64 {7,25,25,25,25,25,25,25,15,0,1,2,3,4,5,6} + #define ALTI_SR2_PERM {7,0,1,2,11,4,5,6,15,8,9,10,17,12,13,14} + #define ALTI_SR2_PERM64 {15,0,1,2,3,4,5,6,17,8,9,10,11,12,13,14} +#endif /* For OSX */ +#define IDSTR "SFMT-86243:366-6-7-19-1:fdbffbff-bff7ff3f-fd77efff-bf9ff3ff" + +#endif /* SFMT_PARAMS86243_H */ diff --git a/deps/jemalloc/test/include/test/SFMT-sse2.h b/deps/jemalloc/test/include/test/SFMT-sse2.h new file mode 100644 index 00000000000..0314a163d58 --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT-sse2.h @@ -0,0 +1,157 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT-sse2.h + * @brief SIMD oriented Fast Mersenne Twister(SFMT) for Intel SSE2 + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * @note We assume LITTLE ENDIAN in this file + * + * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software, see LICENSE.txt + */ + +#ifndef SFMT_SSE2_H +#define SFMT_SSE2_H + +/** + * This function represents the recursion formula. + * @param a a 128-bit part of the interal state array + * @param b a 128-bit part of the interal state array + * @param c a 128-bit part of the interal state array + * @param d a 128-bit part of the interal state array + * @param mask 128-bit mask + * @return output + */ +JEMALLOC_ALWAYS_INLINE __m128i mm_recursion(__m128i *a, __m128i *b, + __m128i c, __m128i d, __m128i mask) { + __m128i v, x, y, z; + + x = _mm_load_si128(a); + y = _mm_srli_epi32(*b, SR1); + z = _mm_srli_si128(c, SR2); + v = _mm_slli_epi32(d, SL1); + z = _mm_xor_si128(z, x); + z = _mm_xor_si128(z, v); + x = _mm_slli_si128(x, SL2); + y = _mm_and_si128(y, mask); + z = _mm_xor_si128(z, x); + z = _mm_xor_si128(z, y); + return z; +} + +/** + * This function fills the internal state array with pseudorandom + * integers. + */ +JEMALLOC_INLINE void gen_rand_all(sfmt_t *ctx) { + int i; + __m128i r, r1, r2, mask; + mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1); + + r1 = _mm_load_si128(&ctx->sfmt[N - 2].si); + r2 = _mm_load_si128(&ctx->sfmt[N - 1].si); + for (i = 0; i < N - POS1; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2, + mask); + _mm_store_si128(&ctx->sfmt[i].si, r); + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&ctx->sfmt[i].si, r); + r1 = r2; + r2 = r; + } +} + +/** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pesudorandom numbers to be generated. + */ +JEMALLOC_INLINE void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) { + int i, j; + __m128i r, r1, r2, mask; + mask = _mm_set_epi32(MSK4, MSK3, MSK2, MSK1); + + r1 = _mm_load_si128(&ctx->sfmt[N - 2].si); + r2 = _mm_load_si128(&ctx->sfmt[N - 1].si); + for (i = 0; i < N - POS1; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &ctx->sfmt[i + POS1].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + r1 = r2; + r2 = r; + } + for (; i < N; i++) { + r = mm_recursion(&ctx->sfmt[i].si, &array[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + r1 = r2; + r2 = r; + } + /* main loop */ + for (; i < size - N; i++) { + r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + r1 = r2; + r2 = r; + } + for (j = 0; j < 2 * N - size; j++) { + r = _mm_load_si128(&array[j + size - N].si); + _mm_store_si128(&ctx->sfmt[j].si, r); + } + for (; i < size; i++) { + r = mm_recursion(&array[i - N].si, &array[i + POS1 - N].si, r1, r2, + mask); + _mm_store_si128(&array[i].si, r); + _mm_store_si128(&ctx->sfmt[j++].si, r); + r1 = r2; + r2 = r; + } +} + +#endif diff --git a/deps/jemalloc/test/include/test/SFMT.h b/deps/jemalloc/test/include/test/SFMT.h new file mode 100644 index 00000000000..09c1607dd4c --- /dev/null +++ b/deps/jemalloc/test/include/test/SFMT.h @@ -0,0 +1,171 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT.h + * + * @brief SIMD oriented Fast Mersenne Twister(SFMT) pseudorandom + * number generator + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * Copyright (C) 2006, 2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software. + * see LICENSE.txt + * + * @note We assume that your system has inttypes.h. If your system + * doesn't have inttypes.h, you have to typedef uint32_t and uint64_t, + * and you have to define PRIu64 and PRIx64 in this file as follows: + * @verbatim + typedef unsigned int uint32_t + typedef unsigned long long uint64_t + #define PRIu64 "llu" + #define PRIx64 "llx" +@endverbatim + * uint32_t must be exactly 32-bit unsigned integer type (no more, no + * less), and uint64_t must be exactly 64-bit unsigned integer type. + * PRIu64 and PRIx64 are used for printf function to print 64-bit + * unsigned int and 64-bit unsigned int in hexadecimal format. + */ + +#ifndef SFMT_H +#define SFMT_H + +typedef struct sfmt_s sfmt_t; + +uint32_t gen_rand32(sfmt_t *ctx); +uint32_t gen_rand32_range(sfmt_t *ctx, uint32_t limit); +uint64_t gen_rand64(sfmt_t *ctx); +uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit); +void fill_array32(sfmt_t *ctx, uint32_t *array, int size); +void fill_array64(sfmt_t *ctx, uint64_t *array, int size); +sfmt_t *init_gen_rand(uint32_t seed); +sfmt_t *init_by_array(uint32_t *init_key, int key_length); +void fini_gen_rand(sfmt_t *ctx); +const char *get_idstring(void); +int get_min_array_size32(void); +int get_min_array_size64(void); + +#ifndef JEMALLOC_ENABLE_INLINE +double to_real1(uint32_t v); +double genrand_real1(sfmt_t *ctx); +double to_real2(uint32_t v); +double genrand_real2(sfmt_t *ctx); +double to_real3(uint32_t v); +double genrand_real3(sfmt_t *ctx); +double to_res53(uint64_t v); +double to_res53_mix(uint32_t x, uint32_t y); +double genrand_res53(sfmt_t *ctx); +double genrand_res53_mix(sfmt_t *ctx); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(SFMT_C_)) +/* These real versions are due to Isaku Wada */ +/** generates a random number on [0,1]-real-interval */ +JEMALLOC_INLINE double to_real1(uint32_t v) +{ + return v * (1.0/4294967295.0); + /* divided by 2^32-1 */ +} + +/** generates a random number on [0,1]-real-interval */ +JEMALLOC_INLINE double genrand_real1(sfmt_t *ctx) +{ + return to_real1(gen_rand32(ctx)); +} + +/** generates a random number on [0,1)-real-interval */ +JEMALLOC_INLINE double to_real2(uint32_t v) +{ + return v * (1.0/4294967296.0); + /* divided by 2^32 */ +} + +/** generates a random number on [0,1)-real-interval */ +JEMALLOC_INLINE double genrand_real2(sfmt_t *ctx) +{ + return to_real2(gen_rand32(ctx)); +} + +/** generates a random number on (0,1)-real-interval */ +JEMALLOC_INLINE double to_real3(uint32_t v) +{ + return (((double)v) + 0.5)*(1.0/4294967296.0); + /* divided by 2^32 */ +} + +/** generates a random number on (0,1)-real-interval */ +JEMALLOC_INLINE double genrand_real3(sfmt_t *ctx) +{ + return to_real3(gen_rand32(ctx)); +} +/** These real versions are due to Isaku Wada */ + +/** generates a random number on [0,1) with 53-bit resolution*/ +JEMALLOC_INLINE double to_res53(uint64_t v) +{ + return v * (1.0/18446744073709551616.0L); +} + +/** generates a random number on [0,1) with 53-bit resolution from two + * 32 bit integers */ +JEMALLOC_INLINE double to_res53_mix(uint32_t x, uint32_t y) +{ + return to_res53(x | ((uint64_t)y << 32)); +} + +/** generates a random number on [0,1) with 53-bit resolution + */ +JEMALLOC_INLINE double genrand_res53(sfmt_t *ctx) +{ + return to_res53(gen_rand64(ctx)); +} + +/** generates a random number on [0,1) with 53-bit resolution + using 32bit integer. + */ +JEMALLOC_INLINE double genrand_res53_mix(sfmt_t *ctx) +{ + uint32_t x, y; + + x = gen_rand32(ctx); + y = gen_rand32(ctx); + return to_res53_mix(x, y); +} +#endif +#endif diff --git a/deps/jemalloc/test/include/test/jemalloc_test.h.in b/deps/jemalloc/test/include/test/jemalloc_test.h.in new file mode 100644 index 00000000000..730a55dba23 --- /dev/null +++ b/deps/jemalloc/test/include/test/jemalloc_test.h.in @@ -0,0 +1,141 @@ +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +# include +#else +# include +#endif + +/******************************************************************************/ +/* + * Define always-enabled assertion macros, so that test assertions execute even + * if assertions are disabled in the library code. These definitions must + * exist prior to including "jemalloc/internal/util.h". + */ +#define assert(e) do { \ + if (!(e)) { \ + malloc_printf( \ + ": %s:%d: Failed assertion: \"%s\"\n", \ + __FILE__, __LINE__, #e); \ + abort(); \ + } \ +} while (0) + +#define not_reached() do { \ + malloc_printf( \ + ": %s:%d: Unreachable code reached\n", \ + __FILE__, __LINE__); \ + abort(); \ +} while (0) + +#define not_implemented() do { \ + malloc_printf(": %s:%d: Not implemented\n", \ + __FILE__, __LINE__); \ + abort(); \ +} while (0) + +#define assert_not_implemented(e) do { \ + if (!(e)) \ + not_implemented(); \ +} while (0) + +#include "test/jemalloc_test_defs.h" + +#ifdef JEMALLOC_OSSPIN +# include +#endif + +#if defined(HAVE_ALTIVEC) && !defined(__APPLE__) +# include +#endif +#ifdef HAVE_SSE2 +# include +#endif + +/******************************************************************************/ +/* + * For unit tests, expose all public and private interfaces. + */ +#ifdef JEMALLOC_UNIT_TEST +# define JEMALLOC_JET +# define JEMALLOC_MANGLE +# include "jemalloc/internal/jemalloc_internal.h" + +/******************************************************************************/ +/* + * For integration tests, expose the public jemalloc interfaces, but only + * expose the minimum necessary internal utility code (to avoid re-implementing + * essentially identical code within the test infrastructure). + */ +#elif defined(JEMALLOC_INTEGRATION_TEST) +# define JEMALLOC_MANGLE +# include "jemalloc/jemalloc@install_suffix@.h" +# include "jemalloc/internal/jemalloc_internal_defs.h" +# include "jemalloc/internal/jemalloc_internal_macros.h" + +# define JEMALLOC_N(n) @private_namespace@##n +# include "jemalloc/internal/private_namespace.h" + +# define JEMALLOC_H_TYPES +# define JEMALLOC_H_STRUCTS +# define JEMALLOC_H_EXTERNS +# define JEMALLOC_H_INLINES +# include "jemalloc/internal/util.h" +# include "jemalloc/internal/qr.h" +# include "jemalloc/internal/ql.h" +# undef JEMALLOC_H_TYPES +# undef JEMALLOC_H_STRUCTS +# undef JEMALLOC_H_EXTERNS +# undef JEMALLOC_H_INLINES + +/******************************************************************************/ +/* + * For stress tests, expose the public jemalloc interfaces with name mangling + * so that they can be tested as e.g. malloc() and free(). Also expose the + * public jemalloc interfaces with jet_ prefixes, so that stress tests can use + * a separate allocator for their internal data structures. + */ +#elif defined(JEMALLOC_STRESS_TEST) +# include "jemalloc/jemalloc@install_suffix@.h" + +# include "jemalloc/jemalloc_protos_jet.h" + +# define JEMALLOC_JET +# include "jemalloc/internal/jemalloc_internal.h" +# include "jemalloc/internal/public_unnamespace.h" +# undef JEMALLOC_JET + +# include "jemalloc/jemalloc_rename.h" +# define JEMALLOC_MANGLE +# ifdef JEMALLOC_STRESS_TESTLIB +# include "jemalloc/jemalloc_mangle_jet.h" +# else +# include "jemalloc/jemalloc_mangle.h" +# endif + +/******************************************************************************/ +/* + * This header does dangerous things, the effects of which only test code + * should be subject to. + */ +#else +# error "This header cannot be included outside a testing context" +#endif + +/******************************************************************************/ +/* + * Common test utilities. + */ +#include "test/math.h" +#include "test/mtx.h" +#include "test/mq.h" +#include "test/test.h" +#include "test/thd.h" +#define MEXP 19937 +#include "test/SFMT.h" diff --git a/deps/jemalloc/test/include/test/jemalloc_test_defs.h.in b/deps/jemalloc/test/include/test/jemalloc_test_defs.h.in new file mode 100644 index 00000000000..18a9773d705 --- /dev/null +++ b/deps/jemalloc/test/include/test/jemalloc_test_defs.h.in @@ -0,0 +1,5 @@ +#include "jemalloc/internal/jemalloc_internal_defs.h" + +/* For use by SFMT. */ +#undef HAVE_SSE2 +#undef HAVE_ALTIVEC diff --git a/deps/jemalloc/test/include/test/math.h b/deps/jemalloc/test/include/test/math.h new file mode 100644 index 00000000000..a862ed7db24 --- /dev/null +++ b/deps/jemalloc/test/include/test/math.h @@ -0,0 +1,311 @@ +#ifndef JEMALLOC_ENABLE_INLINE +double ln_gamma(double x); +double i_gamma(double x, double p, double ln_gamma_p); +double pt_norm(double p); +double pt_chi2(double p, double df, double ln_gamma_df_2); +double pt_gamma(double p, double shape, double scale, double ln_gamma_shape); +#endif + +#if (defined(JEMALLOC_ENABLE_INLINE) || defined(MATH_C_)) +/* + * Compute the natural log of Gamma(x), accurate to 10 decimal places. + * + * This implementation is based on: + * + * Pike, M.C., I.D. Hill (1966) Algorithm 291: Logarithm of Gamma function + * [S14]. Communications of the ACM 9(9):684. + */ +JEMALLOC_INLINE double +ln_gamma(double x) +{ + double f, z; + + assert(x > 0.0); + + if (x < 7.0) { + f = 1.0; + z = x; + while (z < 7.0) { + f *= z; + z += 1.0; + } + x = z; + f = -log(f); + } else + f = 0.0; + + z = 1.0 / (x * x); + + return (f + (x-0.5) * log(x) - x + 0.918938533204673 + + (((-0.000595238095238 * z + 0.000793650793651) * z - + 0.002777777777778) * z + 0.083333333333333) / x); +} + +/* + * Compute the incomplete Gamma ratio for [0..x], where p is the shape + * parameter, and ln_gamma_p is ln_gamma(p). + * + * This implementation is based on: + * + * Bhattacharjee, G.P. (1970) Algorithm AS 32: The incomplete Gamma integral. + * Applied Statistics 19:285-287. + */ +JEMALLOC_INLINE double +i_gamma(double x, double p, double ln_gamma_p) +{ + double acu, factor, oflo, gin, term, rn, a, b, an, dif; + double pn[6]; + unsigned i; + + assert(p > 0.0); + assert(x >= 0.0); + + if (x == 0.0) + return (0.0); + + acu = 1.0e-10; + oflo = 1.0e30; + gin = 0.0; + factor = exp(p * log(x) - x - ln_gamma_p); + + if (x <= 1.0 || x < p) { + /* Calculation by series expansion. */ + gin = 1.0; + term = 1.0; + rn = p; + + while (true) { + rn += 1.0; + term *= x / rn; + gin += term; + if (term <= acu) { + gin *= factor / p; + return (gin); + } + } + } else { + /* Calculation by continued fraction. */ + a = 1.0 - p; + b = a + x + 1.0; + term = 0.0; + pn[0] = 1.0; + pn[1] = x; + pn[2] = x + 1.0; + pn[3] = x * b; + gin = pn[2] / pn[3]; + + while (true) { + a += 1.0; + b += 2.0; + term += 1.0; + an = a * term; + for (i = 0; i < 2; i++) + pn[i+4] = b * pn[i+2] - an * pn[i]; + if (pn[5] != 0.0) { + rn = pn[4] / pn[5]; + dif = fabs(gin - rn); + if (dif <= acu && dif <= acu * rn) { + gin = 1.0 - factor * gin; + return (gin); + } + gin = rn; + } + for (i = 0; i < 4; i++) + pn[i] = pn[i+2]; + + if (fabs(pn[4]) >= oflo) { + for (i = 0; i < 4; i++) + pn[i] /= oflo; + } + } + } +} + +/* + * Given a value p in [0..1] of the lower tail area of the normal distribution, + * compute the limit on the definite integral from [-inf..z] that satisfies p, + * accurate to 16 decimal places. + * + * This implementation is based on: + * + * Wichura, M.J. (1988) Algorithm AS 241: The percentage points of the normal + * distribution. Applied Statistics 37(3):477-484. + */ +JEMALLOC_INLINE double +pt_norm(double p) +{ + double q, r, ret; + + assert(p > 0.0 && p < 1.0); + + q = p - 0.5; + if (fabs(q) <= 0.425) { + /* p close to 1/2. */ + r = 0.180625 - q * q; + return (q * (((((((2.5090809287301226727e3 * r + + 3.3430575583588128105e4) * r + 6.7265770927008700853e4) * r + + 4.5921953931549871457e4) * r + 1.3731693765509461125e4) * + r + 1.9715909503065514427e3) * r + 1.3314166789178437745e2) + * r + 3.3871328727963666080e0) / + (((((((5.2264952788528545610e3 * r + + 2.8729085735721942674e4) * r + 3.9307895800092710610e4) * r + + 2.1213794301586595867e4) * r + 5.3941960214247511077e3) * + r + 6.8718700749205790830e2) * r + 4.2313330701600911252e1) + * r + 1.0)); + } else { + if (q < 0.0) + r = p; + else + r = 1.0 - p; + assert(r > 0.0); + + r = sqrt(-log(r)); + if (r <= 5.0) { + /* p neither close to 1/2 nor 0 or 1. */ + r -= 1.6; + ret = ((((((((7.74545014278341407640e-4 * r + + 2.27238449892691845833e-2) * r + + 2.41780725177450611770e-1) * r + + 1.27045825245236838258e0) * r + + 3.64784832476320460504e0) * r + + 5.76949722146069140550e0) * r + + 4.63033784615654529590e0) * r + + 1.42343711074968357734e0) / + (((((((1.05075007164441684324e-9 * r + + 5.47593808499534494600e-4) * r + + 1.51986665636164571966e-2) + * r + 1.48103976427480074590e-1) * r + + 6.89767334985100004550e-1) * r + + 1.67638483018380384940e0) * r + + 2.05319162663775882187e0) * r + 1.0)); + } else { + /* p near 0 or 1. */ + r -= 5.0; + ret = ((((((((2.01033439929228813265e-7 * r + + 2.71155556874348757815e-5) * r + + 1.24266094738807843860e-3) * r + + 2.65321895265761230930e-2) * r + + 2.96560571828504891230e-1) * r + + 1.78482653991729133580e0) * r + + 5.46378491116411436990e0) * r + + 6.65790464350110377720e0) / + (((((((2.04426310338993978564e-15 * r + + 1.42151175831644588870e-7) * r + + 1.84631831751005468180e-5) * r + + 7.86869131145613259100e-4) * r + + 1.48753612908506148525e-2) * r + + 1.36929880922735805310e-1) * r + + 5.99832206555887937690e-1) + * r + 1.0)); + } + if (q < 0.0) + ret = -ret; + return (ret); + } +} + +/* + * Given a value p in [0..1] of the lower tail area of the Chi^2 distribution + * with df degrees of freedom, where ln_gamma_df_2 is ln_gamma(df/2.0), compute + * the upper limit on the definite integral from [0..z] that satisfies p, + * accurate to 12 decimal places. + * + * This implementation is based on: + * + * Best, D.J., D.E. Roberts (1975) Algorithm AS 91: The percentage points of + * the Chi^2 distribution. Applied Statistics 24(3):385-388. + * + * Shea, B.L. (1991) Algorithm AS R85: A remark on AS 91: The percentage + * points of the Chi^2 distribution. Applied Statistics 40(1):233-235. + */ +JEMALLOC_INLINE double +pt_chi2(double p, double df, double ln_gamma_df_2) +{ + double e, aa, xx, c, ch, a, q, p1, p2, t, x, b, s1, s2, s3, s4, s5, s6; + unsigned i; + + assert(p >= 0.0 && p < 1.0); + assert(df > 0.0); + + e = 5.0e-7; + aa = 0.6931471805; + + xx = 0.5 * df; + c = xx - 1.0; + + if (df < -1.24 * log(p)) { + /* Starting approximation for small Chi^2. */ + ch = pow(p * xx * exp(ln_gamma_df_2 + xx * aa), 1.0 / xx); + if (ch - e < 0.0) + return (ch); + } else { + if (df > 0.32) { + x = pt_norm(p); + /* + * Starting approximation using Wilson and Hilferty + * estimate. + */ + p1 = 0.222222 / df; + ch = df * pow(x * sqrt(p1) + 1.0 - p1, 3.0); + /* Starting approximation for p tending to 1. */ + if (ch > 2.2 * df + 6.0) { + ch = -2.0 * (log(1.0 - p) - c * log(0.5 * ch) + + ln_gamma_df_2); + } + } else { + ch = 0.4; + a = log(1.0 - p); + while (true) { + q = ch; + p1 = 1.0 + ch * (4.67 + ch); + p2 = ch * (6.73 + ch * (6.66 + ch)); + t = -0.5 + (4.67 + 2.0 * ch) / p1 - (6.73 + ch + * (13.32 + 3.0 * ch)) / p2; + ch -= (1.0 - exp(a + ln_gamma_df_2 + 0.5 * ch + + c * aa) * p2 / p1) / t; + if (fabs(q / ch - 1.0) - 0.01 <= 0.0) + break; + } + } + } + + for (i = 0; i < 20; i++) { + /* Calculation of seven-term Taylor series. */ + q = ch; + p1 = 0.5 * ch; + if (p1 < 0.0) + return (-1.0); + p2 = p - i_gamma(p1, xx, ln_gamma_df_2); + t = p2 * exp(xx * aa + ln_gamma_df_2 + p1 - c * log(ch)); + b = t / ch; + a = 0.5 * t - b * c; + s1 = (210.0 + a * (140.0 + a * (105.0 + a * (84.0 + a * (70.0 + + 60.0 * a))))) / 420.0; + s2 = (420.0 + a * (735.0 + a * (966.0 + a * (1141.0 + 1278.0 * + a)))) / 2520.0; + s3 = (210.0 + a * (462.0 + a * (707.0 + 932.0 * a))) / 2520.0; + s4 = (252.0 + a * (672.0 + 1182.0 * a) + c * (294.0 + a * + (889.0 + 1740.0 * a))) / 5040.0; + s5 = (84.0 + 264.0 * a + c * (175.0 + 606.0 * a)) / 2520.0; + s6 = (120.0 + c * (346.0 + 127.0 * c)) / 5040.0; + ch += t * (1.0 + 0.5 * t * s1 - b * c * (s1 - b * (s2 - b * (s3 + - b * (s4 - b * (s5 - b * s6)))))); + if (fabs(q / ch - 1.0) <= e) + break; + } + + return (ch); +} + +/* + * Given a value p in [0..1] and Gamma distribution shape and scale parameters, + * compute the upper limit on the definite integeral from [0..z] that satisfies + * p. + */ +JEMALLOC_INLINE double +pt_gamma(double p, double shape, double scale, double ln_gamma_shape) +{ + + return (pt_chi2(p, shape * 2.0, ln_gamma_shape) * 0.5 * scale); +} +#endif diff --git a/deps/jemalloc/test/include/test/mq.h b/deps/jemalloc/test/include/test/mq.h new file mode 100644 index 00000000000..11188653c65 --- /dev/null +++ b/deps/jemalloc/test/include/test/mq.h @@ -0,0 +1,110 @@ +/* + * Simple templated message queue implementation that relies on only mutexes for + * synchronization (which reduces portability issues). Given the following + * setup: + * + * typedef struct mq_msg_s mq_msg_t; + * struct mq_msg_s { + * mq_msg(mq_msg_t) link; + * [message data] + * }; + * mq_gen(, mq_, mq_t, mq_msg_t, link) + * + * The API is as follows: + * + * bool mq_init(mq_t *mq); + * void mq_fini(mq_t *mq); + * unsigned mq_count(mq_t *mq); + * mq_msg_t *mq_tryget(mq_t *mq); + * mq_msg_t *mq_get(mq_t *mq); + * void mq_put(mq_t *mq, mq_msg_t *msg); + * + * The message queue linkage embedded in each message is to be treated as + * externally opaque (no need to initialize or clean up externally). mq_fini() + * does not perform any cleanup of messages, since it knows nothing of their + * payloads. + */ +#define mq_msg(a_mq_msg_type) ql_elm(a_mq_msg_type) + +#define mq_gen(a_attr, a_prefix, a_mq_type, a_mq_msg_type, a_field) \ +typedef struct { \ + mtx_t lock; \ + ql_head(a_mq_msg_type) msgs; \ + unsigned count; \ +} a_mq_type; \ +a_attr bool \ +a_prefix##init(a_mq_type *mq) { \ + \ + if (mtx_init(&mq->lock)) \ + return (true); \ + ql_new(&mq->msgs); \ + mq->count = 0; \ + return (false); \ +} \ +a_attr void \ +a_prefix##fini(a_mq_type *mq) \ +{ \ + \ + mtx_fini(&mq->lock); \ +} \ +a_attr unsigned \ +a_prefix##count(a_mq_type *mq) \ +{ \ + unsigned count; \ + \ + mtx_lock(&mq->lock); \ + count = mq->count; \ + mtx_unlock(&mq->lock); \ + return (count); \ +} \ +a_attr a_mq_msg_type * \ +a_prefix##tryget(a_mq_type *mq) \ +{ \ + a_mq_msg_type *msg; \ + \ + mtx_lock(&mq->lock); \ + msg = ql_first(&mq->msgs); \ + if (msg != NULL) { \ + ql_head_remove(&mq->msgs, a_mq_msg_type, a_field); \ + mq->count--; \ + } \ + mtx_unlock(&mq->lock); \ + return (msg); \ +} \ +a_attr a_mq_msg_type * \ +a_prefix##get(a_mq_type *mq) \ +{ \ + a_mq_msg_type *msg; \ + struct timespec timeout; \ + \ + msg = a_prefix##tryget(mq); \ + if (msg != NULL) \ + return (msg); \ + \ + timeout.tv_sec = 0; \ + timeout.tv_nsec = 1; \ + while (true) { \ + nanosleep(&timeout, NULL); \ + msg = a_prefix##tryget(mq); \ + if (msg != NULL) \ + return (msg); \ + if (timeout.tv_sec == 0) { \ + /* Double sleep time, up to max 1 second. */ \ + timeout.tv_nsec <<= 1; \ + if (timeout.tv_nsec >= 1000*1000*1000) { \ + timeout.tv_sec = 1; \ + timeout.tv_nsec = 0; \ + } \ + } \ + } \ +} \ +a_attr void \ +a_prefix##put(a_mq_type *mq, a_mq_msg_type *msg) \ +{ \ + \ + mtx_lock(&mq->lock); \ + ql_elm_new(msg, a_field); \ + ql_tail_insert(&mq->msgs, msg, a_field); \ + mq->count++; \ + mtx_unlock(&mq->lock); \ +} diff --git a/deps/jemalloc/test/include/test/mtx.h b/deps/jemalloc/test/include/test/mtx.h new file mode 100644 index 00000000000..bbe822f54b3 --- /dev/null +++ b/deps/jemalloc/test/include/test/mtx.h @@ -0,0 +1,21 @@ +/* + * mtx is a slightly simplified version of malloc_mutex. This code duplication + * is unfortunate, but there are allocator bootstrapping considerations that + * would leak into the test infrastructure if malloc_mutex were used directly + * in tests. + */ + +typedef struct { +#ifdef _WIN32 + CRITICAL_SECTION lock; +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLock lock; +#else + pthread_mutex_t lock; +#endif +} mtx_t; + +bool mtx_init(mtx_t *mtx); +void mtx_fini(mtx_t *mtx); +void mtx_lock(mtx_t *mtx); +void mtx_unlock(mtx_t *mtx); diff --git a/deps/jemalloc/test/include/test/test.h b/deps/jemalloc/test/include/test/test.h new file mode 100644 index 00000000000..a32ec07c4c5 --- /dev/null +++ b/deps/jemalloc/test/include/test/test.h @@ -0,0 +1,329 @@ +#define ASSERT_BUFSIZE 256 + +#define assert_cmp(t, a, b, cmp, neg_cmp, pri, fmt...) do { \ + t a_ = (a); \ + t b_ = (b); \ + if (!(a_ cmp b_)) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) "#cmp" (%s) --> " \ + "%"pri" "#neg_cmp" %"pri": ", \ + __func__, __FILE__, __LINE__, \ + #a, #b, a_, b_); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) + +#define assert_ptr_eq(a, b, fmt...) assert_cmp(void *, a, b, ==, \ + !=, "p", fmt) +#define assert_ptr_ne(a, b, fmt...) assert_cmp(void *, a, b, !=, \ + ==, "p", fmt) +#define assert_ptr_null(a, fmt...) assert_cmp(void *, a, NULL, ==, \ + !=, "p", fmt) +#define assert_ptr_not_null(a, fmt...) assert_cmp(void *, a, NULL, !=, \ + ==, "p", fmt) + +#define assert_c_eq(a, b, fmt...) assert_cmp(char, a, b, ==, !=, "c", fmt) +#define assert_c_ne(a, b, fmt...) assert_cmp(char, a, b, !=, ==, "c", fmt) +#define assert_c_lt(a, b, fmt...) assert_cmp(char, a, b, <, >=, "c", fmt) +#define assert_c_le(a, b, fmt...) assert_cmp(char, a, b, <=, >, "c", fmt) +#define assert_c_ge(a, b, fmt...) assert_cmp(char, a, b, >=, <, "c", fmt) +#define assert_c_gt(a, b, fmt...) assert_cmp(char, a, b, >, <=, "c", fmt) + +#define assert_x_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "#x", fmt) +#define assert_x_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "#x", fmt) +#define assert_x_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "#x", fmt) +#define assert_x_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "#x", fmt) +#define assert_x_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "#x", fmt) +#define assert_x_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "#x", fmt) + +#define assert_d_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "d", fmt) +#define assert_d_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "d", fmt) +#define assert_d_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "d", fmt) +#define assert_d_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "d", fmt) +#define assert_d_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "d", fmt) +#define assert_d_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "d", fmt) + +#define assert_u_eq(a, b, fmt...) assert_cmp(int, a, b, ==, !=, "u", fmt) +#define assert_u_ne(a, b, fmt...) assert_cmp(int, a, b, !=, ==, "u", fmt) +#define assert_u_lt(a, b, fmt...) assert_cmp(int, a, b, <, >=, "u", fmt) +#define assert_u_le(a, b, fmt...) assert_cmp(int, a, b, <=, >, "u", fmt) +#define assert_u_ge(a, b, fmt...) assert_cmp(int, a, b, >=, <, "u", fmt) +#define assert_u_gt(a, b, fmt...) assert_cmp(int, a, b, >, <=, "u", fmt) + +#define assert_ld_eq(a, b, fmt...) assert_cmp(long, a, b, ==, \ + !=, "ld", fmt) +#define assert_ld_ne(a, b, fmt...) assert_cmp(long, a, b, !=, \ + ==, "ld", fmt) +#define assert_ld_lt(a, b, fmt...) assert_cmp(long, a, b, <, \ + >=, "ld", fmt) +#define assert_ld_le(a, b, fmt...) assert_cmp(long, a, b, <=, \ + >, "ld", fmt) +#define assert_ld_ge(a, b, fmt...) assert_cmp(long, a, b, >=, \ + <, "ld", fmt) +#define assert_ld_gt(a, b, fmt...) assert_cmp(long, a, b, >, \ + <=, "ld", fmt) + +#define assert_lu_eq(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, ==, !=, "lu", fmt) +#define assert_lu_ne(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, !=, ==, "lu", fmt) +#define assert_lu_lt(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, <, >=, "lu", fmt) +#define assert_lu_le(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, <=, >, "lu", fmt) +#define assert_lu_ge(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, >=, <, "lu", fmt) +#define assert_lu_gt(a, b, fmt...) assert_cmp(unsigned long, \ + a, b, >, <=, "lu", fmt) + +#define assert_qd_eq(a, b, fmt...) assert_cmp(long long, a, b, ==, \ + !=, "qd", fmt) +#define assert_qd_ne(a, b, fmt...) assert_cmp(long long, a, b, !=, \ + ==, "qd", fmt) +#define assert_qd_lt(a, b, fmt...) assert_cmp(long long, a, b, <, \ + >=, "qd", fmt) +#define assert_qd_le(a, b, fmt...) assert_cmp(long long, a, b, <=, \ + >, "qd", fmt) +#define assert_qd_ge(a, b, fmt...) assert_cmp(long long, a, b, >=, \ + <, "qd", fmt) +#define assert_qd_gt(a, b, fmt...) assert_cmp(long long, a, b, >, \ + <=, "qd", fmt) + +#define assert_qu_eq(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, ==, !=, "qu", fmt) +#define assert_qu_ne(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, !=, ==, "qu", fmt) +#define assert_qu_lt(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, <, >=, "qu", fmt) +#define assert_qu_le(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, <=, >, "qu", fmt) +#define assert_qu_ge(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, >=, <, "qu", fmt) +#define assert_qu_gt(a, b, fmt...) assert_cmp(unsigned long long, \ + a, b, >, <=, "qu", fmt) + +#define assert_jd_eq(a, b, fmt...) assert_cmp(intmax_t, a, b, ==, \ + !=, "jd", fmt) +#define assert_jd_ne(a, b, fmt...) assert_cmp(intmax_t, a, b, !=, \ + ==, "jd", fmt) +#define assert_jd_lt(a, b, fmt...) assert_cmp(intmax_t, a, b, <, \ + >=, "jd", fmt) +#define assert_jd_le(a, b, fmt...) assert_cmp(intmax_t, a, b, <=, \ + >, "jd", fmt) +#define assert_jd_ge(a, b, fmt...) assert_cmp(intmax_t, a, b, >=, \ + <, "jd", fmt) +#define assert_jd_gt(a, b, fmt...) assert_cmp(intmax_t, a, b, >, \ + <=, "jd", fmt) + +#define assert_ju_eq(a, b, fmt...) assert_cmp(uintmax_t, a, b, ==, \ + !=, "ju", fmt) +#define assert_ju_ne(a, b, fmt...) assert_cmp(uintmax_t, a, b, !=, \ + ==, "ju", fmt) +#define assert_ju_lt(a, b, fmt...) assert_cmp(uintmax_t, a, b, <, \ + >=, "ju", fmt) +#define assert_ju_le(a, b, fmt...) assert_cmp(uintmax_t, a, b, <=, \ + >, "ju", fmt) +#define assert_ju_ge(a, b, fmt...) assert_cmp(uintmax_t, a, b, >=, \ + <, "ju", fmt) +#define assert_ju_gt(a, b, fmt...) assert_cmp(uintmax_t, a, b, >, \ + <=, "ju", fmt) + +#define assert_zd_eq(a, b, fmt...) assert_cmp(ssize_t, a, b, ==, \ + !=, "zd", fmt) +#define assert_zd_ne(a, b, fmt...) assert_cmp(ssize_t, a, b, !=, \ + ==, "zd", fmt) +#define assert_zd_lt(a, b, fmt...) assert_cmp(ssize_t, a, b, <, \ + >=, "zd", fmt) +#define assert_zd_le(a, b, fmt...) assert_cmp(ssize_t, a, b, <=, \ + >, "zd", fmt) +#define assert_zd_ge(a, b, fmt...) assert_cmp(ssize_t, a, b, >=, \ + <, "zd", fmt) +#define assert_zd_gt(a, b, fmt...) assert_cmp(ssize_t, a, b, >, \ + <=, "zd", fmt) + +#define assert_zu_eq(a, b, fmt...) assert_cmp(size_t, a, b, ==, \ + !=, "zu", fmt) +#define assert_zu_ne(a, b, fmt...) assert_cmp(size_t, a, b, !=, \ + ==, "zu", fmt) +#define assert_zu_lt(a, b, fmt...) assert_cmp(size_t, a, b, <, \ + >=, "zu", fmt) +#define assert_zu_le(a, b, fmt...) assert_cmp(size_t, a, b, <=, \ + >, "zu", fmt) +#define assert_zu_ge(a, b, fmt...) assert_cmp(size_t, a, b, >=, \ + <, "zu", fmt) +#define assert_zu_gt(a, b, fmt...) assert_cmp(size_t, a, b, >, \ + <=, "zu", fmt) + +#define assert_d32_eq(a, b, fmt...) assert_cmp(int32_t, a, b, ==, \ + !=, PRId32, fmt) +#define assert_d32_ne(a, b, fmt...) assert_cmp(int32_t, a, b, !=, \ + ==, PRId32, fmt) +#define assert_d32_lt(a, b, fmt...) assert_cmp(int32_t, a, b, <, \ + >=, PRId32, fmt) +#define assert_d32_le(a, b, fmt...) assert_cmp(int32_t, a, b, <=, \ + >, PRId32, fmt) +#define assert_d32_ge(a, b, fmt...) assert_cmp(int32_t, a, b, >=, \ + <, PRId32, fmt) +#define assert_d32_gt(a, b, fmt...) assert_cmp(int32_t, a, b, >, \ + <=, PRId32, fmt) + +#define assert_u32_eq(a, b, fmt...) assert_cmp(uint32_t, a, b, ==, \ + !=, PRIu32, fmt) +#define assert_u32_ne(a, b, fmt...) assert_cmp(uint32_t, a, b, !=, \ + ==, PRIu32, fmt) +#define assert_u32_lt(a, b, fmt...) assert_cmp(uint32_t, a, b, <, \ + >=, PRIu32, fmt) +#define assert_u32_le(a, b, fmt...) assert_cmp(uint32_t, a, b, <=, \ + >, PRIu32, fmt) +#define assert_u32_ge(a, b, fmt...) assert_cmp(uint32_t, a, b, >=, \ + <, PRIu32, fmt) +#define assert_u32_gt(a, b, fmt...) assert_cmp(uint32_t, a, b, >, \ + <=, PRIu32, fmt) + +#define assert_d64_eq(a, b, fmt...) assert_cmp(int64_t, a, b, ==, \ + !=, PRId64, fmt) +#define assert_d64_ne(a, b, fmt...) assert_cmp(int64_t, a, b, !=, \ + ==, PRId64, fmt) +#define assert_d64_lt(a, b, fmt...) assert_cmp(int64_t, a, b, <, \ + >=, PRId64, fmt) +#define assert_d64_le(a, b, fmt...) assert_cmp(int64_t, a, b, <=, \ + >, PRId64, fmt) +#define assert_d64_ge(a, b, fmt...) assert_cmp(int64_t, a, b, >=, \ + <, PRId64, fmt) +#define assert_d64_gt(a, b, fmt...) assert_cmp(int64_t, a, b, >, \ + <=, PRId64, fmt) + +#define assert_u64_eq(a, b, fmt...) assert_cmp(uint64_t, a, b, ==, \ + !=, PRIu64, fmt) +#define assert_u64_ne(a, b, fmt...) assert_cmp(uint64_t, a, b, !=, \ + ==, PRIu64, fmt) +#define assert_u64_lt(a, b, fmt...) assert_cmp(uint64_t, a, b, <, \ + >=, PRIu64, fmt) +#define assert_u64_le(a, b, fmt...) assert_cmp(uint64_t, a, b, <=, \ + >, PRIu64, fmt) +#define assert_u64_ge(a, b, fmt...) assert_cmp(uint64_t, a, b, >=, \ + <, PRIu64, fmt) +#define assert_u64_gt(a, b, fmt...) assert_cmp(uint64_t, a, b, >, \ + <=, PRIu64, fmt) + +#define assert_b_eq(a, b, fmt...) do { \ + bool a_ = (a); \ + bool b_ = (b); \ + if (!(a_ == b_)) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) == (%s) --> %s != %s: ", \ + __func__, __FILE__, __LINE__, \ + #a, #b, a_ ? "true" : "false", \ + b_ ? "true" : "false"); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) +#define assert_b_ne(a, b, fmt...) do { \ + bool a_ = (a); \ + bool b_ = (b); \ + if (!(a_ != b_)) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) != (%s) --> %s == %s: ", \ + __func__, __FILE__, __LINE__, \ + #a, #b, a_ ? "true" : "false", \ + b_ ? "true" : "false"); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) +#define assert_true(a, fmt...) assert_b_eq(a, true, fmt) +#define assert_false(a, fmt...) assert_b_eq(a, false, fmt) + +#define assert_str_eq(a, b, fmt...) do { \ + if (strcmp((a), (b))) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) same as (%s) --> " \ + "\"%s\" differs from \"%s\": ", \ + __func__, __FILE__, __LINE__, #a, #b, a, b); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) +#define assert_str_ne(a, b, fmt...) do { \ + if (!strcmp((a), (b))) { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Failed assertion: " \ + "(%s) differs from (%s) --> " \ + "\"%s\" same as \"%s\": ", \ + __func__, __FILE__, __LINE__, #a, #b, a, b); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ + } \ +} while (0) + +#define assert_not_reached(fmt...) do { \ + char prefix[ASSERT_BUFSIZE]; \ + char message[ASSERT_BUFSIZE]; \ + malloc_snprintf(prefix, sizeof(prefix), \ + "%s:%s:%d: Unreachable code reached: ", \ + __func__, __FILE__, __LINE__); \ + malloc_snprintf(message, sizeof(message), fmt); \ + p_test_fail(prefix, message); \ +} while (0) + +/* + * If this enum changes, corresponding changes in test/test.sh.in are also + * necessary. + */ +typedef enum { + test_status_pass = 0, + test_status_skip = 1, + test_status_fail = 2, + + test_status_count = 3 +} test_status_t; + +typedef void (test_t)(void); + +#define TEST_BEGIN(f) \ +static void \ +f(void) \ +{ \ + p_test_init(#f); + +#define TEST_END \ + goto label_test_end; \ +label_test_end: \ + p_test_fini(); \ +} + +#define test(tests...) \ + p_test(tests, NULL) + +#define test_skip_if(e) do { \ + if (e) { \ + test_skip("%s:%s:%d: Test skipped: (%s)", \ + __func__, __FILE__, __LINE__, #e); \ + goto label_test_end; \ + } \ +} while (0) + +void test_skip(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2)); +void test_fail(const char *format, ...) JEMALLOC_ATTR(format(printf, 1, 2)); + +/* For private use by macros. */ +test_status_t p_test(test_t* t, ...); +void p_test_init(const char *name); +void p_test_fini(void); +void p_test_fail(const char *prefix, const char *message); diff --git a/deps/jemalloc/test/include/test/thd.h b/deps/jemalloc/test/include/test/thd.h new file mode 100644 index 00000000000..f941d7a752f --- /dev/null +++ b/deps/jemalloc/test/include/test/thd.h @@ -0,0 +1,9 @@ +/* Abstraction layer for threading in tests */ +#ifdef _WIN32 +typedef HANDLE thd_t; +#else +typedef pthread_t thd_t; +#endif + +void thd_create(thd_t *thd, void *(*proc)(void *), void *arg); +void thd_join(thd_t thd, void **ret); diff --git a/deps/jemalloc/test/integration/MALLOCX_ARENA.c b/deps/jemalloc/test/integration/MALLOCX_ARENA.c new file mode 100644 index 00000000000..71cf6f255e8 --- /dev/null +++ b/deps/jemalloc/test/integration/MALLOCX_ARENA.c @@ -0,0 +1,58 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 10 + +void * +thd_start(void *arg) +{ + unsigned thread_ind = (unsigned)(uintptr_t)arg; + unsigned arena_ind; + void *p; + size_t sz; + + sz = sizeof(arena_ind); + assert_d_eq(mallctl("arenas.extend", &arena_ind, &sz, NULL, 0), 0, + "Error in arenas.extend"); + + if (thread_ind % 4 != 3) { + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(size_t); + const char *dss_precs[] = {"disabled", "primary", "secondary"}; + const char *dss = dss_precs[thread_ind % + (sizeof(dss_precs)/sizeof(char*))]; + assert_d_eq(mallctlnametomib("arena.0.dss", mib, &miblen), 0, + "Error in mallctlnametomib()"); + mib[1] = arena_ind; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, (void *)&dss, + sizeof(const char *)), 0, "Error in mallctlbymib()"); + } + + p = mallocx(1, MALLOCX_ARENA(arena_ind)); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + dallocx(p, 0); + + return (NULL); +} + +TEST_BEGIN(test_ALLOCM_ARENA) +{ + thd_t thds[NTHREADS]; + unsigned i; + + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_start, + (void *)(uintptr_t)i); + } + + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); +} +TEST_END + +int +main(void) +{ + + return (test( + test_ALLOCM_ARENA)); +} diff --git a/deps/jemalloc/test/aligned_alloc.c b/deps/jemalloc/test/integration/aligned_alloc.c similarity index 53% rename from deps/jemalloc/test/aligned_alloc.c rename to deps/jemalloc/test/integration/aligned_alloc.c index 5a9b0caea78..609001487b8 100644 --- a/deps/jemalloc/test/aligned_alloc.c +++ b/deps/jemalloc/test/integration/aligned_alloc.c @@ -1,39 +1,36 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" +#include "test/jemalloc_test.h" -#define CHUNK 0x400000 +#define CHUNK 0x400000 /* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ -#define MAXALIGN ((size_t)0x2000000LU) -#define NITER 4 +#define MAXALIGN ((size_t)0x2000000LU) +#define NITER 4 -int -main(void) +TEST_BEGIN(test_alignment_errors) { - size_t alignment, size, total; - unsigned i; - void *p, *ps[NITER]; - - malloc_printf("Test begin\n"); + size_t alignment; + void *p; - /* Test error conditions. */ alignment = 0; set_errno(0); p = aligned_alloc(alignment, 1); - if (p != NULL || get_errno() != EINVAL) { - malloc_printf( - "Expected error for invalid alignment %zu\n", alignment); - } + assert_false(p != NULL || get_errno() != EINVAL, + "Expected error for invalid alignment %zu", alignment); for (alignment = sizeof(size_t); alignment < MAXALIGN; alignment <<= 1) { set_errno(0); p = aligned_alloc(alignment + 1, 1); - if (p != NULL || get_errno() != EINVAL) { - malloc_printf( - "Expected error for invalid alignment %zu\n", - alignment + 1); - } + assert_false(p != NULL || get_errno() != EINVAL, + "Expected error for invalid alignment %zu", + alignment + 1); } +} +TEST_END + +TEST_BEGIN(test_oom_errors) +{ + size_t alignment, size; + void *p; #if LG_SIZEOF_PTR == 3 alignment = UINT64_C(0x8000000000000000); @@ -44,26 +41,22 @@ main(void) #endif set_errno(0); p = aligned_alloc(alignment, size); - if (p != NULL || get_errno() != ENOMEM) { - malloc_printf( - "Expected error for aligned_alloc(%zu, %zu)\n", - alignment, size); - } + assert_false(p != NULL || get_errno() != ENOMEM, + "Expected error for aligned_alloc(%zu, %zu)", + alignment, size); #if LG_SIZEOF_PTR == 3 alignment = UINT64_C(0x4000000000000000); - size = UINT64_C(0x8400000000000001); + size = UINT64_C(0xc000000000000001); #else alignment = 0x40000000LU; - size = 0x84000001LU; + size = 0xc0000001LU; #endif set_errno(0); p = aligned_alloc(alignment, size); - if (p != NULL || get_errno() != ENOMEM) { - malloc_printf( - "Expected error for aligned_alloc(%zu, %zu)\n", - alignment, size); - } + assert_false(p != NULL || get_errno() != ENOMEM, + "Expected error for aligned_alloc(%zu, %zu)", + alignment, size); alignment = 0x10LU; #if LG_SIZEOF_PTR == 3 @@ -73,11 +66,17 @@ main(void) #endif set_errno(0); p = aligned_alloc(alignment, size); - if (p != NULL || get_errno() != ENOMEM) { - malloc_printf( - "Expected error for aligned_alloc(&p, %zu, %zu)\n", - alignment, size); - } + assert_false(p != NULL || get_errno() != ENOMEM, + "Expected error for aligned_alloc(&p, %zu, %zu)", + alignment, size); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t alignment, size, total; + unsigned i; + void *ps[NITER]; for (i = 0; i < NITER; i++) ps[i] = NULL; @@ -86,7 +85,6 @@ main(void) alignment <= MAXALIGN; alignment <<= 1) { total = 0; - malloc_printf("Alignment: %zu\n", alignment); for (size = 1; size < 3 * alignment && size < (1U << 31); size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { @@ -95,11 +93,11 @@ main(void) if (ps[i] == NULL) { char buf[BUFERROR_BUF]; - buferror(buf, sizeof(buf)); - malloc_printf( - "Error for size %zu (%#zx): %s\n", - size, size, buf); - exit(1); + buferror(get_errno(), buf, sizeof(buf)); + test_fail( + "Error for alignment=%zu, " + "size=%zu (%#zx): %s", + alignment, size, size, buf); } total += malloc_usable_size(ps[i]); if (total >= (MAXALIGN << 1)) @@ -113,7 +111,15 @@ main(void) } } } +} +TEST_END + +int +main(void) +{ - malloc_printf("Test end\n"); - return (0); + return (test( + test_alignment_errors, + test_oom_errors, + test_alignment_and_size)); } diff --git a/deps/jemalloc/test/integration/allocated.c b/deps/jemalloc/test/integration/allocated.c new file mode 100644 index 00000000000..3630e80ce25 --- /dev/null +++ b/deps/jemalloc/test/integration/allocated.c @@ -0,0 +1,125 @@ +#include "test/jemalloc_test.h" + +static const bool config_stats = +#ifdef JEMALLOC_STATS + true +#else + false +#endif + ; + +void * +thd_start(void *arg) +{ + int err; + void *p; + uint64_t a0, a1, d0, d1; + uint64_t *ap0, *ap1, *dp0, *dp1; + size_t sz, usize; + + sz = sizeof(a0); + if ((err = mallctl("thread.allocated", &a0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + sz = sizeof(ap0); + if ((err = mallctl("thread.allocatedp", &ap0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + assert_u64_eq(*ap0, a0, + "\"thread.allocatedp\" should provide a pointer to internal " + "storage"); + + sz = sizeof(d0); + if ((err = mallctl("thread.deallocated", &d0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + sz = sizeof(dp0); + if ((err = mallctl("thread.deallocatedp", &dp0, &sz, NULL, 0))) { + if (err == ENOENT) + goto label_ENOENT; + test_fail("%s(): Error in mallctl(): %s", __func__, + strerror(err)); + } + assert_u64_eq(*dp0, d0, + "\"thread.deallocatedp\" should provide a pointer to internal " + "storage"); + + p = malloc(1); + assert_ptr_not_null(p, "Unexpected malloc() error"); + + sz = sizeof(a1); + mallctl("thread.allocated", &a1, &sz, NULL, 0); + sz = sizeof(ap1); + mallctl("thread.allocatedp", &ap1, &sz, NULL, 0); + assert_u64_eq(*ap1, a1, + "Dereferenced \"thread.allocatedp\" value should equal " + "\"thread.allocated\" value"); + assert_ptr_eq(ap0, ap1, + "Pointer returned by \"thread.allocatedp\" should not change"); + + usize = malloc_usable_size(p); + assert_u64_le(a0 + usize, a1, + "Allocated memory counter should increase by at least the amount " + "explicitly allocated"); + + free(p); + + sz = sizeof(d1); + mallctl("thread.deallocated", &d1, &sz, NULL, 0); + sz = sizeof(dp1); + mallctl("thread.deallocatedp", &dp1, &sz, NULL, 0); + assert_u64_eq(*dp1, d1, + "Dereferenced \"thread.deallocatedp\" value should equal " + "\"thread.deallocated\" value"); + assert_ptr_eq(dp0, dp1, + "Pointer returned by \"thread.deallocatedp\" should not change"); + + assert_u64_le(d0 + usize, d1, + "Deallocated memory counter should increase by at least the amount " + "explicitly deallocated"); + + return (NULL); +label_ENOENT: + assert_false(config_stats, + "ENOENT should only be returned if stats are disabled"); + test_skip("\"thread.allocated\" mallctl not available"); + return (NULL); +} + +TEST_BEGIN(test_main_thread) +{ + + thd_start(NULL); +} +TEST_END + +TEST_BEGIN(test_subthread) +{ + thd_t thd; + + thd_create(&thd, thd_start, NULL); + thd_join(thd, NULL); +} +TEST_END + +int +main(void) +{ + + /* Run tests multiple times to check for bad interactions. */ + return (test( + test_main_thread, + test_subthread, + test_main_thread, + test_subthread, + test_main_thread)); +} diff --git a/deps/jemalloc/test/integration/allocm.c b/deps/jemalloc/test/integration/allocm.c new file mode 100644 index 00000000000..7b4ea0c2c65 --- /dev/null +++ b/deps/jemalloc/test/integration/allocm.c @@ -0,0 +1,107 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +#define MAXALIGN (((size_t)1) << 25) +#define NITER 4 + +TEST_BEGIN(test_basic) +{ + size_t nsz, rsz, sz; + void *p; + + sz = 42; + nsz = 0; + assert_d_eq(nallocm(&nsz, sz, 0), ALLOCM_SUCCESS, + "Unexpected nallocm() error"); + rsz = 0; + assert_d_eq(allocm(&p, &rsz, sz, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch"); + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); + + assert_d_eq(allocm(&p, NULL, sz, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); + + nsz = 0; + assert_d_eq(nallocm(&nsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS, + "Unexpected nallocm() error"); + rsz = 0; + assert_d_eq(allocm(&p, &rsz, sz, ALLOCM_ZERO), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + assert_zu_eq(nsz, rsz, "nallocm()/allocm() rsize mismatch"); + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + int r; + size_t nsz, rsz, sz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = 0; + r = nallocm(&nsz, sz, ALLOCM_ALIGN(alignment) | + ALLOCM_ZERO); + assert_d_eq(r, ALLOCM_SUCCESS, + "nallocm() error for alignment=%zu, " + "size=%zu (%#zx): %d", + alignment, sz, sz, r); + rsz = 0; + r = allocm(&ps[i], &rsz, sz, + ALLOCM_ALIGN(alignment) | ALLOCM_ZERO); + assert_d_eq(r, ALLOCM_SUCCESS, + "allocm() error for alignment=%zu, " + "size=%zu (%#zx): %d", + alignment, sz, sz, r); + assert_zu_ge(rsz, sz, + "Real size smaller than expected for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, rsz, + "nallocm()/allocm() rsize mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_ptr_null( + (void *)((uintptr_t)ps[i] & (alignment-1)), + "%p inadequately aligned for" + " alignment=%zu, size=%zu", ps[i], + alignment, sz); + sallocm(ps[i], &rsz, 0); + total += rsz; + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + dallocm(ps[i], 0); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_basic, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/mallocx.c b/deps/jemalloc/test/integration/mallocx.c new file mode 100644 index 00000000000..123e041fa33 --- /dev/null +++ b/deps/jemalloc/test/integration/mallocx.c @@ -0,0 +1,97 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +#define MAXALIGN (((size_t)1) << 25) +#define NITER 4 + +TEST_BEGIN(test_basic) +{ + size_t nsz, rsz, sz; + void *p; + + sz = 42; + nsz = nallocx(sz, 0); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + p = mallocx(sz, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + rsz = sallocx(p, 0); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); + dallocx(p, 0); + + p = mallocx(sz, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + dallocx(p, 0); + + nsz = nallocx(sz, MALLOCX_ZERO); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + p = mallocx(sz, MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + rsz = sallocx(p, 0); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t nsz, rsz, sz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + assert_zu_ne(nsz, 0, + "nallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + assert_ptr_not_null(ps[i], + "mallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + rsz = sallocx(ps[i], 0); + assert_zu_ge(rsz, sz, + "Real size smaller than expected for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, rsz, + "nallocx()/sallocx() size mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_ptr_null( + (void *)((uintptr_t)ps[i] & (alignment-1)), + "%p inadequately aligned for" + " alignment=%zu, size=%zu", ps[i], + alignment, sz); + total += rsz; + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + dallocx(ps[i], 0); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_basic, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/mremap.c b/deps/jemalloc/test/integration/mremap.c new file mode 100644 index 00000000000..a7fb7ef0abf --- /dev/null +++ b/deps/jemalloc/test/integration/mremap.c @@ -0,0 +1,45 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_mremap) +{ + int err; + size_t sz, lg_chunk, chunksize, i; + char *p, *q; + + sz = sizeof(lg_chunk); + err = mallctl("opt.lg_chunk", &lg_chunk, &sz, NULL, 0); + assert_d_eq(err, 0, "Error in mallctl(): %s", strerror(err)); + chunksize = ((size_t)1U) << lg_chunk; + + p = (char *)malloc(chunksize); + assert_ptr_not_null(p, "malloc(%zu) --> %p", chunksize, p); + memset(p, 'a', chunksize); + + q = (char *)realloc(p, chunksize * 2); + assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize * 2, + q); + for (i = 0; i < chunksize; i++) { + assert_c_eq(q[i], 'a', + "realloc() should preserve existing bytes across copies"); + } + + p = q; + + q = (char *)realloc(p, chunksize); + assert_ptr_not_null(q, "realloc(%p, %zu) --> %p", p, chunksize, q); + for (i = 0; i < chunksize; i++) { + assert_c_eq(q[i], 'a', + "realloc() should preserve existing bytes across copies"); + } + + free(q); +} +TEST_END + +int +main(void) +{ + + return (test( + test_mremap)); +} diff --git a/deps/jemalloc/test/integration/posix_memalign.c b/deps/jemalloc/test/integration/posix_memalign.c new file mode 100644 index 00000000000..19741c6cb50 --- /dev/null +++ b/deps/jemalloc/test/integration/posix_memalign.c @@ -0,0 +1,119 @@ +#include "test/jemalloc_test.h" + +#define CHUNK 0x400000 +/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ +#define MAXALIGN ((size_t)0x2000000LU) +#define NITER 4 + +TEST_BEGIN(test_alignment_errors) +{ + size_t alignment; + void *p; + + for (alignment = 0; alignment < sizeof(void *); alignment++) { + assert_d_eq(posix_memalign(&p, alignment, 1), EINVAL, + "Expected error for invalid alignment %zu", + alignment); + } + + for (alignment = sizeof(size_t); alignment < MAXALIGN; + alignment <<= 1) { + assert_d_ne(posix_memalign(&p, alignment + 1, 1), 0, + "Expected error for invalid alignment %zu", + alignment + 1); + } +} +TEST_END + +TEST_BEGIN(test_oom_errors) +{ + size_t alignment, size; + void *p; + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x8000000000000000); + size = UINT64_C(0x8000000000000000); +#else + alignment = 0x80000000LU; + size = 0x80000000LU; +#endif + assert_d_ne(posix_memalign(&p, alignment, size), 0, + "Expected error for posix_memalign(&p, %zu, %zu)", + alignment, size); + +#if LG_SIZEOF_PTR == 3 + alignment = UINT64_C(0x4000000000000000); + size = UINT64_C(0xc000000000000001); +#else + alignment = 0x40000000LU; + size = 0xc0000001LU; +#endif + assert_d_ne(posix_memalign(&p, alignment, size), 0, + "Expected error for posix_memalign(&p, %zu, %zu)", + alignment, size); + + alignment = 0x10LU; +#if LG_SIZEOF_PTR == 3 + size = UINT64_C(0xfffffffffffffff0); +#else + size = 0xfffffff0LU; +#endif + assert_d_ne(posix_memalign(&p, alignment, size), 0, + "Expected error for posix_memalign(&p, %zu, %zu)", + alignment, size); +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) +{ + size_t alignment, size, total; + unsigned i; + int err; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) + ps[i] = NULL; + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (size = 1; + size < 3 * alignment && size < (1U << 31); + size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + err = posix_memalign(&ps[i], + alignment, size); + if (err) { + char buf[BUFERROR_BUF]; + + buferror(get_errno(), buf, sizeof(buf)); + test_fail( + "Error for alignment=%zu, " + "size=%zu (%#zx): %s", + alignment, size, size, buf); + } + total += malloc_usable_size(ps[i]); + if (total >= (MAXALIGN << 1)) + break; + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + free(ps[i]); + ps[i] = NULL; + } + } + } + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_alignment_errors, + test_oom_errors, + test_alignment_and_size)); +} diff --git a/deps/jemalloc/test/integration/rallocm.c b/deps/jemalloc/test/integration/rallocm.c new file mode 100644 index 00000000000..33c11bb7cf6 --- /dev/null +++ b/deps/jemalloc/test/integration/rallocm.c @@ -0,0 +1,111 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_same_size) +{ + void *p, *q; + size_t sz, tsz; + + assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + q = p; + assert_d_eq(rallocm(&q, &tsz, sz, 0, ALLOCM_NO_MOVE), ALLOCM_SUCCESS, + "Unexpected rallocm() error"); + assert_ptr_eq(q, p, "Unexpected object move"); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_extra_no_move) +{ + void *p, *q; + size_t sz, tsz; + + assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + q = p; + assert_d_eq(rallocm(&q, &tsz, sz, sz-42, ALLOCM_NO_MOVE), + ALLOCM_SUCCESS, "Unexpected rallocm() error"); + assert_ptr_eq(q, p, "Unexpected object move"); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_no_move_fail) +{ + void *p, *q; + size_t sz, tsz; + + assert_d_eq(allocm(&p, &sz, 42, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + q = p; + assert_d_eq(rallocm(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE), + ALLOCM_ERR_NOT_MOVED, "Unexpected rallocm() result"); + assert_ptr_eq(q, p, "Unexpected object move"); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +TEST_BEGIN(test_grow_and_shrink) +{ + void *p, *q; + size_t tsz; +#define NCYCLES 3 + unsigned i, j; +#define NSZS 2500 + size_t szs[NSZS]; +#define MAXSZ ZU(12 * 1024 * 1024) + + assert_d_eq(allocm(&p, &szs[0], 1, 0), ALLOCM_SUCCESS, + "Unexpected allocm() error"); + + for (i = 0; i < NCYCLES; i++) { + for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) { + q = p; + assert_d_eq(rallocm(&q, &szs[j], szs[j-1]+1, 0, 0), + ALLOCM_SUCCESS, + "Unexpected rallocm() error for size=%zu-->%zu", + szs[j-1], szs[j-1]+1); + assert_zu_ne(szs[j], szs[j-1]+1, + "Expected size to at least: %zu", szs[j-1]+1); + p = q; + } + + for (j--; j > 0; j--) { + q = p; + assert_d_eq(rallocm(&q, &tsz, szs[j-1], 0, 0), + ALLOCM_SUCCESS, + "Unexpected rallocm() error for size=%zu-->%zu", + szs[j], szs[j-1]); + assert_zu_eq(tsz, szs[j-1], + "Expected size=%zu, got size=%zu", szs[j-1], tsz); + p = q; + } + } + + assert_d_eq(dallocm(p, 0), ALLOCM_SUCCESS, + "Unexpected dallocm() error"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_same_size, + test_extra_no_move, + test_no_move_fail, + test_grow_and_shrink)); +} diff --git a/deps/jemalloc/test/integration/rallocx.c b/deps/jemalloc/test/integration/rallocx.c new file mode 100644 index 00000000000..ee21aedff70 --- /dev/null +++ b/deps/jemalloc/test/integration/rallocx.c @@ -0,0 +1,182 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_grow_and_shrink) +{ + void *p, *q; + size_t tsz; +#define NCYCLES 3 + unsigned i, j; +#define NSZS 2500 + size_t szs[NSZS]; +#define MAXSZ ZU(12 * 1024 * 1024) + + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + szs[0] = sallocx(p, 0); + + for (i = 0; i < NCYCLES; i++) { + for (j = 1; j < NSZS && szs[j-1] < MAXSZ; j++) { + q = rallocx(p, szs[j-1]+1, 0); + assert_ptr_not_null(q, + "Unexpected rallocx() error for size=%zu-->%zu", + szs[j-1], szs[j-1]+1); + szs[j] = sallocx(q, 0); + assert_zu_ne(szs[j], szs[j-1]+1, + "Expected size to at least: %zu", szs[j-1]+1); + p = q; + } + + for (j--; j > 0; j--) { + q = rallocx(p, szs[j-1], 0); + assert_ptr_not_null(q, + "Unexpected rallocx() error for size=%zu-->%zu", + szs[j], szs[j-1]); + tsz = sallocx(q, 0); + assert_zu_eq(tsz, szs[j-1], + "Expected size=%zu, got size=%zu", szs[j-1], tsz); + p = q; + } + } + + dallocx(p, 0); +#undef MAXSZ +#undef NSZS +#undef NCYCLES +} +TEST_END + +static bool +validate_fill(const void *p, uint8_t c, size_t offset, size_t len) +{ + bool ret = false; + const uint8_t *buf = (const uint8_t *)p; + size_t i; + + for (i = 0; i < len; i++) { + uint8_t b = buf[offset+i]; + if (b != c) { + test_fail("Allocation at %p contains %#x rather than " + "%#x at offset %zu", p, b, c, offset+i); + ret = true; + } + } + + return (ret); +} + +TEST_BEGIN(test_zero) +{ + void *p, *q; + size_t psz, qsz, i, j; + size_t start_sizes[] = {1, 3*1024, 63*1024, 4095*1024}; +#define FILL_BYTE 0xaaU +#define RANGE 2048 + + for (i = 0; i < sizeof(start_sizes)/sizeof(size_t); i++) { + size_t start_size = start_sizes[i]; + p = mallocx(start_size, MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + psz = sallocx(p, 0); + + assert_false(validate_fill(p, 0, 0, psz), + "Expected zeroed memory"); + memset(p, FILL_BYTE, psz); + assert_false(validate_fill(p, FILL_BYTE, 0, psz), + "Expected filled memory"); + + for (j = 1; j < RANGE; j++) { + q = rallocx(p, start_size+j, MALLOCX_ZERO); + assert_ptr_not_null(q, "Unexpected rallocx() error"); + qsz = sallocx(q, 0); + if (q != p || qsz != psz) { + assert_false(validate_fill(q, FILL_BYTE, 0, + psz), "Expected filled memory"); + assert_false(validate_fill(q, 0, psz, qsz-psz), + "Expected zeroed memory"); + } + if (psz != qsz) { + memset(q+psz, FILL_BYTE, qsz-psz); + psz = qsz; + } + p = q; + } + assert_false(validate_fill(p, FILL_BYTE, 0, psz), + "Expected filled memory"); + dallocx(p, 0); + } +#undef FILL_BYTE +} +TEST_END + +TEST_BEGIN(test_align) +{ + void *p, *q; + size_t align; +#define MAX_ALIGN (ZU(1) << 25) + + align = ZU(1); + p = mallocx(1, MALLOCX_ALIGN(align)); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + for (align <<= 1; align <= MAX_ALIGN; align <<= 1) { + q = rallocx(p, 1, MALLOCX_ALIGN(align)); + assert_ptr_not_null(q, + "Unexpected rallocx() error for align=%zu", align); + assert_ptr_null( + (void *)((uintptr_t)q & (align-1)), + "%p inadequately aligned for align=%zu", + q, align); + p = q; + } + dallocx(p, 0); +#undef MAX_ALIGN +} +TEST_END + +TEST_BEGIN(test_lg_align_and_zero) +{ + void *p, *q; + size_t lg_align, sz; +#define MAX_LG_ALIGN 25 +#define MAX_VALIDATE (ZU(1) << 22) + + lg_align = ZU(0); + p = mallocx(1, MALLOCX_LG_ALIGN(lg_align)|MALLOCX_ZERO); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + + for (lg_align++; lg_align <= MAX_LG_ALIGN; lg_align++) { + q = rallocx(p, 1, MALLOCX_LG_ALIGN(lg_align)|MALLOCX_ZERO); + assert_ptr_not_null(q, + "Unexpected rallocx() error for lg_align=%zu", lg_align); + assert_ptr_null( + (void *)((uintptr_t)q & ((ZU(1) << lg_align)-1)), + "%p inadequately aligned for lg_align=%zu", + q, lg_align); + sz = sallocx(q, 0); + if ((sz << 1) <= MAX_VALIDATE) { + assert_false(validate_fill(q, 0, 0, sz), + "Expected zeroed memory"); + } else { + assert_false(validate_fill(q, 0, 0, MAX_VALIDATE), + "Expected zeroed memory"); + assert_false(validate_fill(q+sz-MAX_VALIDATE, 0, 0, + MAX_VALIDATE), "Expected zeroed memory"); + } + p = q; + } + dallocx(p, 0); +#undef MAX_VALIDATE +#undef MAX_LG_ALIGN +} +TEST_END + +int +main(void) +{ + + return (test( + test_grow_and_shrink, + test_zero, + test_align, + test_lg_align_and_zero)); +} diff --git a/deps/jemalloc/test/integration/thread_arena.c b/deps/jemalloc/test/integration/thread_arena.c new file mode 100644 index 00000000000..67be5351335 --- /dev/null +++ b/deps/jemalloc/test/integration/thread_arena.c @@ -0,0 +1,79 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 10 + +void * +thd_start(void *arg) +{ + unsigned main_arena_ind = *(unsigned *)arg; + void *p; + unsigned arena_ind; + size_t size; + int err; + + p = malloc(1); + assert_ptr_not_null(p, "Error in malloc()"); + free(p); + + size = sizeof(arena_ind); + if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind, + sizeof(main_arena_ind)))) { + char buf[BUFERROR_BUF]; + + buferror(err, buf, sizeof(buf)); + test_fail("Error in mallctl(): %s", buf); + } + + size = sizeof(arena_ind); + if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { + char buf[BUFERROR_BUF]; + + buferror(err, buf, sizeof(buf)); + test_fail("Error in mallctl(): %s", buf); + } + assert_u_eq(arena_ind, main_arena_ind, + "Arena index should be same as for main thread"); + + return (NULL); +} + +TEST_BEGIN(test_thread_arena) +{ + void *p; + unsigned arena_ind; + size_t size; + int err; + thd_t thds[NTHREADS]; + unsigned i; + + p = malloc(1); + assert_ptr_not_null(p, "Error in malloc()"); + + size = sizeof(arena_ind); + if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { + char buf[BUFERROR_BUF]; + + buferror(err, buf, sizeof(buf)); + test_fail("Error in mallctl(): %s", buf); + } + + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_start, + (void *)&arena_ind); + } + + for (i = 0; i < NTHREADS; i++) { + intptr_t join_ret; + thd_join(thds[i], (void *)&join_ret); + assert_zd_eq(join_ret, 0, "Unexpected thread join error"); + } +} +TEST_END + +int +main(void) +{ + + return (test( + test_thread_arena)); +} diff --git a/deps/jemalloc/test/integration/thread_tcache_enabled.c b/deps/jemalloc/test/integration/thread_tcache_enabled.c new file mode 100644 index 00000000000..f4e89c682a7 --- /dev/null +++ b/deps/jemalloc/test/integration/thread_tcache_enabled.c @@ -0,0 +1,113 @@ +#include "test/jemalloc_test.h" + +static const bool config_tcache = +#ifdef JEMALLOC_TCACHE + true +#else + false +#endif + ; + +void * +thd_start(void *arg) +{ + int err; + size_t sz; + bool e0, e1; + + sz = sizeof(bool); + if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) { + if (err == ENOENT) { + assert_false(config_tcache, + "ENOENT should only be returned if tcache is " + "disabled"); + } + goto label_ENOENT; + } + + if (e0) { + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), + 0, "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + } + + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + free(malloc(1)); + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + free(malloc(1)); + e1 = true; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + free(malloc(1)); + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_true(e0, "tcache should be enabled"); + + free(malloc(1)); + e1 = false; + assert_d_eq(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz), 0, + "Unexpected mallctl() error"); + assert_false(e0, "tcache should be disabled"); + + free(malloc(1)); + return (NULL); +label_ENOENT: + test_skip("\"thread.tcache.enabled\" mallctl not available"); + return (NULL); +} + +TEST_BEGIN(test_main_thread) +{ + + thd_start(NULL); +} +TEST_END + +TEST_BEGIN(test_subthread) +{ + thd_t thd; + + thd_create(&thd, thd_start, NULL); + thd_join(thd, NULL); +} +TEST_END + +int +main(void) +{ + + /* Run tests multiple times to check for bad interactions. */ + return (test( + test_main_thread, + test_subthread, + test_main_thread, + test_subthread, + test_main_thread)); +} diff --git a/deps/jemalloc/test/integration/xallocx.c b/deps/jemalloc/test/integration/xallocx.c new file mode 100644 index 00000000000..ab4cf945e54 --- /dev/null +++ b/deps/jemalloc/test/integration/xallocx.c @@ -0,0 +1,59 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_same_size) +{ + void *p; + size_t sz, tsz; + + p = mallocx(42, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + sz = sallocx(p, 0); + + tsz = xallocx(p, sz, 0, 0); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_extra_no_move) +{ + void *p; + size_t sz, tsz; + + p = mallocx(42, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + sz = sallocx(p, 0); + + tsz = xallocx(p, sz, sz-42, 0); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_no_move_fail) +{ + void *p; + size_t sz, tsz; + + p = mallocx(42, 0); + assert_ptr_not_null(p, "Unexpected mallocx() error"); + sz = sallocx(p, 0); + + tsz = xallocx(p, sz + 5, 0, 0); + assert_zu_eq(tsz, sz, "Unexpected size change: %zu --> %zu", sz, tsz); + + dallocx(p, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_same_size, + test_extra_no_move, + test_no_move_fail)); +} diff --git a/deps/jemalloc/test/jemalloc_test.h.in b/deps/jemalloc/test/jemalloc_test.h.in deleted file mode 100644 index e38b48efa41..00000000000 --- a/deps/jemalloc/test/jemalloc_test.h.in +++ /dev/null @@ -1,53 +0,0 @@ -/* - * This header should be included by tests, rather than directly including - * jemalloc/jemalloc.h, because --with-install-suffix may cause the header to - * have a different name. - */ -#include "jemalloc/jemalloc@install_suffix@.h" -#include "jemalloc/internal/jemalloc_internal.h" - -/* Abstraction layer for threading in tests */ -#ifdef _WIN32 -#include - -typedef HANDLE je_thread_t; - -void -je_thread_create(je_thread_t *thread, void *(*proc)(void *), void *arg) -{ - LPTHREAD_START_ROUTINE routine = (LPTHREAD_START_ROUTINE)proc; - *thread = CreateThread(NULL, 0, routine, arg, 0, NULL); - if (*thread == NULL) { - malloc_printf("Error in CreateThread()\n"); - exit(1); - } -} - -void -je_thread_join(je_thread_t thread, void **ret) -{ - WaitForSingleObject(thread, INFINITE); -} - -#else -#include - -typedef pthread_t je_thread_t; - -void -je_thread_create(je_thread_t *thread, void *(*proc)(void *), void *arg) -{ - - if (pthread_create(thread, NULL, proc, arg) != 0) { - malloc_printf("Error in pthread_create()\n"); - exit(1); - } -} - -void -je_thread_join(je_thread_t thread, void **ret) -{ - - pthread_join(thread, ret); -} -#endif diff --git a/deps/jemalloc/test/mremap.c b/deps/jemalloc/test/mremap.c deleted file mode 100644 index 47efa7c415b..00000000000 --- a/deps/jemalloc/test/mremap.c +++ /dev/null @@ -1,60 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -int -main(void) -{ - int ret, err; - size_t sz, lg_chunk, chunksize, i; - char *p, *q; - - malloc_printf("Test begin\n"); - - sz = sizeof(lg_chunk); - if ((err = mallctl("opt.lg_chunk", &lg_chunk, &sz, NULL, 0))) { - assert(err != ENOENT); - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - ret = 1; - goto label_return; - } - chunksize = ((size_t)1U) << lg_chunk; - - p = (char *)malloc(chunksize); - if (p == NULL) { - malloc_printf("malloc(%zu) --> %p\n", chunksize, p); - ret = 1; - goto label_return; - } - memset(p, 'a', chunksize); - - q = (char *)realloc(p, chunksize * 2); - if (q == NULL) { - malloc_printf("realloc(%p, %zu) --> %p\n", p, chunksize * 2, - q); - ret = 1; - goto label_return; - } - for (i = 0; i < chunksize; i++) { - assert(q[i] == 'a'); - } - - p = q; - - q = (char *)realloc(p, chunksize); - if (q == NULL) { - malloc_printf("realloc(%p, %zu) --> %p\n", p, chunksize, q); - ret = 1; - goto label_return; - } - for (i = 0; i < chunksize; i++) { - assert(q[i] == 'a'); - } - - free(q); - - ret = 0; -label_return: - malloc_printf("Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/mremap.exp b/deps/jemalloc/test/mremap.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/mremap.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/posix_memalign.c b/deps/jemalloc/test/posix_memalign.c deleted file mode 100644 index 2185bcf762a..00000000000 --- a/deps/jemalloc/test/posix_memalign.c +++ /dev/null @@ -1,115 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define CHUNK 0x400000 -/* #define MAXALIGN ((size_t)UINT64_C(0x80000000000)) */ -#define MAXALIGN ((size_t)0x2000000LU) -#define NITER 4 - -int -main(void) -{ - size_t alignment, size, total; - unsigned i; - int err; - void *p, *ps[NITER]; - - malloc_printf("Test begin\n"); - - /* Test error conditions. */ - for (alignment = 0; alignment < sizeof(void *); alignment++) { - err = posix_memalign(&p, alignment, 1); - if (err != EINVAL) { - malloc_printf( - "Expected error for invalid alignment %zu\n", - alignment); - } - } - - for (alignment = sizeof(size_t); alignment < MAXALIGN; - alignment <<= 1) { - err = posix_memalign(&p, alignment + 1, 1); - if (err == 0) { - malloc_printf( - "Expected error for invalid alignment %zu\n", - alignment + 1); - } - } - -#if LG_SIZEOF_PTR == 3 - alignment = UINT64_C(0x8000000000000000); - size = UINT64_C(0x8000000000000000); -#else - alignment = 0x80000000LU; - size = 0x80000000LU; -#endif - err = posix_memalign(&p, alignment, size); - if (err == 0) { - malloc_printf( - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - -#if LG_SIZEOF_PTR == 3 - alignment = UINT64_C(0x4000000000000000); - size = UINT64_C(0x8400000000000001); -#else - alignment = 0x40000000LU; - size = 0x84000001LU; -#endif - err = posix_memalign(&p, alignment, size); - if (err == 0) { - malloc_printf( - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - - alignment = 0x10LU; -#if LG_SIZEOF_PTR == 3 - size = UINT64_C(0xfffffffffffffff0); -#else - size = 0xfffffff0LU; -#endif - err = posix_memalign(&p, alignment, size); - if (err == 0) { - malloc_printf( - "Expected error for posix_memalign(&p, %zu, %zu)\n", - alignment, size); - } - - for (i = 0; i < NITER; i++) - ps[i] = NULL; - - for (alignment = 8; - alignment <= MAXALIGN; - alignment <<= 1) { - total = 0; - malloc_printf("Alignment: %zu\n", alignment); - for (size = 1; - size < 3 * alignment && size < (1U << 31); - size += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { - for (i = 0; i < NITER; i++) { - err = posix_memalign(&ps[i], - alignment, size); - if (err) { - malloc_printf( - "Error for size %zu (%#zx): %s\n", - size, size, strerror(err)); - exit(1); - } - total += malloc_usable_size(ps[i]); - if (total >= (MAXALIGN << 1)) - break; - } - for (i = 0; i < NITER; i++) { - if (ps[i] != NULL) { - free(ps[i]); - ps[i] = NULL; - } - } - } - } - - malloc_printf("Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/posix_memalign.exp b/deps/jemalloc/test/posix_memalign.exp deleted file mode 100644 index b5061c7277e..00000000000 --- a/deps/jemalloc/test/posix_memalign.exp +++ /dev/null @@ -1,25 +0,0 @@ -Test begin -Alignment: 8 -Alignment: 16 -Alignment: 32 -Alignment: 64 -Alignment: 128 -Alignment: 256 -Alignment: 512 -Alignment: 1024 -Alignment: 2048 -Alignment: 4096 -Alignment: 8192 -Alignment: 16384 -Alignment: 32768 -Alignment: 65536 -Alignment: 131072 -Alignment: 262144 -Alignment: 524288 -Alignment: 1048576 -Alignment: 2097152 -Alignment: 4194304 -Alignment: 8388608 -Alignment: 16777216 -Alignment: 33554432 -Test end diff --git a/deps/jemalloc/test/rallocm.c b/deps/jemalloc/test/rallocm.c deleted file mode 100644 index c5dedf48d7b..00000000000 --- a/deps/jemalloc/test/rallocm.c +++ /dev/null @@ -1,127 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -int -main(void) -{ - size_t pagesize; - void *p, *q; - size_t sz, tsz; - int r; - - malloc_printf("Test begin\n"); - - /* Get page size. */ - { -#ifdef _WIN32 - SYSTEM_INFO si; - GetSystemInfo(&si); - pagesize = (size_t)si.dwPageSize; -#else - long result = sysconf(_SC_PAGESIZE); - assert(result != -1); - pagesize = (size_t)result; -#endif - } - - r = allocm(&p, &sz, 42, 0); - if (r != ALLOCM_SUCCESS) { - malloc_printf("Unexpected allocm() error\n"); - abort(); - } - - q = p; - r = rallocm(&q, &tsz, sz, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (q != p) - malloc_printf("Unexpected object move\n"); - if (tsz != sz) { - malloc_printf("Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = rallocm(&q, &tsz, sz, 5, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (q != p) - malloc_printf("Unexpected object move\n"); - if (tsz != sz) { - malloc_printf("Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = rallocm(&q, &tsz, sz + 5, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_ERR_NOT_MOVED) - malloc_printf("Unexpected rallocm() result\n"); - if (q != p) - malloc_printf("Unexpected object move\n"); - if (tsz != sz) { - malloc_printf("Unexpected size change: %zu --> %zu\n", - sz, tsz); - } - - q = p; - r = rallocm(&q, &tsz, sz + 5, 0, 0); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (q == p) - malloc_printf("Expected object move\n"); - if (tsz == sz) { - malloc_printf("Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = rallocm(&q, &tsz, pagesize*2, 0, 0); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (q == p) - malloc_printf("Expected object move\n"); - if (tsz == sz) { - malloc_printf("Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = rallocm(&q, &tsz, pagesize*4, 0, 0); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (tsz == sz) { - malloc_printf("Expected size change: %zu --> %zu\n", - sz, tsz); - } - p = q; - sz = tsz; - - r = rallocm(&q, &tsz, pagesize*2, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (q != p) - malloc_printf("Unexpected object move\n"); - if (tsz == sz) { - malloc_printf("Expected size change: %zu --> %zu\n", - sz, tsz); - } - sz = tsz; - - r = rallocm(&q, &tsz, pagesize*4, 0, ALLOCM_NO_MOVE); - if (r != ALLOCM_SUCCESS) - malloc_printf("Unexpected rallocm() error\n"); - if (q != p) - malloc_printf("Unexpected object move\n"); - if (tsz == sz) { - malloc_printf("Expected size change: %zu --> %zu\n", - sz, tsz); - } - sz = tsz; - - dallocm(p, 0); - - malloc_printf("Test end\n"); - return (0); -} diff --git a/deps/jemalloc/test/rallocm.exp b/deps/jemalloc/test/rallocm.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/rallocm.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/src/SFMT.c b/deps/jemalloc/test/src/SFMT.c new file mode 100644 index 00000000000..e6f8deecb37 --- /dev/null +++ b/deps/jemalloc/test/src/SFMT.c @@ -0,0 +1,719 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +/** + * @file SFMT.c + * @brief SIMD oriented Fast Mersenne Twister(SFMT) + * + * @author Mutsuo Saito (Hiroshima University) + * @author Makoto Matsumoto (Hiroshima University) + * + * Copyright (C) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * The new BSD License is applied to this software, see LICENSE.txt + */ +#define SFMT_C_ +#include "test/jemalloc_test.h" +#include "test/SFMT-params.h" + +#if defined(JEMALLOC_BIG_ENDIAN) && !defined(BIG_ENDIAN64) +#define BIG_ENDIAN64 1 +#endif +#if defined(__BIG_ENDIAN__) && !defined(__amd64) && !defined(BIG_ENDIAN64) +#define BIG_ENDIAN64 1 +#endif +#if defined(HAVE_ALTIVEC) && !defined(BIG_ENDIAN64) +#define BIG_ENDIAN64 1 +#endif +#if defined(ONLY64) && !defined(BIG_ENDIAN64) + #if defined(__GNUC__) + #error "-DONLY64 must be specified with -DBIG_ENDIAN64" + #endif +#undef ONLY64 +#endif +/*------------------------------------------------------ + 128-bit SIMD data type for Altivec, SSE2 or standard C + ------------------------------------------------------*/ +#if defined(HAVE_ALTIVEC) +/** 128-bit data structure */ +union W128_T { + vector unsigned int s; + uint32_t u[4]; +}; +/** 128-bit data type */ +typedef union W128_T w128_t; + +#elif defined(HAVE_SSE2) +/** 128-bit data structure */ +union W128_T { + __m128i si; + uint32_t u[4]; +}; +/** 128-bit data type */ +typedef union W128_T w128_t; + +#else + +/** 128-bit data structure */ +struct W128_T { + uint32_t u[4]; +}; +/** 128-bit data type */ +typedef struct W128_T w128_t; + +#endif + +struct sfmt_s { + /** the 128-bit internal state array */ + w128_t sfmt[N]; + /** index counter to the 32-bit internal state array */ + int idx; + /** a flag: it is 0 if and only if the internal state is not yet + * initialized. */ + int initialized; +}; + +/*-------------------------------------- + FILE GLOBAL VARIABLES + internal state, index counter and flag + --------------------------------------*/ + +/** a parity check vector which certificate the period of 2^{MEXP} */ +static uint32_t parity[4] = {PARITY1, PARITY2, PARITY3, PARITY4}; + +/*---------------- + STATIC FUNCTIONS + ----------------*/ +JEMALLOC_INLINE_C int idxof(int i); +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +JEMALLOC_INLINE_C void rshift128(w128_t *out, w128_t const *in, int shift); +JEMALLOC_INLINE_C void lshift128(w128_t *out, w128_t const *in, int shift); +#endif +JEMALLOC_INLINE_C void gen_rand_all(sfmt_t *ctx); +JEMALLOC_INLINE_C void gen_rand_array(sfmt_t *ctx, w128_t *array, int size); +JEMALLOC_INLINE_C uint32_t func1(uint32_t x); +JEMALLOC_INLINE_C uint32_t func2(uint32_t x); +static void period_certification(sfmt_t *ctx); +#if defined(BIG_ENDIAN64) && !defined(ONLY64) +JEMALLOC_INLINE_C void swap(w128_t *array, int size); +#endif + +#if defined(HAVE_ALTIVEC) + #include "test/SFMT-alti.h" +#elif defined(HAVE_SSE2) + #include "test/SFMT-sse2.h" +#endif + +/** + * This function simulate a 64-bit index of LITTLE ENDIAN + * in BIG ENDIAN machine. + */ +#ifdef ONLY64 +JEMALLOC_INLINE_C int idxof(int i) { + return i ^ 1; +} +#else +JEMALLOC_INLINE_C int idxof(int i) { + return i; +} +#endif +/** + * This function simulates SIMD 128-bit right shift by the standard C. + * The 128-bit integer given in in is shifted by (shift * 8) bits. + * This function simulates the LITTLE ENDIAN SIMD. + * @param out the output of this function + * @param in the 128-bit data to be shifted + * @param shift the shift value + */ +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +#ifdef ONLY64 +JEMALLOC_INLINE_C void rshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[2] << 32) | ((uint64_t)in->u[3]); + tl = ((uint64_t)in->u[0] << 32) | ((uint64_t)in->u[1]); + + oh = th >> (shift * 8); + ol = tl >> (shift * 8); + ol |= th << (64 - shift * 8); + out->u[0] = (uint32_t)(ol >> 32); + out->u[1] = (uint32_t)ol; + out->u[2] = (uint32_t)(oh >> 32); + out->u[3] = (uint32_t)oh; +} +#else +JEMALLOC_INLINE_C void rshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[3] << 32) | ((uint64_t)in->u[2]); + tl = ((uint64_t)in->u[1] << 32) | ((uint64_t)in->u[0]); + + oh = th >> (shift * 8); + ol = tl >> (shift * 8); + ol |= th << (64 - shift * 8); + out->u[1] = (uint32_t)(ol >> 32); + out->u[0] = (uint32_t)ol; + out->u[3] = (uint32_t)(oh >> 32); + out->u[2] = (uint32_t)oh; +} +#endif +/** + * This function simulates SIMD 128-bit left shift by the standard C. + * The 128-bit integer given in in is shifted by (shift * 8) bits. + * This function simulates the LITTLE ENDIAN SIMD. + * @param out the output of this function + * @param in the 128-bit data to be shifted + * @param shift the shift value + */ +#ifdef ONLY64 +JEMALLOC_INLINE_C void lshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[2] << 32) | ((uint64_t)in->u[3]); + tl = ((uint64_t)in->u[0] << 32) | ((uint64_t)in->u[1]); + + oh = th << (shift * 8); + ol = tl << (shift * 8); + oh |= tl >> (64 - shift * 8); + out->u[0] = (uint32_t)(ol >> 32); + out->u[1] = (uint32_t)ol; + out->u[2] = (uint32_t)(oh >> 32); + out->u[3] = (uint32_t)oh; +} +#else +JEMALLOC_INLINE_C void lshift128(w128_t *out, w128_t const *in, int shift) { + uint64_t th, tl, oh, ol; + + th = ((uint64_t)in->u[3] << 32) | ((uint64_t)in->u[2]); + tl = ((uint64_t)in->u[1] << 32) | ((uint64_t)in->u[0]); + + oh = th << (shift * 8); + ol = tl << (shift * 8); + oh |= tl >> (64 - shift * 8); + out->u[1] = (uint32_t)(ol >> 32); + out->u[0] = (uint32_t)ol; + out->u[3] = (uint32_t)(oh >> 32); + out->u[2] = (uint32_t)oh; +} +#endif +#endif + +/** + * This function represents the recursion formula. + * @param r output + * @param a a 128-bit part of the internal state array + * @param b a 128-bit part of the internal state array + * @param c a 128-bit part of the internal state array + * @param d a 128-bit part of the internal state array + */ +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +#ifdef ONLY64 +JEMALLOC_INLINE_C void do_recursion(w128_t *r, w128_t *a, w128_t *b, w128_t *c, + w128_t *d) { + w128_t x; + w128_t y; + + lshift128(&x, a, SL2); + rshift128(&y, c, SR2); + r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK2) ^ y.u[0] + ^ (d->u[0] << SL1); + r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK1) ^ y.u[1] + ^ (d->u[1] << SL1); + r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK4) ^ y.u[2] + ^ (d->u[2] << SL1); + r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK3) ^ y.u[3] + ^ (d->u[3] << SL1); +} +#else +JEMALLOC_INLINE_C void do_recursion(w128_t *r, w128_t *a, w128_t *b, w128_t *c, + w128_t *d) { + w128_t x; + w128_t y; + + lshift128(&x, a, SL2); + rshift128(&y, c, SR2); + r->u[0] = a->u[0] ^ x.u[0] ^ ((b->u[0] >> SR1) & MSK1) ^ y.u[0] + ^ (d->u[0] << SL1); + r->u[1] = a->u[1] ^ x.u[1] ^ ((b->u[1] >> SR1) & MSK2) ^ y.u[1] + ^ (d->u[1] << SL1); + r->u[2] = a->u[2] ^ x.u[2] ^ ((b->u[2] >> SR1) & MSK3) ^ y.u[2] + ^ (d->u[2] << SL1); + r->u[3] = a->u[3] ^ x.u[3] ^ ((b->u[3] >> SR1) & MSK4) ^ y.u[3] + ^ (d->u[3] << SL1); +} +#endif +#endif + +#if (!defined(HAVE_ALTIVEC)) && (!defined(HAVE_SSE2)) +/** + * This function fills the internal state array with pseudorandom + * integers. + */ +JEMALLOC_INLINE_C void gen_rand_all(sfmt_t *ctx) { + int i; + w128_t *r1, *r2; + + r1 = &ctx->sfmt[N - 2]; + r2 = &ctx->sfmt[N - 1]; + for (i = 0; i < N - POS1; i++) { + do_recursion(&ctx->sfmt[i], &ctx->sfmt[i], &ctx->sfmt[i + POS1], r1, + r2); + r1 = r2; + r2 = &ctx->sfmt[i]; + } + for (; i < N; i++) { + do_recursion(&ctx->sfmt[i], &ctx->sfmt[i], &ctx->sfmt[i + POS1 - N], r1, + r2); + r1 = r2; + r2 = &ctx->sfmt[i]; + } +} + +/** + * This function fills the user-specified array with pseudorandom + * integers. + * + * @param array an 128-bit array to be filled by pseudorandom numbers. + * @param size number of 128-bit pseudorandom numbers to be generated. + */ +JEMALLOC_INLINE_C void gen_rand_array(sfmt_t *ctx, w128_t *array, int size) { + int i, j; + w128_t *r1, *r2; + + r1 = &ctx->sfmt[N - 2]; + r2 = &ctx->sfmt[N - 1]; + for (i = 0; i < N - POS1; i++) { + do_recursion(&array[i], &ctx->sfmt[i], &ctx->sfmt[i + POS1], r1, r2); + r1 = r2; + r2 = &array[i]; + } + for (; i < N; i++) { + do_recursion(&array[i], &ctx->sfmt[i], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + } + for (; i < size - N; i++) { + do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + } + for (j = 0; j < 2 * N - size; j++) { + ctx->sfmt[j] = array[j + size - N]; + } + for (; i < size; i++, j++) { + do_recursion(&array[i], &array[i - N], &array[i + POS1 - N], r1, r2); + r1 = r2; + r2 = &array[i]; + ctx->sfmt[j] = array[i]; + } +} +#endif + +#if defined(BIG_ENDIAN64) && !defined(ONLY64) && !defined(HAVE_ALTIVEC) +JEMALLOC_INLINE_C void swap(w128_t *array, int size) { + int i; + uint32_t x, y; + + for (i = 0; i < size; i++) { + x = array[i].u[0]; + y = array[i].u[2]; + array[i].u[0] = array[i].u[1]; + array[i].u[2] = array[i].u[3]; + array[i].u[1] = x; + array[i].u[3] = y; + } +} +#endif +/** + * This function represents a function used in the initialization + * by init_by_array + * @param x 32-bit integer + * @return 32-bit integer + */ +static uint32_t func1(uint32_t x) { + return (x ^ (x >> 27)) * (uint32_t)1664525UL; +} + +/** + * This function represents a function used in the initialization + * by init_by_array + * @param x 32-bit integer + * @return 32-bit integer + */ +static uint32_t func2(uint32_t x) { + return (x ^ (x >> 27)) * (uint32_t)1566083941UL; +} + +/** + * This function certificate the period of 2^{MEXP} + */ +static void period_certification(sfmt_t *ctx) { + int inner = 0; + int i, j; + uint32_t work; + uint32_t *psfmt32 = &ctx->sfmt[0].u[0]; + + for (i = 0; i < 4; i++) + inner ^= psfmt32[idxof(i)] & parity[i]; + for (i = 16; i > 0; i >>= 1) + inner ^= inner >> i; + inner &= 1; + /* check OK */ + if (inner == 1) { + return; + } + /* check NG, and modification */ + for (i = 0; i < 4; i++) { + work = 1; + for (j = 0; j < 32; j++) { + if ((work & parity[i]) != 0) { + psfmt32[idxof(i)] ^= work; + return; + } + work = work << 1; + } + } +} + +/*---------------- + PUBLIC FUNCTIONS + ----------------*/ +/** + * This function returns the identification string. + * The string shows the word size, the Mersenne exponent, + * and all parameters of this generator. + */ +const char *get_idstring(void) { + return IDSTR; +} + +/** + * This function returns the minimum size of array used for \b + * fill_array32() function. + * @return minimum size of array used for fill_array32() function. + */ +int get_min_array_size32(void) { + return N32; +} + +/** + * This function returns the minimum size of array used for \b + * fill_array64() function. + * @return minimum size of array used for fill_array64() function. + */ +int get_min_array_size64(void) { + return N64; +} + +#ifndef ONLY64 +/** + * This function generates and returns 32-bit pseudorandom number. + * init_gen_rand or init_by_array must be called before this function. + * @return 32-bit pseudorandom number + */ +uint32_t gen_rand32(sfmt_t *ctx) { + uint32_t r; + uint32_t *psfmt32 = &ctx->sfmt[0].u[0]; + + assert(ctx->initialized); + if (ctx->idx >= N32) { + gen_rand_all(ctx); + ctx->idx = 0; + } + r = psfmt32[ctx->idx++]; + return r; +} + +/* Generate a random integer in [0..limit). */ +uint32_t gen_rand32_range(sfmt_t *ctx, uint32_t limit) { + uint32_t ret, above; + + above = 0xffffffffU - (0xffffffffU % limit); + while (1) { + ret = gen_rand32(ctx); + if (ret < above) { + ret %= limit; + break; + } + } + return ret; +} +#endif +/** + * This function generates and returns 64-bit pseudorandom number. + * init_gen_rand or init_by_array must be called before this function. + * The function gen_rand64 should not be called after gen_rand32, + * unless an initialization is again executed. + * @return 64-bit pseudorandom number + */ +uint64_t gen_rand64(sfmt_t *ctx) { +#if defined(BIG_ENDIAN64) && !defined(ONLY64) + uint32_t r1, r2; + uint32_t *psfmt32 = &ctx->sfmt[0].u[0]; +#else + uint64_t r; + uint64_t *psfmt64 = (uint64_t *)&ctx->sfmt[0].u[0]; +#endif + + assert(ctx->initialized); + assert(ctx->idx % 2 == 0); + + if (ctx->idx >= N32) { + gen_rand_all(ctx); + ctx->idx = 0; + } +#if defined(BIG_ENDIAN64) && !defined(ONLY64) + r1 = psfmt32[ctx->idx]; + r2 = psfmt32[ctx->idx + 1]; + ctx->idx += 2; + return ((uint64_t)r2 << 32) | r1; +#else + r = psfmt64[ctx->idx / 2]; + ctx->idx += 2; + return r; +#endif +} + +/* Generate a random integer in [0..limit). */ +uint64_t gen_rand64_range(sfmt_t *ctx, uint64_t limit) { + uint64_t ret, above; + + above = 0xffffffffffffffffLLU - (0xffffffffffffffffLLU % limit); + while (1) { + ret = gen_rand64(ctx); + if (ret < above) { + ret %= limit; + break; + } + } + return ret; +} + +#ifndef ONLY64 +/** + * This function generates pseudorandom 32-bit integers in the + * specified array[] by one call. The number of pseudorandom integers + * is specified by the argument size, which must be at least 624 and a + * multiple of four. The generation by this function is much faster + * than the following gen_rand function. + * + * For initialization, init_gen_rand or init_by_array must be called + * before the first call of this function. This function can not be + * used after calling gen_rand function, without initialization. + * + * @param array an array where pseudorandom 32-bit integers are filled + * by this function. The pointer to the array must be \b "aligned" + * (namely, must be a multiple of 16) in the SIMD version, since it + * refers to the address of a 128-bit integer. In the standard C + * version, the pointer is arbitrary. + * + * @param size the number of 32-bit pseudorandom integers to be + * generated. size must be a multiple of 4, and greater than or equal + * to (MEXP / 128 + 1) * 4. + * + * @note \b memalign or \b posix_memalign is available to get aligned + * memory. Mac OSX doesn't have these functions, but \b malloc of OSX + * returns the pointer to the aligned memory block. + */ +void fill_array32(sfmt_t *ctx, uint32_t *array, int size) { + assert(ctx->initialized); + assert(ctx->idx == N32); + assert(size % 4 == 0); + assert(size >= N32); + + gen_rand_array(ctx, (w128_t *)array, size / 4); + ctx->idx = N32; +} +#endif + +/** + * This function generates pseudorandom 64-bit integers in the + * specified array[] by one call. The number of pseudorandom integers + * is specified by the argument size, which must be at least 312 and a + * multiple of two. The generation by this function is much faster + * than the following gen_rand function. + * + * For initialization, init_gen_rand or init_by_array must be called + * before the first call of this function. This function can not be + * used after calling gen_rand function, without initialization. + * + * @param array an array where pseudorandom 64-bit integers are filled + * by this function. The pointer to the array must be "aligned" + * (namely, must be a multiple of 16) in the SIMD version, since it + * refers to the address of a 128-bit integer. In the standard C + * version, the pointer is arbitrary. + * + * @param size the number of 64-bit pseudorandom integers to be + * generated. size must be a multiple of 2, and greater than or equal + * to (MEXP / 128 + 1) * 2 + * + * @note \b memalign or \b posix_memalign is available to get aligned + * memory. Mac OSX doesn't have these functions, but \b malloc of OSX + * returns the pointer to the aligned memory block. + */ +void fill_array64(sfmt_t *ctx, uint64_t *array, int size) { + assert(ctx->initialized); + assert(ctx->idx == N32); + assert(size % 2 == 0); + assert(size >= N64); + + gen_rand_array(ctx, (w128_t *)array, size / 2); + ctx->idx = N32; + +#if defined(BIG_ENDIAN64) && !defined(ONLY64) + swap((w128_t *)array, size /2); +#endif +} + +/** + * This function initializes the internal state array with a 32-bit + * integer seed. + * + * @param seed a 32-bit integer used as the seed. + */ +sfmt_t *init_gen_rand(uint32_t seed) { + void *p; + sfmt_t *ctx; + int i; + uint32_t *psfmt32; + + if (posix_memalign(&p, sizeof(w128_t), sizeof(sfmt_t)) != 0) { + return NULL; + } + ctx = (sfmt_t *)p; + psfmt32 = &ctx->sfmt[0].u[0]; + + psfmt32[idxof(0)] = seed; + for (i = 1; i < N32; i++) { + psfmt32[idxof(i)] = 1812433253UL * (psfmt32[idxof(i - 1)] + ^ (psfmt32[idxof(i - 1)] >> 30)) + + i; + } + ctx->idx = N32; + period_certification(ctx); + ctx->initialized = 1; + + return ctx; +} + +/** + * This function initializes the internal state array, + * with an array of 32-bit integers used as the seeds + * @param init_key the array of 32-bit integers, used as a seed. + * @param key_length the length of init_key. + */ +sfmt_t *init_by_array(uint32_t *init_key, int key_length) { + void *p; + sfmt_t *ctx; + int i, j, count; + uint32_t r; + int lag; + int mid; + int size = N * 4; + uint32_t *psfmt32; + + if (posix_memalign(&p, sizeof(w128_t), sizeof(sfmt_t)) != 0) { + return NULL; + } + ctx = (sfmt_t *)p; + psfmt32 = &ctx->sfmt[0].u[0]; + + if (size >= 623) { + lag = 11; + } else if (size >= 68) { + lag = 7; + } else if (size >= 39) { + lag = 5; + } else { + lag = 3; + } + mid = (size - lag) / 2; + + memset(ctx->sfmt, 0x8b, sizeof(ctx->sfmt)); + if (key_length + 1 > N32) { + count = key_length + 1; + } else { + count = N32; + } + r = func1(psfmt32[idxof(0)] ^ psfmt32[idxof(mid)] + ^ psfmt32[idxof(N32 - 1)]); + psfmt32[idxof(mid)] += r; + r += key_length; + psfmt32[idxof(mid + lag)] += r; + psfmt32[idxof(0)] = r; + + count--; + for (i = 1, j = 0; (j < count) && (j < key_length); j++) { + r = func1(psfmt32[idxof(i)] ^ psfmt32[idxof((i + mid) % N32)] + ^ psfmt32[idxof((i + N32 - 1) % N32)]); + psfmt32[idxof((i + mid) % N32)] += r; + r += init_key[j] + i; + psfmt32[idxof((i + mid + lag) % N32)] += r; + psfmt32[idxof(i)] = r; + i = (i + 1) % N32; + } + for (; j < count; j++) { + r = func1(psfmt32[idxof(i)] ^ psfmt32[idxof((i + mid) % N32)] + ^ psfmt32[idxof((i + N32 - 1) % N32)]); + psfmt32[idxof((i + mid) % N32)] += r; + r += i; + psfmt32[idxof((i + mid + lag) % N32)] += r; + psfmt32[idxof(i)] = r; + i = (i + 1) % N32; + } + for (j = 0; j < N32; j++) { + r = func2(psfmt32[idxof(i)] + psfmt32[idxof((i + mid) % N32)] + + psfmt32[idxof((i + N32 - 1) % N32)]); + psfmt32[idxof((i + mid) % N32)] ^= r; + r -= i; + psfmt32[idxof((i + mid + lag) % N32)] ^= r; + psfmt32[idxof(i)] = r; + i = (i + 1) % N32; + } + + ctx->idx = N32; + period_certification(ctx); + ctx->initialized = 1; + + return ctx; +} + +void fini_gen_rand(sfmt_t *ctx) { + assert(ctx != NULL); + + ctx->initialized = 0; + free(ctx); +} diff --git a/deps/jemalloc/test/src/math.c b/deps/jemalloc/test/src/math.c new file mode 100644 index 00000000000..887a36390e4 --- /dev/null +++ b/deps/jemalloc/test/src/math.c @@ -0,0 +1,2 @@ +#define MATH_C_ +#include "test/jemalloc_test.h" diff --git a/deps/jemalloc/test/src/mtx.c b/deps/jemalloc/test/src/mtx.c new file mode 100644 index 00000000000..41b95d59de8 --- /dev/null +++ b/deps/jemalloc/test/src/mtx.c @@ -0,0 +1,62 @@ +#include "test/jemalloc_test.h" + +bool +mtx_init(mtx_t *mtx) +{ + +#ifdef _WIN32 + if (!InitializeCriticalSectionAndSpinCount(&mtx->lock, _CRT_SPINCOUNT)) + return (true); +#elif (defined(JEMALLOC_OSSPIN)) + mtx->lock = 0; +#else + pthread_mutexattr_t attr; + + if (pthread_mutexattr_init(&attr) != 0) + return (true); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_DEFAULT); + if (pthread_mutex_init(&mtx->lock, &attr) != 0) { + pthread_mutexattr_destroy(&attr); + return (true); + } + pthread_mutexattr_destroy(&attr); +#endif + return (false); +} + +void +mtx_fini(mtx_t *mtx) +{ + +#ifdef _WIN32 +#elif (defined(JEMALLOC_OSSPIN)) +#else + pthread_mutex_destroy(&mtx->lock); +#endif +} + +void +mtx_lock(mtx_t *mtx) +{ + +#ifdef _WIN32 + EnterCriticalSection(&mtx->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockLock(&mtx->lock); +#else + pthread_mutex_lock(&mtx->lock); +#endif +} + +void +mtx_unlock(mtx_t *mtx) +{ + +#ifdef _WIN32 + LeaveCriticalSection(&mtx->lock); +#elif (defined(JEMALLOC_OSSPIN)) + OSSpinLockUnlock(&mtx->lock); +#else + pthread_mutex_unlock(&mtx->lock); +#endif +} diff --git a/deps/jemalloc/test/src/test.c b/deps/jemalloc/test/src/test.c new file mode 100644 index 00000000000..528d858317a --- /dev/null +++ b/deps/jemalloc/test/src/test.c @@ -0,0 +1,94 @@ +#include "test/jemalloc_test.h" + +static unsigned test_count = 0; +static test_status_t test_counts[test_status_count] = {0, 0, 0}; +static test_status_t test_status = test_status_pass; +static const char * test_name = ""; + +JEMALLOC_ATTR(format(printf, 1, 2)) +void +test_skip(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); + malloc_printf("\n"); + test_status = test_status_skip; +} + +JEMALLOC_ATTR(format(printf, 1, 2)) +void +test_fail(const char *format, ...) +{ + va_list ap; + + va_start(ap, format); + malloc_vcprintf(NULL, NULL, format, ap); + va_end(ap); + malloc_printf("\n"); + test_status = test_status_fail; +} + +static const char * +test_status_string(test_status_t test_status) +{ + + switch (test_status) { + case test_status_pass: return "pass"; + case test_status_skip: return "skip"; + case test_status_fail: return "fail"; + default: not_reached(); + } +} + +void +p_test_init(const char *name) +{ + + test_count++; + test_status = test_status_pass; + test_name = name; +} + +void +p_test_fini(void) +{ + + test_counts[test_status]++; + malloc_printf("%s: %s\n", test_name, test_status_string(test_status)); +} + +test_status_t +p_test(test_t* t, ...) +{ + test_status_t ret = test_status_pass; + va_list ap; + + va_start(ap, t); + for (; t != NULL; t = va_arg(ap, test_t*)) { + t(); + if (test_status > ret) + ret = test_status; + } + va_end(ap); + + malloc_printf("--- %s: %u/%u, %s: %u/%u, %s: %u/%u ---\n", + test_status_string(test_status_pass), + test_counts[test_status_pass], test_count, + test_status_string(test_status_skip), + test_counts[test_status_skip], test_count, + test_status_string(test_status_fail), + test_counts[test_status_fail], test_count); + + return (ret); +} + +void +p_test_fail(const char *prefix, const char *message) +{ + + malloc_cprintf(NULL, NULL, "%s%s\n", prefix, message); + test_status = test_status_fail; +} diff --git a/deps/jemalloc/test/src/thd.c b/deps/jemalloc/test/src/thd.c new file mode 100644 index 00000000000..233242a161e --- /dev/null +++ b/deps/jemalloc/test/src/thd.c @@ -0,0 +1,35 @@ +#include "test/jemalloc_test.h" + +#ifdef _WIN32 +void +thd_create(thd_t *thd, void *(*proc)(void *), void *arg) +{ + LPTHREAD_START_ROUTINE routine = (LPTHREAD_START_ROUTINE)proc; + *thd = CreateThread(NULL, 0, routine, arg, 0, NULL); + if (*thd == NULL) + test_fail("Error in CreateThread()\n"); +} + +void +thd_join(thd_t thd, void **ret) +{ + + WaitForSingleObject(thd, INFINITE); +} + +#else +void +thd_create(thd_t *thd, void *(*proc)(void *), void *arg) +{ + + if (pthread_create(thd, NULL, proc, arg) != 0) + test_fail("Error in pthread_create()\n"); +} + +void +thd_join(thd_t thd, void **ret) +{ + + pthread_join(thd, ret); +} +#endif diff --git a/deps/jemalloc/test/test.sh.in b/deps/jemalloc/test/test.sh.in new file mode 100644 index 00000000000..a39f99f6b54 --- /dev/null +++ b/deps/jemalloc/test/test.sh.in @@ -0,0 +1,53 @@ +#!/bin/sh + +case @abi@ in + macho) + export DYLD_FALLBACK_LIBRARY_PATH="@objroot@lib" + ;; + pecoff) + export PATH="${PATH}:@objroot@lib" + ;; + *) + ;; +esac + +# Corresponds to test_status_t. +pass_code=0 +skip_code=1 +fail_code=2 + +pass_count=0 +skip_count=0 +fail_count=0 +for t in $@; do + if [ $pass_count -ne 0 -o $skip_count -ne 0 -o $fail_count != 0 ] ; then + echo + fi + echo "=== ${t} ===" + ${t}@exe@ @abs_srcroot@ @abs_objroot@ + result_code=$? + case ${result_code} in + ${pass_code}) + pass_count=$((pass_count+1)) + ;; + ${skip_code}) + skip_count=$((skip_count+1)) + ;; + ${fail_code}) + fail_count=$((fail_count+1)) + ;; + *) + echo "Test harness error" 1>&2 + exit 1 + esac +done + +total_count=`expr ${pass_count} + ${skip_count} + ${fail_count}` +echo +echo "Test suite summary: pass: ${pass_count}/${total_count}, skip: ${skip_count}/${total_count}, fail: ${fail_count}/${total_count}" + +if [ ${fail_count} -eq 0 ] ; then + exit 0 +else + exit 1 +fi diff --git a/deps/jemalloc/test/thread_arena.c b/deps/jemalloc/test/thread_arena.c deleted file mode 100644 index 2ffdb5e8004..00000000000 --- a/deps/jemalloc/test/thread_arena.c +++ /dev/null @@ -1,80 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -#define NTHREADS 10 - -void * -je_thread_start(void *arg) -{ - unsigned main_arena_ind = *(unsigned *)arg; - void *p; - unsigned arena_ind; - size_t size; - int err; - - p = malloc(1); - if (p == NULL) { - malloc_printf("%s(): Error in malloc()\n", __func__); - return (void *)1; - } - - size = sizeof(arena_ind); - if ((err = mallctl("thread.arena", &arena_ind, &size, &main_arena_ind, - sizeof(main_arena_ind)))) { - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - return (void *)1; - } - - size = sizeof(arena_ind); - if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, - 0))) { - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - return (void *)1; - } - assert(arena_ind == main_arena_ind); - - return (NULL); -} - -int -main(void) -{ - int ret = 0; - void *p; - unsigned arena_ind; - size_t size; - int err; - je_thread_t threads[NTHREADS]; - unsigned i; - - malloc_printf("Test begin\n"); - - p = malloc(1); - if (p == NULL) { - malloc_printf("%s(): Error in malloc()\n", __func__); - ret = 1; - goto label_return; - } - - size = sizeof(arena_ind); - if ((err = mallctl("thread.arena", &arena_ind, &size, NULL, 0))) { - malloc_printf("%s(): Error in mallctl(): %s\n", __func__, - strerror(err)); - ret = 1; - goto label_return; - } - - for (i = 0; i < NTHREADS; i++) { - je_thread_create(&threads[i], je_thread_start, - (void *)&arena_ind); - } - - for (i = 0; i < NTHREADS; i++) - je_thread_join(threads[i], (void *)&ret); - -label_return: - malloc_printf("Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/thread_arena.exp b/deps/jemalloc/test/thread_arena.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/thread_arena.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/thread_tcache_enabled.c b/deps/jemalloc/test/thread_tcache_enabled.c deleted file mode 100644 index 2061b7bbaff..00000000000 --- a/deps/jemalloc/test/thread_tcache_enabled.c +++ /dev/null @@ -1,91 +0,0 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" - -void * -je_thread_start(void *arg) -{ - int err; - size_t sz; - bool e0, e1; - - sz = sizeof(bool); - if ((err = mallctl("thread.tcache.enabled", &e0, &sz, NULL, 0))) { - if (err == ENOENT) { -#ifdef JEMALLOC_TCACHE - assert(false); -#endif - } - goto label_return; - } - - if (e0) { - e1 = false; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) - == 0); - assert(e0); - } - - e1 = true; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0 == false); - - e1 = true; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0); - - e1 = false; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0); - - e1 = false; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0 == false); - - free(malloc(1)); - e1 = true; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0 == false); - - free(malloc(1)); - e1 = true; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0); - - free(malloc(1)); - e1 = false; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0); - - free(malloc(1)); - e1 = false; - assert(mallctl("thread.tcache.enabled", &e0, &sz, &e1, sz) == 0); - assert(e0 == false); - - free(malloc(1)); -label_return: - return (NULL); -} - -int -main(void) -{ - int ret = 0; - je_thread_t thread; - - malloc_printf("Test begin\n"); - - je_thread_start(NULL); - - je_thread_create(&thread, je_thread_start, NULL); - je_thread_join(thread, (void *)&ret); - - je_thread_start(NULL); - - je_thread_create(&thread, je_thread_start, NULL); - je_thread_join(thread, (void *)&ret); - - je_thread_start(NULL); - - malloc_printf("Test end\n"); - return (ret); -} diff --git a/deps/jemalloc/test/thread_tcache_enabled.exp b/deps/jemalloc/test/thread_tcache_enabled.exp deleted file mode 100644 index 369a88dd240..00000000000 --- a/deps/jemalloc/test/thread_tcache_enabled.exp +++ /dev/null @@ -1,2 +0,0 @@ -Test begin -Test end diff --git a/deps/jemalloc/test/unit/SFMT.c b/deps/jemalloc/test/unit/SFMT.c new file mode 100644 index 00000000000..c57bd68df3d --- /dev/null +++ b/deps/jemalloc/test/unit/SFMT.c @@ -0,0 +1,1605 @@ +/* + * This file derives from SFMT 1.3.3 + * (http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/SFMT/index.html), which was + * released under the terms of the following license: + * + * Copyright (c) 2006,2007 Mutsuo Saito, Makoto Matsumoto and Hiroshima + * University. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * * Neither the name of the Hiroshima University nor the names of + * its contributors may be used to endorse or promote products + * derived from this software without specific prior written + * permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +#include "test/jemalloc_test.h" + +#define BLOCK_SIZE 10000 +#define BLOCK_SIZE64 (BLOCK_SIZE / 2) +#define COUNT_1 1000 +#define COUNT_2 700 + +static const uint32_t init_gen_rand_32_expected[] = { + 3440181298U, 1564997079U, 1510669302U, 2930277156U, 1452439940U, + 3796268453U, 423124208U, 2143818589U, 3827219408U, 2987036003U, + 2674978610U, 1536842514U, 2027035537U, 2534897563U, 1686527725U, + 545368292U, 1489013321U, 1370534252U, 4231012796U, 3994803019U, + 1764869045U, 824597505U, 862581900U, 2469764249U, 812862514U, + 359318673U, 116957936U, 3367389672U, 2327178354U, 1898245200U, + 3206507879U, 2378925033U, 1040214787U, 2524778605U, 3088428700U, + 1417665896U, 964324147U, 2282797708U, 2456269299U, 313400376U, + 2245093271U, 1015729427U, 2694465011U, 3246975184U, 1992793635U, + 463679346U, 3721104591U, 3475064196U, 856141236U, 1499559719U, + 3522818941U, 3721533109U, 1954826617U, 1282044024U, 1543279136U, + 1301863085U, 2669145051U, 4221477354U, 3896016841U, 3392740262U, + 462466863U, 1037679449U, 1228140306U, 922298197U, 1205109853U, + 1872938061U, 3102547608U, 2742766808U, 1888626088U, 4028039414U, + 157593879U, 1136901695U, 4038377686U, 3572517236U, 4231706728U, + 2997311961U, 1189931652U, 3981543765U, 2826166703U, 87159245U, + 1721379072U, 3897926942U, 1790395498U, 2569178939U, 1047368729U, + 2340259131U, 3144212906U, 2301169789U, 2442885464U, 3034046771U, + 3667880593U, 3935928400U, 2372805237U, 1666397115U, 2460584504U, + 513866770U, 3810869743U, 2147400037U, 2792078025U, 2941761810U, + 3212265810U, 984692259U, 346590253U, 1804179199U, 3298543443U, + 750108141U, 2880257022U, 243310542U, 1869036465U, 1588062513U, + 2983949551U, 1931450364U, 4034505847U, 2735030199U, 1628461061U, + 2539522841U, 127965585U, 3992448871U, 913388237U, 559130076U, + 1202933193U, 4087643167U, 2590021067U, 2256240196U, 1746697293U, + 1013913783U, 1155864921U, 2715773730U, 915061862U, 1948766573U, + 2322882854U, 3761119102U, 1343405684U, 3078711943U, 3067431651U, + 3245156316U, 3588354584U, 3484623306U, 3899621563U, 4156689741U, + 3237090058U, 3880063844U, 862416318U, 4039923869U, 2303788317U, + 3073590536U, 701653667U, 2131530884U, 3169309950U, 2028486980U, + 747196777U, 3620218225U, 432016035U, 1449580595U, 2772266392U, + 444224948U, 1662832057U, 3184055582U, 3028331792U, 1861686254U, + 1104864179U, 342430307U, 1350510923U, 3024656237U, 1028417492U, + 2870772950U, 290847558U, 3675663500U, 508431529U, 4264340390U, + 2263569913U, 1669302976U, 519511383U, 2706411211U, 3764615828U, + 3883162495U, 4051445305U, 2412729798U, 3299405164U, 3991911166U, + 2348767304U, 2664054906U, 3763609282U, 593943581U, 3757090046U, + 2075338894U, 2020550814U, 4287452920U, 4290140003U, 1422957317U, + 2512716667U, 2003485045U, 2307520103U, 2288472169U, 3940751663U, + 4204638664U, 2892583423U, 1710068300U, 3904755993U, 2363243951U, + 3038334120U, 547099465U, 771105860U, 3199983734U, 4282046461U, + 2298388363U, 934810218U, 2837827901U, 3952500708U, 2095130248U, + 3083335297U, 26885281U, 3932155283U, 1531751116U, 1425227133U, + 495654159U, 3279634176U, 3855562207U, 3957195338U, 4159985527U, + 893375062U, 1875515536U, 1327247422U, 3754140693U, 1028923197U, + 1729880440U, 805571298U, 448971099U, 2726757106U, 2749436461U, + 2485987104U, 175337042U, 3235477922U, 3882114302U, 2020970972U, + 943926109U, 2762587195U, 1904195558U, 3452650564U, 108432281U, + 3893463573U, 3977583081U, 2636504348U, 1110673525U, 3548479841U, + 4258854744U, 980047703U, 4057175418U, 3890008292U, 145653646U, + 3141868989U, 3293216228U, 1194331837U, 1254570642U, 3049934521U, + 2868313360U, 2886032750U, 1110873820U, 279553524U, 3007258565U, + 1104807822U, 3186961098U, 315764646U, 2163680838U, 3574508994U, + 3099755655U, 191957684U, 3642656737U, 3317946149U, 3522087636U, + 444526410U, 779157624U, 1088229627U, 1092460223U, 1856013765U, + 3659877367U, 368270451U, 503570716U, 3000984671U, 2742789647U, + 928097709U, 2914109539U, 308843566U, 2816161253U, 3667192079U, + 2762679057U, 3395240989U, 2928925038U, 1491465914U, 3458702834U, + 3787782576U, 2894104823U, 1296880455U, 1253636503U, 989959407U, + 2291560361U, 2776790436U, 1913178042U, 1584677829U, 689637520U, + 1898406878U, 688391508U, 3385234998U, 845493284U, 1943591856U, + 2720472050U, 222695101U, 1653320868U, 2904632120U, 4084936008U, + 1080720688U, 3938032556U, 387896427U, 2650839632U, 99042991U, + 1720913794U, 1047186003U, 1877048040U, 2090457659U, 517087501U, + 4172014665U, 2129713163U, 2413533132U, 2760285054U, 4129272496U, + 1317737175U, 2309566414U, 2228873332U, 3889671280U, 1110864630U, + 3576797776U, 2074552772U, 832002644U, 3097122623U, 2464859298U, + 2679603822U, 1667489885U, 3237652716U, 1478413938U, 1719340335U, + 2306631119U, 639727358U, 3369698270U, 226902796U, 2099920751U, + 1892289957U, 2201594097U, 3508197013U, 3495811856U, 3900381493U, + 841660320U, 3974501451U, 3360949056U, 1676829340U, 728899254U, + 2047809627U, 2390948962U, 670165943U, 3412951831U, 4189320049U, + 1911595255U, 2055363086U, 507170575U, 418219594U, 4141495280U, + 2692088692U, 4203630654U, 3540093932U, 791986533U, 2237921051U, + 2526864324U, 2956616642U, 1394958700U, 1983768223U, 1893373266U, + 591653646U, 228432437U, 1611046598U, 3007736357U, 1040040725U, + 2726180733U, 2789804360U, 4263568405U, 829098158U, 3847722805U, + 1123578029U, 1804276347U, 997971319U, 4203797076U, 4185199713U, + 2811733626U, 2343642194U, 2985262313U, 1417930827U, 3759587724U, + 1967077982U, 1585223204U, 1097475516U, 1903944948U, 740382444U, + 1114142065U, 1541796065U, 1718384172U, 1544076191U, 1134682254U, + 3519754455U, 2866243923U, 341865437U, 645498576U, 2690735853U, + 1046963033U, 2493178460U, 1187604696U, 1619577821U, 488503634U, + 3255768161U, 2306666149U, 1630514044U, 2377698367U, 2751503746U, + 3794467088U, 1796415981U, 3657173746U, 409136296U, 1387122342U, + 1297726519U, 219544855U, 4270285558U, 437578827U, 1444698679U, + 2258519491U, 963109892U, 3982244073U, 3351535275U, 385328496U, + 1804784013U, 698059346U, 3920535147U, 708331212U, 784338163U, + 785678147U, 1238376158U, 1557298846U, 2037809321U, 271576218U, + 4145155269U, 1913481602U, 2763691931U, 588981080U, 1201098051U, + 3717640232U, 1509206239U, 662536967U, 3180523616U, 1133105435U, + 2963500837U, 2253971215U, 3153642623U, 1066925709U, 2582781958U, + 3034720222U, 1090798544U, 2942170004U, 4036187520U, 686972531U, + 2610990302U, 2641437026U, 1837562420U, 722096247U, 1315333033U, + 2102231203U, 3402389208U, 3403698140U, 1312402831U, 2898426558U, + 814384596U, 385649582U, 1916643285U, 1924625106U, 2512905582U, + 2501170304U, 4275223366U, 2841225246U, 1467663688U, 3563567847U, + 2969208552U, 884750901U, 102992576U, 227844301U, 3681442994U, + 3502881894U, 4034693299U, 1166727018U, 1697460687U, 1737778332U, + 1787161139U, 1053003655U, 1215024478U, 2791616766U, 2525841204U, + 1629323443U, 3233815U, 2003823032U, 3083834263U, 2379264872U, + 3752392312U, 1287475550U, 3770904171U, 3004244617U, 1502117784U, + 918698423U, 2419857538U, 3864502062U, 1751322107U, 2188775056U, + 4018728324U, 983712955U, 440071928U, 3710838677U, 2001027698U, + 3994702151U, 22493119U, 3584400918U, 3446253670U, 4254789085U, + 1405447860U, 1240245579U, 1800644159U, 1661363424U, 3278326132U, + 3403623451U, 67092802U, 2609352193U, 3914150340U, 1814842761U, + 3610830847U, 591531412U, 3880232807U, 1673505890U, 2585326991U, + 1678544474U, 3148435887U, 3457217359U, 1193226330U, 2816576908U, + 154025329U, 121678860U, 1164915738U, 973873761U, 269116100U, + 52087970U, 744015362U, 498556057U, 94298882U, 1563271621U, + 2383059628U, 4197367290U, 3958472990U, 2592083636U, 2906408439U, + 1097742433U, 3924840517U, 264557272U, 2292287003U, 3203307984U, + 4047038857U, 3820609705U, 2333416067U, 1839206046U, 3600944252U, + 3412254904U, 583538222U, 2390557166U, 4140459427U, 2810357445U, + 226777499U, 2496151295U, 2207301712U, 3283683112U, 611630281U, + 1933218215U, 3315610954U, 3889441987U, 3719454256U, 3957190521U, + 1313998161U, 2365383016U, 3146941060U, 1801206260U, 796124080U, + 2076248581U, 1747472464U, 3254365145U, 595543130U, 3573909503U, + 3758250204U, 2020768540U, 2439254210U, 93368951U, 3155792250U, + 2600232980U, 3709198295U, 3894900440U, 2971850836U, 1578909644U, + 1443493395U, 2581621665U, 3086506297U, 2443465861U, 558107211U, + 1519367835U, 249149686U, 908102264U, 2588765675U, 1232743965U, + 1001330373U, 3561331654U, 2259301289U, 1564977624U, 3835077093U, + 727244906U, 4255738067U, 1214133513U, 2570786021U, 3899704621U, + 1633861986U, 1636979509U, 1438500431U, 58463278U, 2823485629U, + 2297430187U, 2926781924U, 3371352948U, 1864009023U, 2722267973U, + 1444292075U, 437703973U, 1060414512U, 189705863U, 910018135U, + 4077357964U, 884213423U, 2644986052U, 3973488374U, 1187906116U, + 2331207875U, 780463700U, 3713351662U, 3854611290U, 412805574U, + 2978462572U, 2176222820U, 829424696U, 2790788332U, 2750819108U, + 1594611657U, 3899878394U, 3032870364U, 1702887682U, 1948167778U, + 14130042U, 192292500U, 947227076U, 90719497U, 3854230320U, + 784028434U, 2142399787U, 1563449646U, 2844400217U, 819143172U, + 2883302356U, 2328055304U, 1328532246U, 2603885363U, 3375188924U, + 933941291U, 3627039714U, 2129697284U, 2167253953U, 2506905438U, + 1412424497U, 2981395985U, 1418359660U, 2925902456U, 52752784U, + 3713667988U, 3924669405U, 648975707U, 1145520213U, 4018650664U, + 3805915440U, 2380542088U, 2013260958U, 3262572197U, 2465078101U, + 1114540067U, 3728768081U, 2396958768U, 590672271U, 904818725U, + 4263660715U, 700754408U, 1042601829U, 4094111823U, 4274838909U, + 2512692617U, 2774300207U, 2057306915U, 3470942453U, 99333088U, + 1142661026U, 2889931380U, 14316674U, 2201179167U, 415289459U, + 448265759U, 3515142743U, 3254903683U, 246633281U, 1184307224U, + 2418347830U, 2092967314U, 2682072314U, 2558750234U, 2000352263U, + 1544150531U, 399010405U, 1513946097U, 499682937U, 461167460U, + 3045570638U, 1633669705U, 851492362U, 4052801922U, 2055266765U, + 635556996U, 368266356U, 2385737383U, 3218202352U, 2603772408U, + 349178792U, 226482567U, 3102426060U, 3575998268U, 2103001871U, + 3243137071U, 225500688U, 1634718593U, 4283311431U, 4292122923U, + 3842802787U, 811735523U, 105712518U, 663434053U, 1855889273U, + 2847972595U, 1196355421U, 2552150115U, 4254510614U, 3752181265U, + 3430721819U, 3828705396U, 3436287905U, 3441964937U, 4123670631U, + 353001539U, 459496439U, 3799690868U, 1293777660U, 2761079737U, + 498096339U, 3398433374U, 4080378380U, 2304691596U, 2995729055U, + 4134660419U, 3903444024U, 3576494993U, 203682175U, 3321164857U, + 2747963611U, 79749085U, 2992890370U, 1240278549U, 1772175713U, + 2111331972U, 2655023449U, 1683896345U, 2836027212U, 3482868021U, + 2489884874U, 756853961U, 2298874501U, 4013448667U, 4143996022U, + 2948306858U, 4132920035U, 1283299272U, 995592228U, 3450508595U, + 1027845759U, 1766942720U, 3861411826U, 1446861231U, 95974993U, + 3502263554U, 1487532194U, 601502472U, 4129619129U, 250131773U, + 2050079547U, 3198903947U, 3105589778U, 4066481316U, 3026383978U, + 2276901713U, 365637751U, 2260718426U, 1394775634U, 1791172338U, + 2690503163U, 2952737846U, 1568710462U, 732623190U, 2980358000U, + 1053631832U, 1432426951U, 3229149635U, 1854113985U, 3719733532U, + 3204031934U, 735775531U, 107468620U, 3734611984U, 631009402U, + 3083622457U, 4109580626U, 159373458U, 1301970201U, 4132389302U, + 1293255004U, 847182752U, 4170022737U, 96712900U, 2641406755U, + 1381727755U, 405608287U, 4287919625U, 1703554290U, 3589580244U, + 2911403488U, 2166565U, 2647306451U, 2330535117U, 1200815358U, + 1165916754U, 245060911U, 4040679071U, 3684908771U, 2452834126U, + 2486872773U, 2318678365U, 2940627908U, 1837837240U, 3447897409U, + 4270484676U, 1495388728U, 3754288477U, 4204167884U, 1386977705U, + 2692224733U, 3076249689U, 4109568048U, 4170955115U, 4167531356U, + 4020189950U, 4261855038U, 3036907575U, 3410399885U, 3076395737U, + 1046178638U, 144496770U, 230725846U, 3349637149U, 17065717U, + 2809932048U, 2054581785U, 3608424964U, 3259628808U, 134897388U, + 3743067463U, 257685904U, 3795656590U, 1562468719U, 3589103904U, + 3120404710U, 254684547U, 2653661580U, 3663904795U, 2631942758U, + 1063234347U, 2609732900U, 2332080715U, 3521125233U, 1180599599U, + 1935868586U, 4110970440U, 296706371U, 2128666368U, 1319875791U, + 1570900197U, 3096025483U, 1799882517U, 1928302007U, 1163707758U, + 1244491489U, 3533770203U, 567496053U, 2757924305U, 2781639343U, + 2818420107U, 560404889U, 2619609724U, 4176035430U, 2511289753U, + 2521842019U, 3910553502U, 2926149387U, 3302078172U, 4237118867U, + 330725126U, 367400677U, 888239854U, 545570454U, 4259590525U, + 134343617U, 1102169784U, 1647463719U, 3260979784U, 1518840883U, + 3631537963U, 3342671457U, 1301549147U, 2083739356U, 146593792U, + 3217959080U, 652755743U, 2032187193U, 3898758414U, 1021358093U, + 4037409230U, 2176407931U, 3427391950U, 2883553603U, 985613827U, + 3105265092U, 3423168427U, 3387507672U, 467170288U, 2141266163U, + 3723870208U, 916410914U, 1293987799U, 2652584950U, 769160137U, + 3205292896U, 1561287359U, 1684510084U, 3136055621U, 3765171391U, + 639683232U, 2639569327U, 1218546948U, 4263586685U, 3058215773U, + 2352279820U, 401870217U, 2625822463U, 1529125296U, 2981801895U, + 1191285226U, 4027725437U, 3432700217U, 4098835661U, 971182783U, + 2443861173U, 3881457123U, 3874386651U, 457276199U, 2638294160U, + 4002809368U, 421169044U, 1112642589U, 3076213779U, 3387033971U, + 2499610950U, 3057240914U, 1662679783U, 461224431U, 1168395933U +}; +static const uint32_t init_by_array_32_expected[] = { + 2920711183U, 3885745737U, 3501893680U, 856470934U, 1421864068U, + 277361036U, 1518638004U, 2328404353U, 3355513634U, 64329189U, + 1624587673U, 3508467182U, 2481792141U, 3706480799U, 1925859037U, + 2913275699U, 882658412U, 384641219U, 422202002U, 1873384891U, + 2006084383U, 3924929912U, 1636718106U, 3108838742U, 1245465724U, + 4195470535U, 779207191U, 1577721373U, 1390469554U, 2928648150U, + 121399709U, 3170839019U, 4044347501U, 953953814U, 3821710850U, + 3085591323U, 3666535579U, 3577837737U, 2012008410U, 3565417471U, + 4044408017U, 433600965U, 1637785608U, 1798509764U, 860770589U, + 3081466273U, 3982393409U, 2451928325U, 3437124742U, 4093828739U, + 3357389386U, 2154596123U, 496568176U, 2650035164U, 2472361850U, + 3438299U, 2150366101U, 1577256676U, 3802546413U, 1787774626U, + 4078331588U, 3706103141U, 170391138U, 3806085154U, 1680970100U, + 1961637521U, 3316029766U, 890610272U, 1453751581U, 1430283664U, + 3051057411U, 3597003186U, 542563954U, 3796490244U, 1690016688U, + 3448752238U, 440702173U, 347290497U, 1121336647U, 2540588620U, + 280881896U, 2495136428U, 213707396U, 15104824U, 2946180358U, + 659000016U, 566379385U, 2614030979U, 2855760170U, 334526548U, + 2315569495U, 2729518615U, 564745877U, 1263517638U, 3157185798U, + 1604852056U, 1011639885U, 2950579535U, 2524219188U, 312951012U, + 1528896652U, 1327861054U, 2846910138U, 3966855905U, 2536721582U, + 855353911U, 1685434729U, 3303978929U, 1624872055U, 4020329649U, + 3164802143U, 1642802700U, 1957727869U, 1792352426U, 3334618929U, + 2631577923U, 3027156164U, 842334259U, 3353446843U, 1226432104U, + 1742801369U, 3552852535U, 3471698828U, 1653910186U, 3380330939U, + 2313782701U, 3351007196U, 2129839995U, 1800682418U, 4085884420U, + 1625156629U, 3669701987U, 615211810U, 3294791649U, 4131143784U, + 2590843588U, 3207422808U, 3275066464U, 561592872U, 3957205738U, + 3396578098U, 48410678U, 3505556445U, 1005764855U, 3920606528U, + 2936980473U, 2378918600U, 2404449845U, 1649515163U, 701203563U, + 3705256349U, 83714199U, 3586854132U, 922978446U, 2863406304U, + 3523398907U, 2606864832U, 2385399361U, 3171757816U, 4262841009U, + 3645837721U, 1169579486U, 3666433897U, 3174689479U, 1457866976U, + 3803895110U, 3346639145U, 1907224409U, 1978473712U, 1036712794U, + 980754888U, 1302782359U, 1765252468U, 459245755U, 3728923860U, + 1512894209U, 2046491914U, 207860527U, 514188684U, 2288713615U, + 1597354672U, 3349636117U, 2357291114U, 3995796221U, 945364213U, + 1893326518U, 3770814016U, 1691552714U, 2397527410U, 967486361U, + 776416472U, 4197661421U, 951150819U, 1852770983U, 4044624181U, + 1399439738U, 4194455275U, 2284037669U, 1550734958U, 3321078108U, + 1865235926U, 2912129961U, 2664980877U, 1357572033U, 2600196436U, + 2486728200U, 2372668724U, 1567316966U, 2374111491U, 1839843570U, + 20815612U, 3727008608U, 3871996229U, 824061249U, 1932503978U, + 3404541726U, 758428924U, 2609331364U, 1223966026U, 1299179808U, + 648499352U, 2180134401U, 880821170U, 3781130950U, 113491270U, + 1032413764U, 4185884695U, 2490396037U, 1201932817U, 4060951446U, + 4165586898U, 1629813212U, 2887821158U, 415045333U, 628926856U, + 2193466079U, 3391843445U, 2227540681U, 1907099846U, 2848448395U, + 1717828221U, 1372704537U, 1707549841U, 2294058813U, 2101214437U, + 2052479531U, 1695809164U, 3176587306U, 2632770465U, 81634404U, + 1603220563U, 644238487U, 302857763U, 897352968U, 2613146653U, + 1391730149U, 4245717312U, 4191828749U, 1948492526U, 2618174230U, + 3992984522U, 2178852787U, 3596044509U, 3445573503U, 2026614616U, + 915763564U, 3415689334U, 2532153403U, 3879661562U, 2215027417U, + 3111154986U, 2929478371U, 668346391U, 1152241381U, 2632029711U, + 3004150659U, 2135025926U, 948690501U, 2799119116U, 4228829406U, + 1981197489U, 4209064138U, 684318751U, 3459397845U, 201790843U, + 4022541136U, 3043635877U, 492509624U, 3263466772U, 1509148086U, + 921459029U, 3198857146U, 705479721U, 3835966910U, 3603356465U, + 576159741U, 1742849431U, 594214882U, 2055294343U, 3634861861U, + 449571793U, 3246390646U, 3868232151U, 1479156585U, 2900125656U, + 2464815318U, 3960178104U, 1784261920U, 18311476U, 3627135050U, + 644609697U, 424968996U, 919890700U, 2986824110U, 816423214U, + 4003562844U, 1392714305U, 1757384428U, 2569030598U, 995949559U, + 3875659880U, 2933807823U, 2752536860U, 2993858466U, 4030558899U, + 2770783427U, 2775406005U, 2777781742U, 1931292655U, 472147933U, + 3865853827U, 2726470545U, 2668412860U, 2887008249U, 408979190U, + 3578063323U, 3242082049U, 1778193530U, 27981909U, 2362826515U, + 389875677U, 1043878156U, 581653903U, 3830568952U, 389535942U, + 3713523185U, 2768373359U, 2526101582U, 1998618197U, 1160859704U, + 3951172488U, 1098005003U, 906275699U, 3446228002U, 2220677963U, + 2059306445U, 132199571U, 476838790U, 1868039399U, 3097344807U, + 857300945U, 396345050U, 2835919916U, 1782168828U, 1419519470U, + 4288137521U, 819087232U, 596301494U, 872823172U, 1526888217U, + 805161465U, 1116186205U, 2829002754U, 2352620120U, 620121516U, + 354159268U, 3601949785U, 209568138U, 1352371732U, 2145977349U, + 4236871834U, 1539414078U, 3558126206U, 3224857093U, 4164166682U, + 3817553440U, 3301780278U, 2682696837U, 3734994768U, 1370950260U, + 1477421202U, 2521315749U, 1330148125U, 1261554731U, 2769143688U, + 3554756293U, 4235882678U, 3254686059U, 3530579953U, 1215452615U, + 3574970923U, 4057131421U, 589224178U, 1000098193U, 171190718U, + 2521852045U, 2351447494U, 2284441580U, 2646685513U, 3486933563U, + 3789864960U, 1190528160U, 1702536782U, 1534105589U, 4262946827U, + 2726686826U, 3584544841U, 2348270128U, 2145092281U, 2502718509U, + 1027832411U, 3571171153U, 1287361161U, 4011474411U, 3241215351U, + 2419700818U, 971242709U, 1361975763U, 1096842482U, 3271045537U, + 81165449U, 612438025U, 3912966678U, 1356929810U, 733545735U, + 537003843U, 1282953084U, 884458241U, 588930090U, 3930269801U, + 2961472450U, 1219535534U, 3632251943U, 268183903U, 1441240533U, + 3653903360U, 3854473319U, 2259087390U, 2548293048U, 2022641195U, + 2105543911U, 1764085217U, 3246183186U, 482438805U, 888317895U, + 2628314765U, 2466219854U, 717546004U, 2322237039U, 416725234U, + 1544049923U, 1797944973U, 3398652364U, 3111909456U, 485742908U, + 2277491072U, 1056355088U, 3181001278U, 129695079U, 2693624550U, + 1764438564U, 3797785470U, 195503713U, 3266519725U, 2053389444U, + 1961527818U, 3400226523U, 3777903038U, 2597274307U, 4235851091U, + 4094406648U, 2171410785U, 1781151386U, 1378577117U, 654643266U, + 3424024173U, 3385813322U, 679385799U, 479380913U, 681715441U, + 3096225905U, 276813409U, 3854398070U, 2721105350U, 831263315U, + 3276280337U, 2628301522U, 3984868494U, 1466099834U, 2104922114U, + 1412672743U, 820330404U, 3491501010U, 942735832U, 710652807U, + 3972652090U, 679881088U, 40577009U, 3705286397U, 2815423480U, + 3566262429U, 663396513U, 3777887429U, 4016670678U, 404539370U, + 1142712925U, 1140173408U, 2913248352U, 2872321286U, 263751841U, + 3175196073U, 3162557581U, 2878996619U, 75498548U, 3836833140U, + 3284664959U, 1157523805U, 112847376U, 207855609U, 1337979698U, + 1222578451U, 157107174U, 901174378U, 3883717063U, 1618632639U, + 1767889440U, 4264698824U, 1582999313U, 884471997U, 2508825098U, + 3756370771U, 2457213553U, 3565776881U, 3709583214U, 915609601U, + 460833524U, 1091049576U, 85522880U, 2553251U, 132102809U, + 2429882442U, 2562084610U, 1386507633U, 4112471229U, 21965213U, + 1981516006U, 2418435617U, 3054872091U, 4251511224U, 2025783543U, + 1916911512U, 2454491136U, 3938440891U, 3825869115U, 1121698605U, + 3463052265U, 802340101U, 1912886800U, 4031997367U, 3550640406U, + 1596096923U, 610150600U, 431464457U, 2541325046U, 486478003U, + 739704936U, 2862696430U, 3037903166U, 1129749694U, 2611481261U, + 1228993498U, 510075548U, 3424962587U, 2458689681U, 818934833U, + 4233309125U, 1608196251U, 3419476016U, 1858543939U, 2682166524U, + 3317854285U, 631986188U, 3008214764U, 613826412U, 3567358221U, + 3512343882U, 1552467474U, 3316162670U, 1275841024U, 4142173454U, + 565267881U, 768644821U, 198310105U, 2396688616U, 1837659011U, + 203429334U, 854539004U, 4235811518U, 3338304926U, 3730418692U, + 3852254981U, 3032046452U, 2329811860U, 2303590566U, 2696092212U, + 3894665932U, 145835667U, 249563655U, 1932210840U, 2431696407U, + 3312636759U, 214962629U, 2092026914U, 3020145527U, 4073039873U, + 2739105705U, 1308336752U, 855104522U, 2391715321U, 67448785U, + 547989482U, 854411802U, 3608633740U, 431731530U, 537375589U, + 3888005760U, 696099141U, 397343236U, 1864511780U, 44029739U, + 1729526891U, 1993398655U, 2010173426U, 2591546756U, 275223291U, + 1503900299U, 4217765081U, 2185635252U, 1122436015U, 3550155364U, + 681707194U, 3260479338U, 933579397U, 2983029282U, 2505504587U, + 2667410393U, 2962684490U, 4139721708U, 2658172284U, 2452602383U, + 2607631612U, 1344296217U, 3075398709U, 2949785295U, 1049956168U, + 3917185129U, 2155660174U, 3280524475U, 1503827867U, 674380765U, + 1918468193U, 3843983676U, 634358221U, 2538335643U, 1873351298U, + 3368723763U, 2129144130U, 3203528633U, 3087174986U, 2691698871U, + 2516284287U, 24437745U, 1118381474U, 2816314867U, 2448576035U, + 4281989654U, 217287825U, 165872888U, 2628995722U, 3533525116U, + 2721669106U, 872340568U, 3429930655U, 3309047304U, 3916704967U, + 3270160355U, 1348884255U, 1634797670U, 881214967U, 4259633554U, + 174613027U, 1103974314U, 1625224232U, 2678368291U, 1133866707U, + 3853082619U, 4073196549U, 1189620777U, 637238656U, 930241537U, + 4042750792U, 3842136042U, 2417007212U, 2524907510U, 1243036827U, + 1282059441U, 3764588774U, 1394459615U, 2323620015U, 1166152231U, + 3307479609U, 3849322257U, 3507445699U, 4247696636U, 758393720U, + 967665141U, 1095244571U, 1319812152U, 407678762U, 2640605208U, + 2170766134U, 3663594275U, 4039329364U, 2512175520U, 725523154U, + 2249807004U, 3312617979U, 2414634172U, 1278482215U, 349206484U, + 1573063308U, 1196429124U, 3873264116U, 2400067801U, 268795167U, + 226175489U, 2961367263U, 1968719665U, 42656370U, 1010790699U, + 561600615U, 2422453992U, 3082197735U, 1636700484U, 3977715296U, + 3125350482U, 3478021514U, 2227819446U, 1540868045U, 3061908980U, + 1087362407U, 3625200291U, 361937537U, 580441897U, 1520043666U, + 2270875402U, 1009161260U, 2502355842U, 4278769785U, 473902412U, + 1057239083U, 1905829039U, 1483781177U, 2080011417U, 1207494246U, + 1806991954U, 2194674403U, 3455972205U, 807207678U, 3655655687U, + 674112918U, 195425752U, 3917890095U, 1874364234U, 1837892715U, + 3663478166U, 1548892014U, 2570748714U, 2049929836U, 2167029704U, + 697543767U, 3499545023U, 3342496315U, 1725251190U, 3561387469U, + 2905606616U, 1580182447U, 3934525927U, 4103172792U, 1365672522U, + 1534795737U, 3308667416U, 2841911405U, 3943182730U, 4072020313U, + 3494770452U, 3332626671U, 55327267U, 478030603U, 411080625U, + 3419529010U, 1604767823U, 3513468014U, 570668510U, 913790824U, + 2283967995U, 695159462U, 3825542932U, 4150698144U, 1829758699U, + 202895590U, 1609122645U, 1267651008U, 2910315509U, 2511475445U, + 2477423819U, 3932081579U, 900879979U, 2145588390U, 2670007504U, + 580819444U, 1864996828U, 2526325979U, 1019124258U, 815508628U, + 2765933989U, 1277301341U, 3006021786U, 855540956U, 288025710U, + 1919594237U, 2331223864U, 177452412U, 2475870369U, 2689291749U, + 865194284U, 253432152U, 2628531804U, 2861208555U, 2361597573U, + 1653952120U, 1039661024U, 2159959078U, 3709040440U, 3564718533U, + 2596878672U, 2041442161U, 31164696U, 2662962485U, 3665637339U, + 1678115244U, 2699839832U, 3651968520U, 3521595541U, 458433303U, + 2423096824U, 21831741U, 380011703U, 2498168716U, 861806087U, + 1673574843U, 4188794405U, 2520563651U, 2632279153U, 2170465525U, + 4171949898U, 3886039621U, 1661344005U, 3424285243U, 992588372U, + 2500984144U, 2993248497U, 3590193895U, 1535327365U, 515645636U, + 131633450U, 3729760261U, 1613045101U, 3254194278U, 15889678U, + 1493590689U, 244148718U, 2991472662U, 1401629333U, 777349878U, + 2501401703U, 4285518317U, 3794656178U, 955526526U, 3442142820U, + 3970298374U, 736025417U, 2737370764U, 1271509744U, 440570731U, + 136141826U, 1596189518U, 923399175U, 257541519U, 3505774281U, + 2194358432U, 2518162991U, 1379893637U, 2667767062U, 3748146247U, + 1821712620U, 3923161384U, 1947811444U, 2392527197U, 4127419685U, + 1423694998U, 4156576871U, 1382885582U, 3420127279U, 3617499534U, + 2994377493U, 4038063986U, 1918458672U, 2983166794U, 4200449033U, + 353294540U, 1609232588U, 243926648U, 2332803291U, 507996832U, + 2392838793U, 4075145196U, 2060984340U, 4287475136U, 88232602U, + 2491531140U, 4159725633U, 2272075455U, 759298618U, 201384554U, + 838356250U, 1416268324U, 674476934U, 90795364U, 141672229U, + 3660399588U, 4196417251U, 3249270244U, 3774530247U, 59587265U, + 3683164208U, 19392575U, 1463123697U, 1882205379U, 293780489U, + 2553160622U, 2933904694U, 675638239U, 2851336944U, 1435238743U, + 2448730183U, 804436302U, 2119845972U, 322560608U, 4097732704U, + 2987802540U, 641492617U, 2575442710U, 4217822703U, 3271835300U, + 2836418300U, 3739921620U, 2138378768U, 2879771855U, 4294903423U, + 3121097946U, 2603440486U, 2560820391U, 1012930944U, 2313499967U, + 584489368U, 3431165766U, 897384869U, 2062537737U, 2847889234U, + 3742362450U, 2951174585U, 4204621084U, 1109373893U, 3668075775U, + 2750138839U, 3518055702U, 733072558U, 4169325400U, 788493625U +}; +static const uint64_t init_gen_rand_64_expected[] = { + QU(16924766246869039260LLU), QU( 8201438687333352714LLU), + QU( 2265290287015001750LLU), QU(18397264611805473832LLU), + QU( 3375255223302384358LLU), QU( 6345559975416828796LLU), + QU(18229739242790328073LLU), QU( 7596792742098800905LLU), + QU( 255338647169685981LLU), QU( 2052747240048610300LLU), + QU(18328151576097299343LLU), QU(12472905421133796567LLU), + QU(11315245349717600863LLU), QU(16594110197775871209LLU), + QU(15708751964632456450LLU), QU(10452031272054632535LLU), + QU(11097646720811454386LLU), QU( 4556090668445745441LLU), + QU(17116187693090663106LLU), QU(14931526836144510645LLU), + QU( 9190752218020552591LLU), QU( 9625800285771901401LLU), + QU(13995141077659972832LLU), QU( 5194209094927829625LLU), + QU( 4156788379151063303LLU), QU( 8523452593770139494LLU), + QU(14082382103049296727LLU), QU( 2462601863986088483LLU), + QU( 3030583461592840678LLU), QU( 5221622077872827681LLU), + QU( 3084210671228981236LLU), QU(13956758381389953823LLU), + QU(13503889856213423831LLU), QU(15696904024189836170LLU), + QU( 4612584152877036206LLU), QU( 6231135538447867881LLU), + QU(10172457294158869468LLU), QU( 6452258628466708150LLU), + QU(14044432824917330221LLU), QU( 370168364480044279LLU), + QU(10102144686427193359LLU), QU( 667870489994776076LLU), + QU( 2732271956925885858LLU), QU(18027788905977284151LLU), + QU(15009842788582923859LLU), QU( 7136357960180199542LLU), + QU(15901736243475578127LLU), QU(16951293785352615701LLU), + QU(10551492125243691632LLU), QU(17668869969146434804LLU), + QU(13646002971174390445LLU), QU( 9804471050759613248LLU), + QU( 5511670439655935493LLU), QU(18103342091070400926LLU), + QU(17224512747665137533LLU), QU(15534627482992618168LLU), + QU( 1423813266186582647LLU), QU(15821176807932930024LLU), + QU( 30323369733607156LLU), QU(11599382494723479403LLU), + QU( 653856076586810062LLU), QU( 3176437395144899659LLU), + QU(14028076268147963917LLU), QU(16156398271809666195LLU), + QU( 3166955484848201676LLU), QU( 5746805620136919390LLU), + QU(17297845208891256593LLU), QU(11691653183226428483LLU), + QU(17900026146506981577LLU), QU(15387382115755971042LLU), + QU(16923567681040845943LLU), QU( 8039057517199388606LLU), + QU(11748409241468629263LLU), QU( 794358245539076095LLU), + QU(13438501964693401242LLU), QU(14036803236515618962LLU), + QU( 5252311215205424721LLU), QU(17806589612915509081LLU), + QU( 6802767092397596006LLU), QU(14212120431184557140LLU), + QU( 1072951366761385712LLU), QU(13098491780722836296LLU), + QU( 9466676828710797353LLU), QU(12673056849042830081LLU), + QU(12763726623645357580LLU), QU(16468961652999309493LLU), + QU(15305979875636438926LLU), QU(17444713151223449734LLU), + QU( 5692214267627883674LLU), QU(13049589139196151505LLU), + QU( 880115207831670745LLU), QU( 1776529075789695498LLU), + QU(16695225897801466485LLU), QU(10666901778795346845LLU), + QU( 6164389346722833869LLU), QU( 2863817793264300475LLU), + QU( 9464049921886304754LLU), QU( 3993566636740015468LLU), + QU( 9983749692528514136LLU), QU(16375286075057755211LLU), + QU(16042643417005440820LLU), QU(11445419662923489877LLU), + QU( 7999038846885158836LLU), QU( 6721913661721511535LLU), + QU( 5363052654139357320LLU), QU( 1817788761173584205LLU), + QU(13290974386445856444LLU), QU( 4650350818937984680LLU), + QU( 8219183528102484836LLU), QU( 1569862923500819899LLU), + QU( 4189359732136641860LLU), QU(14202822961683148583LLU), + QU( 4457498315309429058LLU), QU(13089067387019074834LLU), + QU(11075517153328927293LLU), QU(10277016248336668389LLU), + QU( 7070509725324401122LLU), QU(17808892017780289380LLU), + QU(13143367339909287349LLU), QU( 1377743745360085151LLU), + QU( 5749341807421286485LLU), QU(14832814616770931325LLU), + QU( 7688820635324359492LLU), QU(10960474011539770045LLU), + QU( 81970066653179790LLU), QU(12619476072607878022LLU), + QU( 4419566616271201744LLU), QU(15147917311750568503LLU), + QU( 5549739182852706345LLU), QU( 7308198397975204770LLU), + QU(13580425496671289278LLU), QU(17070764785210130301LLU), + QU( 8202832846285604405LLU), QU( 6873046287640887249LLU), + QU( 6927424434308206114LLU), QU( 6139014645937224874LLU), + QU(10290373645978487639LLU), QU(15904261291701523804LLU), + QU( 9628743442057826883LLU), QU(18383429096255546714LLU), + QU( 4977413265753686967LLU), QU( 7714317492425012869LLU), + QU( 9025232586309926193LLU), QU(14627338359776709107LLU), + QU(14759849896467790763LLU), QU(10931129435864423252LLU), + QU( 4588456988775014359LLU), QU(10699388531797056724LLU), + QU( 468652268869238792LLU), QU( 5755943035328078086LLU), + QU( 2102437379988580216LLU), QU( 9986312786506674028LLU), + QU( 2654207180040945604LLU), QU( 8726634790559960062LLU), + QU( 100497234871808137LLU), QU( 2800137176951425819LLU), + QU( 6076627612918553487LLU), QU( 5780186919186152796LLU), + QU( 8179183595769929098LLU), QU( 6009426283716221169LLU), + QU( 2796662551397449358LLU), QU( 1756961367041986764LLU), + QU( 6972897917355606205LLU), QU(14524774345368968243LLU), + QU( 2773529684745706940LLU), QU( 4853632376213075959LLU), + QU( 4198177923731358102LLU), QU( 8271224913084139776LLU), + QU( 2741753121611092226LLU), QU(16782366145996731181LLU), + QU(15426125238972640790LLU), QU(13595497100671260342LLU), + QU( 3173531022836259898LLU), QU( 6573264560319511662LLU), + QU(18041111951511157441LLU), QU( 2351433581833135952LLU), + QU( 3113255578908173487LLU), QU( 1739371330877858784LLU), + QU(16046126562789165480LLU), QU( 8072101652214192925LLU), + QU(15267091584090664910LLU), QU( 9309579200403648940LLU), + QU( 5218892439752408722LLU), QU(14492477246004337115LLU), + QU(17431037586679770619LLU), QU( 7385248135963250480LLU), + QU( 9580144956565560660LLU), QU( 4919546228040008720LLU), + QU(15261542469145035584LLU), QU(18233297270822253102LLU), + QU( 5453248417992302857LLU), QU( 9309519155931460285LLU), + QU(10342813012345291756LLU), QU(15676085186784762381LLU), + QU(15912092950691300645LLU), QU( 9371053121499003195LLU), + QU( 9897186478226866746LLU), QU(14061858287188196327LLU), + QU( 122575971620788119LLU), QU(12146750969116317754LLU), + QU( 4438317272813245201LLU), QU( 8332576791009527119LLU), + QU(13907785691786542057LLU), QU(10374194887283287467LLU), + QU( 2098798755649059566LLU), QU( 3416235197748288894LLU), + QU( 8688269957320773484LLU), QU( 7503964602397371571LLU), + QU(16724977015147478236LLU), QU( 9461512855439858184LLU), + QU(13259049744534534727LLU), QU( 3583094952542899294LLU), + QU( 8764245731305528292LLU), QU(13240823595462088985LLU), + QU(13716141617617910448LLU), QU(18114969519935960955LLU), + QU( 2297553615798302206LLU), QU( 4585521442944663362LLU), + QU(17776858680630198686LLU), QU( 4685873229192163363LLU), + QU( 152558080671135627LLU), QU(15424900540842670088LLU), + QU(13229630297130024108LLU), QU(17530268788245718717LLU), + QU(16675633913065714144LLU), QU( 3158912717897568068LLU), + QU(15399132185380087288LLU), QU( 7401418744515677872LLU), + QU(13135412922344398535LLU), QU( 6385314346100509511LLU), + QU(13962867001134161139LLU), QU(10272780155442671999LLU), + QU(12894856086597769142LLU), QU(13340877795287554994LLU), + QU(12913630602094607396LLU), QU(12543167911119793857LLU), + QU(17343570372251873096LLU), QU(10959487764494150545LLU), + QU( 6966737953093821128LLU), QU(13780699135496988601LLU), + QU( 4405070719380142046LLU), QU(14923788365607284982LLU), + QU( 2869487678905148380LLU), QU( 6416272754197188403LLU), + QU(15017380475943612591LLU), QU( 1995636220918429487LLU), + QU( 3402016804620122716LLU), QU(15800188663407057080LLU), + QU(11362369990390932882LLU), QU(15262183501637986147LLU), + QU(10239175385387371494LLU), QU( 9352042420365748334LLU), + QU( 1682457034285119875LLU), QU( 1724710651376289644LLU), + QU( 2038157098893817966LLU), QU( 9897825558324608773LLU), + QU( 1477666236519164736LLU), QU(16835397314511233640LLU), + QU(10370866327005346508LLU), QU(10157504370660621982LLU), + QU(12113904045335882069LLU), QU(13326444439742783008LLU), + QU(11302769043000765804LLU), QU(13594979923955228484LLU), + QU(11779351762613475968LLU), QU( 3786101619539298383LLU), + QU( 8021122969180846063LLU), QU(15745904401162500495LLU), + QU(10762168465993897267LLU), QU(13552058957896319026LLU), + QU(11200228655252462013LLU), QU( 5035370357337441226LLU), + QU( 7593918984545500013LLU), QU( 5418554918361528700LLU), + QU( 4858270799405446371LLU), QU( 9974659566876282544LLU), + QU(18227595922273957859LLU), QU( 2772778443635656220LLU), + QU(14285143053182085385LLU), QU( 9939700992429600469LLU), + QU(12756185904545598068LLU), QU( 2020783375367345262LLU), + QU( 57026775058331227LLU), QU( 950827867930065454LLU), + QU( 6602279670145371217LLU), QU( 2291171535443566929LLU), + QU( 5832380724425010313LLU), QU( 1220343904715982285LLU), + QU(17045542598598037633LLU), QU(15460481779702820971LLU), + QU(13948388779949365130LLU), QU(13975040175430829518LLU), + QU(17477538238425541763LLU), QU(11104663041851745725LLU), + QU(15860992957141157587LLU), QU(14529434633012950138LLU), + QU( 2504838019075394203LLU), QU( 7512113882611121886LLU), + QU( 4859973559980886617LLU), QU( 1258601555703250219LLU), + QU(15594548157514316394LLU), QU( 4516730171963773048LLU), + QU(11380103193905031983LLU), QU( 6809282239982353344LLU), + QU(18045256930420065002LLU), QU( 2453702683108791859LLU), + QU( 977214582986981460LLU), QU( 2006410402232713466LLU), + QU( 6192236267216378358LLU), QU( 3429468402195675253LLU), + QU(18146933153017348921LLU), QU(17369978576367231139LLU), + QU( 1246940717230386603LLU), QU(11335758870083327110LLU), + QU(14166488801730353682LLU), QU( 9008573127269635732LLU), + QU(10776025389820643815LLU), QU(15087605441903942962LLU), + QU( 1359542462712147922LLU), QU(13898874411226454206LLU), + QU(17911176066536804411LLU), QU( 9435590428600085274LLU), + QU( 294488509967864007LLU), QU( 8890111397567922046LLU), + QU( 7987823476034328778LLU), QU(13263827582440967651LLU), + QU( 7503774813106751573LLU), QU(14974747296185646837LLU), + QU( 8504765037032103375LLU), QU(17340303357444536213LLU), + QU( 7704610912964485743LLU), QU( 8107533670327205061LLU), + QU( 9062969835083315985LLU), QU(16968963142126734184LLU), + QU(12958041214190810180LLU), QU( 2720170147759570200LLU), + QU( 2986358963942189566LLU), QU(14884226322219356580LLU), + QU( 286224325144368520LLU), QU(11313800433154279797LLU), + QU(18366849528439673248LLU), QU(17899725929482368789LLU), + QU( 3730004284609106799LLU), QU( 1654474302052767205LLU), + QU( 5006698007047077032LLU), QU( 8196893913601182838LLU), + QU(15214541774425211640LLU), QU(17391346045606626073LLU), + QU( 8369003584076969089LLU), QU( 3939046733368550293LLU), + QU(10178639720308707785LLU), QU( 2180248669304388697LLU), + QU( 62894391300126322LLU), QU( 9205708961736223191LLU), + QU( 6837431058165360438LLU), QU( 3150743890848308214LLU), + QU(17849330658111464583LLU), QU(12214815643135450865LLU), + QU(13410713840519603402LLU), QU( 3200778126692046802LLU), + QU(13354780043041779313LLU), QU( 800850022756886036LLU), + QU(15660052933953067433LLU), QU( 6572823544154375676LLU), + QU(11030281857015819266LLU), QU(12682241941471433835LLU), + QU(11654136407300274693LLU), QU( 4517795492388641109LLU), + QU( 9757017371504524244LLU), QU(17833043400781889277LLU), + QU(12685085201747792227LLU), QU(10408057728835019573LLU), + QU( 98370418513455221LLU), QU( 6732663555696848598LLU), + QU(13248530959948529780LLU), QU( 3530441401230622826LLU), + QU(18188251992895660615LLU), QU( 1847918354186383756LLU), + QU( 1127392190402660921LLU), QU(11293734643143819463LLU), + QU( 3015506344578682982LLU), QU(13852645444071153329LLU), + QU( 2121359659091349142LLU), QU( 1294604376116677694LLU), + QU( 5616576231286352318LLU), QU( 7112502442954235625LLU), + QU(11676228199551561689LLU), QU(12925182803007305359LLU), + QU( 7852375518160493082LLU), QU( 1136513130539296154LLU), + QU( 5636923900916593195LLU), QU( 3221077517612607747LLU), + QU(17784790465798152513LLU), QU( 3554210049056995938LLU), + QU(17476839685878225874LLU), QU( 3206836372585575732LLU), + QU( 2765333945644823430LLU), QU(10080070903718799528LLU), + QU( 5412370818878286353LLU), QU( 9689685887726257728LLU), + QU( 8236117509123533998LLU), QU( 1951139137165040214LLU), + QU( 4492205209227980349LLU), QU(16541291230861602967LLU), + QU( 1424371548301437940LLU), QU( 9117562079669206794LLU), + QU(14374681563251691625LLU), QU(13873164030199921303LLU), + QU( 6680317946770936731LLU), QU(15586334026918276214LLU), + QU(10896213950976109802LLU), QU( 9506261949596413689LLU), + QU( 9903949574308040616LLU), QU( 6038397344557204470LLU), + QU( 174601465422373648LLU), QU(15946141191338238030LLU), + QU(17142225620992044937LLU), QU( 7552030283784477064LLU), + QU( 2947372384532947997LLU), QU( 510797021688197711LLU), + QU( 4962499439249363461LLU), QU( 23770320158385357LLU), + QU( 959774499105138124LLU), QU( 1468396011518788276LLU), + QU( 2015698006852312308LLU), QU( 4149400718489980136LLU), + QU( 5992916099522371188LLU), QU(10819182935265531076LLU), + QU(16189787999192351131LLU), QU( 342833961790261950LLU), + QU(12470830319550495336LLU), QU(18128495041912812501LLU), + QU( 1193600899723524337LLU), QU( 9056793666590079770LLU), + QU( 2154021227041669041LLU), QU( 4963570213951235735LLU), + QU( 4865075960209211409LLU), QU( 2097724599039942963LLU), + QU( 2024080278583179845LLU), QU(11527054549196576736LLU), + QU(10650256084182390252LLU), QU( 4808408648695766755LLU), + QU( 1642839215013788844LLU), QU(10607187948250398390LLU), + QU( 7076868166085913508LLU), QU( 730522571106887032LLU), + QU(12500579240208524895LLU), QU( 4484390097311355324LLU), + QU(15145801330700623870LLU), QU( 8055827661392944028LLU), + QU( 5865092976832712268LLU), QU(15159212508053625143LLU), + QU( 3560964582876483341LLU), QU( 4070052741344438280LLU), + QU( 6032585709886855634LLU), QU(15643262320904604873LLU), + QU( 2565119772293371111LLU), QU( 318314293065348260LLU), + QU(15047458749141511872LLU), QU( 7772788389811528730LLU), + QU( 7081187494343801976LLU), QU( 6465136009467253947LLU), + QU(10425940692543362069LLU), QU( 554608190318339115LLU), + QU(14796699860302125214LLU), QU( 1638153134431111443LLU), + QU(10336967447052276248LLU), QU( 8412308070396592958LLU), + QU( 4004557277152051226LLU), QU( 8143598997278774834LLU), + QU(16413323996508783221LLU), QU(13139418758033994949LLU), + QU( 9772709138335006667LLU), QU( 2818167159287157659LLU), + QU(17091740573832523669LLU), QU(14629199013130751608LLU), + QU(18268322711500338185LLU), QU( 8290963415675493063LLU), + QU( 8830864907452542588LLU), QU( 1614839084637494849LLU), + QU(14855358500870422231LLU), QU( 3472996748392519937LLU), + QU(15317151166268877716LLU), QU( 5825895018698400362LLU), + QU(16730208429367544129LLU), QU(10481156578141202800LLU), + QU( 4746166512382823750LLU), QU(12720876014472464998LLU), + QU( 8825177124486735972LLU), QU(13733447296837467838LLU), + QU( 6412293741681359625LLU), QU( 8313213138756135033LLU), + QU(11421481194803712517LLU), QU( 7997007691544174032LLU), + QU( 6812963847917605930LLU), QU( 9683091901227558641LLU), + QU(14703594165860324713LLU), QU( 1775476144519618309LLU), + QU( 2724283288516469519LLU), QU( 717642555185856868LLU), + QU( 8736402192215092346LLU), QU(11878800336431381021LLU), + QU( 4348816066017061293LLU), QU( 6115112756583631307LLU), + QU( 9176597239667142976LLU), QU(12615622714894259204LLU), + QU(10283406711301385987LLU), QU( 5111762509485379420LLU), + QU( 3118290051198688449LLU), QU( 7345123071632232145LLU), + QU( 9176423451688682359LLU), QU( 4843865456157868971LLU), + QU(12008036363752566088LLU), QU(12058837181919397720LLU), + QU( 2145073958457347366LLU), QU( 1526504881672818067LLU), + QU( 3488830105567134848LLU), QU(13208362960674805143LLU), + QU( 4077549672899572192LLU), QU( 7770995684693818365LLU), + QU( 1398532341546313593LLU), QU(12711859908703927840LLU), + QU( 1417561172594446813LLU), QU(17045191024194170604LLU), + QU( 4101933177604931713LLU), QU(14708428834203480320LLU), + QU(17447509264469407724LLU), QU(14314821973983434255LLU), + QU(17990472271061617265LLU), QU( 5087756685841673942LLU), + QU(12797820586893859939LLU), QU( 1778128952671092879LLU), + QU( 3535918530508665898LLU), QU( 9035729701042481301LLU), + QU(14808661568277079962LLU), QU(14587345077537747914LLU), + QU(11920080002323122708LLU), QU( 6426515805197278753LLU), + QU( 3295612216725984831LLU), QU(11040722532100876120LLU), + QU(12305952936387598754LLU), QU(16097391899742004253LLU), + QU( 4908537335606182208LLU), QU(12446674552196795504LLU), + QU(16010497855816895177LLU), QU( 9194378874788615551LLU), + QU( 3382957529567613384LLU), QU( 5154647600754974077LLU), + QU( 9801822865328396141LLU), QU( 9023662173919288143LLU), + QU(17623115353825147868LLU), QU( 8238115767443015816LLU), + QU(15811444159859002560LLU), QU( 9085612528904059661LLU), + QU( 6888601089398614254LLU), QU( 258252992894160189LLU), + QU( 6704363880792428622LLU), QU( 6114966032147235763LLU), + QU(11075393882690261875LLU), QU( 8797664238933620407LLU), + QU( 5901892006476726920LLU), QU( 5309780159285518958LLU), + QU(14940808387240817367LLU), QU(14642032021449656698LLU), + QU( 9808256672068504139LLU), QU( 3670135111380607658LLU), + QU(11211211097845960152LLU), QU( 1474304506716695808LLU), + QU(15843166204506876239LLU), QU( 7661051252471780561LLU), + QU(10170905502249418476LLU), QU( 7801416045582028589LLU), + QU( 2763981484737053050LLU), QU( 9491377905499253054LLU), + QU(16201395896336915095LLU), QU( 9256513756442782198LLU), + QU( 5411283157972456034LLU), QU( 5059433122288321676LLU), + QU( 4327408006721123357LLU), QU( 9278544078834433377LLU), + QU( 7601527110882281612LLU), QU(11848295896975505251LLU), + QU(12096998801094735560LLU), QU(14773480339823506413LLU), + QU(15586227433895802149LLU), QU(12786541257830242872LLU), + QU( 6904692985140503067LLU), QU( 5309011515263103959LLU), + QU(12105257191179371066LLU), QU(14654380212442225037LLU), + QU( 2556774974190695009LLU), QU( 4461297399927600261LLU), + QU(14888225660915118646LLU), QU(14915459341148291824LLU), + QU( 2738802166252327631LLU), QU( 6047155789239131512LLU), + QU(12920545353217010338LLU), QU(10697617257007840205LLU), + QU( 2751585253158203504LLU), QU(13252729159780047496LLU), + QU(14700326134672815469LLU), QU(14082527904374600529LLU), + QU(16852962273496542070LLU), QU(17446675504235853907LLU), + QU(15019600398527572311LLU), QU(12312781346344081551LLU), + QU(14524667935039810450LLU), QU( 5634005663377195738LLU), + QU(11375574739525000569LLU), QU( 2423665396433260040LLU), + QU( 5222836914796015410LLU), QU( 4397666386492647387LLU), + QU( 4619294441691707638LLU), QU( 665088602354770716LLU), + QU(13246495665281593610LLU), QU( 6564144270549729409LLU), + QU(10223216188145661688LLU), QU( 3961556907299230585LLU), + QU(11543262515492439914LLU), QU(16118031437285993790LLU), + QU( 7143417964520166465LLU), QU(13295053515909486772LLU), + QU( 40434666004899675LLU), QU(17127804194038347164LLU), + QU( 8599165966560586269LLU), QU( 8214016749011284903LLU), + QU(13725130352140465239LLU), QU( 5467254474431726291LLU), + QU( 7748584297438219877LLU), QU(16933551114829772472LLU), + QU( 2169618439506799400LLU), QU( 2169787627665113463LLU), + QU(17314493571267943764LLU), QU(18053575102911354912LLU), + QU(11928303275378476973LLU), QU(11593850925061715550LLU), + QU(17782269923473589362LLU), QU( 3280235307704747039LLU), + QU( 6145343578598685149LLU), QU(17080117031114086090LLU), + QU(18066839902983594755LLU), QU( 6517508430331020706LLU), + QU( 8092908893950411541LLU), QU(12558378233386153732LLU), + QU( 4476532167973132976LLU), QU(16081642430367025016LLU), + QU( 4233154094369139361LLU), QU( 8693630486693161027LLU), + QU(11244959343027742285LLU), QU(12273503967768513508LLU), + QU(14108978636385284876LLU), QU( 7242414665378826984LLU), + QU( 6561316938846562432LLU), QU( 8601038474994665795LLU), + QU(17532942353612365904LLU), QU(17940076637020912186LLU), + QU( 7340260368823171304LLU), QU( 7061807613916067905LLU), + QU(10561734935039519326LLU), QU(17990796503724650862LLU), + QU( 6208732943911827159LLU), QU( 359077562804090617LLU), + QU(14177751537784403113LLU), QU(10659599444915362902LLU), + QU(15081727220615085833LLU), QU(13417573895659757486LLU), + QU(15513842342017811524LLU), QU(11814141516204288231LLU), + QU( 1827312513875101814LLU), QU( 2804611699894603103LLU), + QU(17116500469975602763LLU), QU(12270191815211952087LLU), + QU(12256358467786024988LLU), QU(18435021722453971267LLU), + QU( 671330264390865618LLU), QU( 476504300460286050LLU), + QU(16465470901027093441LLU), QU( 4047724406247136402LLU), + QU( 1322305451411883346LLU), QU( 1388308688834322280LLU), + QU( 7303989085269758176LLU), QU( 9323792664765233642LLU), + QU( 4542762575316368936LLU), QU(17342696132794337618LLU), + QU( 4588025054768498379LLU), QU(13415475057390330804LLU), + QU(17880279491733405570LLU), QU(10610553400618620353LLU), + QU( 3180842072658960139LLU), QU(13002966655454270120LLU), + QU( 1665301181064982826LLU), QU( 7083673946791258979LLU), + QU( 190522247122496820LLU), QU(17388280237250677740LLU), + QU( 8430770379923642945LLU), QU(12987180971921668584LLU), + QU( 2311086108365390642LLU), QU( 2870984383579822345LLU), + QU(14014682609164653318LLU), QU(14467187293062251484LLU), + QU( 192186361147413298LLU), QU(15171951713531796524LLU), + QU( 9900305495015948728LLU), QU(17958004775615466344LLU), + QU(14346380954498606514LLU), QU(18040047357617407096LLU), + QU( 5035237584833424532LLU), QU(15089555460613972287LLU), + QU( 4131411873749729831LLU), QU( 1329013581168250330LLU), + QU(10095353333051193949LLU), QU(10749518561022462716LLU), + QU( 9050611429810755847LLU), QU(15022028840236655649LLU), + QU( 8775554279239748298LLU), QU(13105754025489230502LLU), + QU(15471300118574167585LLU), QU( 89864764002355628LLU), + QU( 8776416323420466637LLU), QU( 5280258630612040891LLU), + QU( 2719174488591862912LLU), QU( 7599309137399661994LLU), + QU(15012887256778039979LLU), QU(14062981725630928925LLU), + QU(12038536286991689603LLU), QU( 7089756544681775245LLU), + QU(10376661532744718039LLU), QU( 1265198725901533130LLU), + QU(13807996727081142408LLU), QU( 2935019626765036403LLU), + QU( 7651672460680700141LLU), QU( 3644093016200370795LLU), + QU( 2840982578090080674LLU), QU(17956262740157449201LLU), + QU(18267979450492880548LLU), QU(11799503659796848070LLU), + QU( 9942537025669672388LLU), QU(11886606816406990297LLU), + QU( 5488594946437447576LLU), QU( 7226714353282744302LLU), + QU( 3784851653123877043LLU), QU( 878018453244803041LLU), + QU(12110022586268616085LLU), QU( 734072179404675123LLU), + QU(11869573627998248542LLU), QU( 469150421297783998LLU), + QU( 260151124912803804LLU), QU(11639179410120968649LLU), + QU( 9318165193840846253LLU), QU(12795671722734758075LLU), + QU(15318410297267253933LLU), QU( 691524703570062620LLU), + QU( 5837129010576994601LLU), QU(15045963859726941052LLU), + QU( 5850056944932238169LLU), QU(12017434144750943807LLU), + QU( 7447139064928956574LLU), QU( 3101711812658245019LLU), + QU(16052940704474982954LLU), QU(18195745945986994042LLU), + QU( 8932252132785575659LLU), QU(13390817488106794834LLU), + QU(11582771836502517453LLU), QU( 4964411326683611686LLU), + QU( 2195093981702694011LLU), QU(14145229538389675669LLU), + QU(16459605532062271798LLU), QU( 866316924816482864LLU), + QU( 4593041209937286377LLU), QU( 8415491391910972138LLU), + QU( 4171236715600528969LLU), QU(16637569303336782889LLU), + QU( 2002011073439212680LLU), QU(17695124661097601411LLU), + QU( 4627687053598611702LLU), QU( 7895831936020190403LLU), + QU( 8455951300917267802LLU), QU( 2923861649108534854LLU), + QU( 8344557563927786255LLU), QU( 6408671940373352556LLU), + QU(12210227354536675772LLU), QU(14294804157294222295LLU), + QU(10103022425071085127LLU), QU(10092959489504123771LLU), + QU( 6554774405376736268LLU), QU(12629917718410641774LLU), + QU( 6260933257596067126LLU), QU( 2460827021439369673LLU), + QU( 2541962996717103668LLU), QU( 597377203127351475LLU), + QU( 5316984203117315309LLU), QU( 4811211393563241961LLU), + QU(13119698597255811641LLU), QU( 8048691512862388981LLU), + QU(10216818971194073842LLU), QU( 4612229970165291764LLU), + QU(10000980798419974770LLU), QU( 6877640812402540687LLU), + QU( 1488727563290436992LLU), QU( 2227774069895697318LLU), + QU(11237754507523316593LLU), QU(13478948605382290972LLU), + QU( 1963583846976858124LLU), QU( 5512309205269276457LLU), + QU( 3972770164717652347LLU), QU( 3841751276198975037LLU), + QU(10283343042181903117LLU), QU( 8564001259792872199LLU), + QU(16472187244722489221LLU), QU( 8953493499268945921LLU), + QU( 3518747340357279580LLU), QU( 4003157546223963073LLU), + QU( 3270305958289814590LLU), QU( 3966704458129482496LLU), + QU( 8122141865926661939LLU), QU(14627734748099506653LLU), + QU(13064426990862560568LLU), QU( 2414079187889870829LLU), + QU( 5378461209354225306LLU), QU(10841985740128255566LLU), + QU( 538582442885401738LLU), QU( 7535089183482905946LLU), + QU(16117559957598879095LLU), QU( 8477890721414539741LLU), + QU( 1459127491209533386LLU), QU(17035126360733620462LLU), + QU( 8517668552872379126LLU), QU(10292151468337355014LLU), + QU(17081267732745344157LLU), QU(13751455337946087178LLU), + QU(14026945459523832966LLU), QU( 6653278775061723516LLU), + QU(10619085543856390441LLU), QU( 2196343631481122885LLU), + QU(10045966074702826136LLU), QU(10082317330452718282LLU), + QU( 5920859259504831242LLU), QU( 9951879073426540617LLU), + QU( 7074696649151414158LLU), QU(15808193543879464318LLU), + QU( 7385247772746953374LLU), QU( 3192003544283864292LLU), + QU(18153684490917593847LLU), QU(12423498260668568905LLU), + QU(10957758099756378169LLU), QU(11488762179911016040LLU), + QU( 2099931186465333782LLU), QU(11180979581250294432LLU), + QU( 8098916250668367933LLU), QU( 3529200436790763465LLU), + QU(12988418908674681745LLU), QU( 6147567275954808580LLU), + QU( 3207503344604030989LLU), QU(10761592604898615360LLU), + QU( 229854861031893504LLU), QU( 8809853962667144291LLU), + QU(13957364469005693860LLU), QU( 7634287665224495886LLU), + QU(12353487366976556874LLU), QU( 1134423796317152034LLU), + QU( 2088992471334107068LLU), QU( 7393372127190799698LLU), + QU( 1845367839871058391LLU), QU( 207922563987322884LLU), + QU(11960870813159944976LLU), QU(12182120053317317363LLU), + QU(17307358132571709283LLU), QU(13871081155552824936LLU), + QU(18304446751741566262LLU), QU( 7178705220184302849LLU), + QU(10929605677758824425LLU), QU(16446976977835806844LLU), + QU(13723874412159769044LLU), QU( 6942854352100915216LLU), + QU( 1726308474365729390LLU), QU( 2150078766445323155LLU), + QU(15345558947919656626LLU), QU(12145453828874527201LLU), + QU( 2054448620739726849LLU), QU( 2740102003352628137LLU), + QU(11294462163577610655LLU), QU( 756164283387413743LLU), + QU(17841144758438810880LLU), QU(10802406021185415861LLU), + QU( 8716455530476737846LLU), QU( 6321788834517649606LLU), + QU(14681322910577468426LLU), QU(17330043563884336387LLU), + QU(12701802180050071614LLU), QU(14695105111079727151LLU), + QU( 5112098511654172830LLU), QU( 4957505496794139973LLU), + QU( 8270979451952045982LLU), QU(12307685939199120969LLU), + QU(12425799408953443032LLU), QU( 8376410143634796588LLU), + QU(16621778679680060464LLU), QU( 3580497854566660073LLU), + QU( 1122515747803382416LLU), QU( 857664980960597599LLU), + QU( 6343640119895925918LLU), QU(12878473260854462891LLU), + QU(10036813920765722626LLU), QU(14451335468363173812LLU), + QU( 5476809692401102807LLU), QU(16442255173514366342LLU), + QU(13060203194757167104LLU), QU(14354124071243177715LLU), + QU(15961249405696125227LLU), QU(13703893649690872584LLU), + QU( 363907326340340064LLU), QU( 6247455540491754842LLU), + QU(12242249332757832361LLU), QU( 156065475679796717LLU), + QU( 9351116235749732355LLU), QU( 4590350628677701405LLU), + QU( 1671195940982350389LLU), QU(13501398458898451905LLU), + QU( 6526341991225002255LLU), QU( 1689782913778157592LLU), + QU( 7439222350869010334LLU), QU(13975150263226478308LLU), + QU(11411961169932682710LLU), QU(17204271834833847277LLU), + QU( 541534742544435367LLU), QU( 6591191931218949684LLU), + QU( 2645454775478232486LLU), QU( 4322857481256485321LLU), + QU( 8477416487553065110LLU), QU(12902505428548435048LLU), + QU( 971445777981341415LLU), QU(14995104682744976712LLU), + QU( 4243341648807158063LLU), QU( 8695061252721927661LLU), + QU( 5028202003270177222LLU), QU( 2289257340915567840LLU), + QU(13870416345121866007LLU), QU(13994481698072092233LLU), + QU( 6912785400753196481LLU), QU( 2278309315841980139LLU), + QU( 4329765449648304839LLU), QU( 5963108095785485298LLU), + QU( 4880024847478722478LLU), QU(16015608779890240947LLU), + QU( 1866679034261393544LLU), QU( 914821179919731519LLU), + QU( 9643404035648760131LLU), QU( 2418114953615593915LLU), + QU( 944756836073702374LLU), QU(15186388048737296834LLU), + QU( 7723355336128442206LLU), QU( 7500747479679599691LLU), + QU(18013961306453293634LLU), QU( 2315274808095756456LLU), + QU(13655308255424029566LLU), QU(17203800273561677098LLU), + QU( 1382158694422087756LLU), QU( 5090390250309588976LLU), + QU( 517170818384213989LLU), QU( 1612709252627729621LLU), + QU( 1330118955572449606LLU), QU( 300922478056709885LLU), + QU(18115693291289091987LLU), QU(13491407109725238321LLU), + QU(15293714633593827320LLU), QU( 5151539373053314504LLU), + QU( 5951523243743139207LLU), QU(14459112015249527975LLU), + QU( 5456113959000700739LLU), QU( 3877918438464873016LLU), + QU(12534071654260163555LLU), QU(15871678376893555041LLU), + QU(11005484805712025549LLU), QU(16353066973143374252LLU), + QU( 4358331472063256685LLU), QU( 8268349332210859288LLU), + QU(12485161590939658075LLU), QU(13955993592854471343LLU), + QU( 5911446886848367039LLU), QU(14925834086813706974LLU), + QU( 6590362597857994805LLU), QU( 1280544923533661875LLU), + QU( 1637756018947988164LLU), QU( 4734090064512686329LLU), + QU(16693705263131485912LLU), QU( 6834882340494360958LLU), + QU( 8120732176159658505LLU), QU( 2244371958905329346LLU), + QU(10447499707729734021LLU), QU( 7318742361446942194LLU), + QU( 8032857516355555296LLU), QU(14023605983059313116LLU), + QU( 1032336061815461376LLU), QU( 9840995337876562612LLU), + QU( 9869256223029203587LLU), QU(12227975697177267636LLU), + QU(12728115115844186033LLU), QU( 7752058479783205470LLU), + QU( 729733219713393087LLU), QU(12954017801239007622LLU) +}; +static const uint64_t init_by_array_64_expected[] = { + QU( 2100341266307895239LLU), QU( 8344256300489757943LLU), + QU(15687933285484243894LLU), QU( 8268620370277076319LLU), + QU(12371852309826545459LLU), QU( 8800491541730110238LLU), + QU(18113268950100835773LLU), QU( 2886823658884438119LLU), + QU( 3293667307248180724LLU), QU( 9307928143300172731LLU), + QU( 7688082017574293629LLU), QU( 900986224735166665LLU), + QU( 9977972710722265039LLU), QU( 6008205004994830552LLU), + QU( 546909104521689292LLU), QU( 7428471521869107594LLU), + QU(14777563419314721179LLU), QU(16116143076567350053LLU), + QU( 5322685342003142329LLU), QU( 4200427048445863473LLU), + QU( 4693092150132559146LLU), QU(13671425863759338582LLU), + QU( 6747117460737639916LLU), QU( 4732666080236551150LLU), + QU( 5912839950611941263LLU), QU( 3903717554504704909LLU), + QU( 2615667650256786818LLU), QU(10844129913887006352LLU), + QU(13786467861810997820LLU), QU(14267853002994021570LLU), + QU(13767807302847237439LLU), QU(16407963253707224617LLU), + QU( 4802498363698583497LLU), QU( 2523802839317209764LLU), + QU( 3822579397797475589LLU), QU( 8950320572212130610LLU), + QU( 3745623504978342534LLU), QU(16092609066068482806LLU), + QU( 9817016950274642398LLU), QU(10591660660323829098LLU), + QU(11751606650792815920LLU), QU( 5122873818577122211LLU), + QU(17209553764913936624LLU), QU( 6249057709284380343LLU), + QU(15088791264695071830LLU), QU(15344673071709851930LLU), + QU( 4345751415293646084LLU), QU( 2542865750703067928LLU), + QU(13520525127852368784LLU), QU(18294188662880997241LLU), + QU( 3871781938044881523LLU), QU( 2873487268122812184LLU), + QU(15099676759482679005LLU), QU(15442599127239350490LLU), + QU( 6311893274367710888LLU), QU( 3286118760484672933LLU), + QU( 4146067961333542189LLU), QU(13303942567897208770LLU), + QU( 8196013722255630418LLU), QU( 4437815439340979989LLU), + QU(15433791533450605135LLU), QU( 4254828956815687049LLU), + QU( 1310903207708286015LLU), QU(10529182764462398549LLU), + QU(14900231311660638810LLU), QU( 9727017277104609793LLU), + QU( 1821308310948199033LLU), QU(11628861435066772084LLU), + QU( 9469019138491546924LLU), QU( 3145812670532604988LLU), + QU( 9938468915045491919LLU), QU( 1562447430672662142LLU), + QU(13963995266697989134LLU), QU( 3356884357625028695LLU), + QU( 4499850304584309747LLU), QU( 8456825817023658122LLU), + QU(10859039922814285279LLU), QU( 8099512337972526555LLU), + QU( 348006375109672149LLU), QU(11919893998241688603LLU), + QU( 1104199577402948826LLU), QU(16689191854356060289LLU), + QU(10992552041730168078LLU), QU( 7243733172705465836LLU), + QU( 5668075606180319560LLU), QU(18182847037333286970LLU), + QU( 4290215357664631322LLU), QU( 4061414220791828613LLU), + QU(13006291061652989604LLU), QU( 7140491178917128798LLU), + QU(12703446217663283481LLU), QU( 5500220597564558267LLU), + QU(10330551509971296358LLU), QU(15958554768648714492LLU), + QU( 5174555954515360045LLU), QU( 1731318837687577735LLU), + QU( 3557700801048354857LLU), QU(13764012341928616198LLU), + QU(13115166194379119043LLU), QU( 7989321021560255519LLU), + QU( 2103584280905877040LLU), QU( 9230788662155228488LLU), + QU(16396629323325547654LLU), QU( 657926409811318051LLU), + QU(15046700264391400727LLU), QU( 5120132858771880830LLU), + QU( 7934160097989028561LLU), QU( 6963121488531976245LLU), + QU(17412329602621742089LLU), QU(15144843053931774092LLU), + QU(17204176651763054532LLU), QU(13166595387554065870LLU), + QU( 8590377810513960213LLU), QU( 5834365135373991938LLU), + QU( 7640913007182226243LLU), QU( 3479394703859418425LLU), + QU(16402784452644521040LLU), QU( 4993979809687083980LLU), + QU(13254522168097688865LLU), QU(15643659095244365219LLU), + QU( 5881437660538424982LLU), QU(11174892200618987379LLU), + QU( 254409966159711077LLU), QU(17158413043140549909LLU), + QU( 3638048789290376272LLU), QU( 1376816930299489190LLU), + QU( 4622462095217761923LLU), QU(15086407973010263515LLU), + QU(13253971772784692238LLU), QU( 5270549043541649236LLU), + QU(11182714186805411604LLU), QU(12283846437495577140LLU), + QU( 5297647149908953219LLU), QU(10047451738316836654LLU), + QU( 4938228100367874746LLU), QU(12328523025304077923LLU), + QU( 3601049438595312361LLU), QU( 9313624118352733770LLU), + QU(13322966086117661798LLU), QU(16660005705644029394LLU), + QU(11337677526988872373LLU), QU(13869299102574417795LLU), + QU(15642043183045645437LLU), QU( 3021755569085880019LLU), + QU( 4979741767761188161LLU), QU(13679979092079279587LLU), + QU( 3344685842861071743LLU), QU(13947960059899588104LLU), + QU( 305806934293368007LLU), QU( 5749173929201650029LLU), + QU(11123724852118844098LLU), QU(15128987688788879802LLU), + QU(15251651211024665009LLU), QU( 7689925933816577776LLU), + QU(16732804392695859449LLU), QU(17087345401014078468LLU), + QU(14315108589159048871LLU), QU( 4820700266619778917LLU), + QU(16709637539357958441LLU), QU( 4936227875177351374LLU), + QU( 2137907697912987247LLU), QU(11628565601408395420LLU), + QU( 2333250549241556786LLU), QU( 5711200379577778637LLU), + QU( 5170680131529031729LLU), QU(12620392043061335164LLU), + QU( 95363390101096078LLU), QU( 5487981914081709462LLU), + QU( 1763109823981838620LLU), QU( 3395861271473224396LLU), + QU( 1300496844282213595LLU), QU( 6894316212820232902LLU), + QU(10673859651135576674LLU), QU( 5911839658857903252LLU), + QU(17407110743387299102LLU), QU( 8257427154623140385LLU), + QU(11389003026741800267LLU), QU( 4070043211095013717LLU), + QU(11663806997145259025LLU), QU(15265598950648798210LLU), + QU( 630585789434030934LLU), QU( 3524446529213587334LLU), + QU( 7186424168495184211LLU), QU(10806585451386379021LLU), + QU(11120017753500499273LLU), QU( 1586837651387701301LLU), + QU(17530454400954415544LLU), QU( 9991670045077880430LLU), + QU( 7550997268990730180LLU), QU( 8640249196597379304LLU), + QU( 3522203892786893823LLU), QU(10401116549878854788LLU), + QU(13690285544733124852LLU), QU( 8295785675455774586LLU), + QU(15535716172155117603LLU), QU( 3112108583723722511LLU), + QU(17633179955339271113LLU), QU(18154208056063759375LLU), + QU( 1866409236285815666LLU), QU(13326075895396412882LLU), + QU( 8756261842948020025LLU), QU( 6281852999868439131LLU), + QU(15087653361275292858LLU), QU(10333923911152949397LLU), + QU( 5265567645757408500LLU), QU(12728041843210352184LLU), + QU( 6347959327507828759LLU), QU( 154112802625564758LLU), + QU(18235228308679780218LLU), QU( 3253805274673352418LLU), + QU( 4849171610689031197LLU), QU(17948529398340432518LLU), + QU(13803510475637409167LLU), QU(13506570190409883095LLU), + QU(15870801273282960805LLU), QU( 8451286481299170773LLU), + QU( 9562190620034457541LLU), QU( 8518905387449138364LLU), + QU(12681306401363385655LLU), QU( 3788073690559762558LLU), + QU( 5256820289573487769LLU), QU( 2752021372314875467LLU), + QU( 6354035166862520716LLU), QU( 4328956378309739069LLU), + QU( 449087441228269600LLU), QU( 5533508742653090868LLU), + QU( 1260389420404746988LLU), QU(18175394473289055097LLU), + QU( 1535467109660399420LLU), QU( 8818894282874061442LLU), + QU(12140873243824811213LLU), QU(15031386653823014946LLU), + QU( 1286028221456149232LLU), QU( 6329608889367858784LLU), + QU( 9419654354945132725LLU), QU( 6094576547061672379LLU), + QU(17706217251847450255LLU), QU( 1733495073065878126LLU), + QU(16918923754607552663LLU), QU( 8881949849954945044LLU), + QU(12938977706896313891LLU), QU(14043628638299793407LLU), + QU(18393874581723718233LLU), QU( 6886318534846892044LLU), + QU(14577870878038334081LLU), QU(13541558383439414119LLU), + QU(13570472158807588273LLU), QU(18300760537910283361LLU), + QU( 818368572800609205LLU), QU( 1417000585112573219LLU), + QU(12337533143867683655LLU), QU(12433180994702314480LLU), + QU( 778190005829189083LLU), QU(13667356216206524711LLU), + QU( 9866149895295225230LLU), QU(11043240490417111999LLU), + QU( 1123933826541378598LLU), QU( 6469631933605123610LLU), + QU(14508554074431980040LLU), QU(13918931242962026714LLU), + QU( 2870785929342348285LLU), QU(14786362626740736974LLU), + QU(13176680060902695786LLU), QU( 9591778613541679456LLU), + QU( 9097662885117436706LLU), QU( 749262234240924947LLU), + QU( 1944844067793307093LLU), QU( 4339214904577487742LLU), + QU( 8009584152961946551LLU), QU(16073159501225501777LLU), + QU( 3335870590499306217LLU), QU(17088312653151202847LLU), + QU( 3108893142681931848LLU), QU(16636841767202792021LLU), + QU(10423316431118400637LLU), QU( 8008357368674443506LLU), + QU(11340015231914677875LLU), QU(17687896501594936090LLU), + QU(15173627921763199958LLU), QU( 542569482243721959LLU), + QU(15071714982769812975LLU), QU( 4466624872151386956LLU), + QU( 1901780715602332461LLU), QU( 9822227742154351098LLU), + QU( 1479332892928648780LLU), QU( 6981611948382474400LLU), + QU( 7620824924456077376LLU), QU(14095973329429406782LLU), + QU( 7902744005696185404LLU), QU(15830577219375036920LLU), + QU(10287076667317764416LLU), QU(12334872764071724025LLU), + QU( 4419302088133544331LLU), QU(14455842851266090520LLU), + QU(12488077416504654222LLU), QU( 7953892017701886766LLU), + QU( 6331484925529519007LLU), QU( 4902145853785030022LLU), + QU(17010159216096443073LLU), QU(11945354668653886087LLU), + QU(15112022728645230829LLU), QU(17363484484522986742LLU), + QU( 4423497825896692887LLU), QU( 8155489510809067471LLU), + QU( 258966605622576285LLU), QU( 5462958075742020534LLU), + QU( 6763710214913276228LLU), QU( 2368935183451109054LLU), + QU(14209506165246453811LLU), QU( 2646257040978514881LLU), + QU( 3776001911922207672LLU), QU( 1419304601390147631LLU), + QU(14987366598022458284LLU), QU( 3977770701065815721LLU), + QU( 730820417451838898LLU), QU( 3982991703612885327LLU), + QU( 2803544519671388477LLU), QU(17067667221114424649LLU), + QU( 2922555119737867166LLU), QU( 1989477584121460932LLU), + QU(15020387605892337354LLU), QU( 9293277796427533547LLU), + QU(10722181424063557247LLU), QU(16704542332047511651LLU), + QU( 5008286236142089514LLU), QU(16174732308747382540LLU), + QU(17597019485798338402LLU), QU(13081745199110622093LLU), + QU( 8850305883842258115LLU), QU(12723629125624589005LLU), + QU( 8140566453402805978LLU), QU(15356684607680935061LLU), + QU(14222190387342648650LLU), QU(11134610460665975178LLU), + QU( 1259799058620984266LLU), QU(13281656268025610041LLU), + QU( 298262561068153992LLU), QU(12277871700239212922LLU), + QU(13911297774719779438LLU), QU(16556727962761474934LLU), + QU(17903010316654728010LLU), QU( 9682617699648434744LLU), + QU(14757681836838592850LLU), QU( 1327242446558524473LLU), + QU(11126645098780572792LLU), QU( 1883602329313221774LLU), + QU( 2543897783922776873LLU), QU(15029168513767772842LLU), + QU(12710270651039129878LLU), QU(16118202956069604504LLU), + QU(15010759372168680524LLU), QU( 2296827082251923948LLU), + QU(10793729742623518101LLU), QU(13829764151845413046LLU), + QU(17769301223184451213LLU), QU( 3118268169210783372LLU), + QU(17626204544105123127LLU), QU( 7416718488974352644LLU), + QU(10450751996212925994LLU), QU( 9352529519128770586LLU), + QU( 259347569641110140LLU), QU( 8048588892269692697LLU), + QU( 1774414152306494058LLU), QU(10669548347214355622LLU), + QU(13061992253816795081LLU), QU(18432677803063861659LLU), + QU( 8879191055593984333LLU), QU(12433753195199268041LLU), + QU(14919392415439730602LLU), QU( 6612848378595332963LLU), + QU( 6320986812036143628LLU), QU(10465592420226092859LLU), + QU( 4196009278962570808LLU), QU( 3747816564473572224LLU), + QU(17941203486133732898LLU), QU( 2350310037040505198LLU), + QU( 5811779859134370113LLU), QU(10492109599506195126LLU), + QU( 7699650690179541274LLU), QU( 1954338494306022961LLU), + QU(14095816969027231152LLU), QU( 5841346919964852061LLU), + QU(14945969510148214735LLU), QU( 3680200305887550992LLU), + QU( 6218047466131695792LLU), QU( 8242165745175775096LLU), + QU(11021371934053307357LLU), QU( 1265099502753169797LLU), + QU( 4644347436111321718LLU), QU( 3609296916782832859LLU), + QU( 8109807992218521571LLU), QU(18387884215648662020LLU), + QU(14656324896296392902LLU), QU(17386819091238216751LLU), + QU(17788300878582317152LLU), QU( 7919446259742399591LLU), + QU( 4466613134576358004LLU), QU(12928181023667938509LLU), + QU(13147446154454932030LLU), QU(16552129038252734620LLU), + QU( 8395299403738822450LLU), QU(11313817655275361164LLU), + QU( 434258809499511718LLU), QU( 2074882104954788676LLU), + QU( 7929892178759395518LLU), QU( 9006461629105745388LLU), + QU( 5176475650000323086LLU), QU(11128357033468341069LLU), + QU(12026158851559118955LLU), QU(14699716249471156500LLU), + QU( 448982497120206757LLU), QU( 4156475356685519900LLU), + QU( 6063816103417215727LLU), QU(10073289387954971479LLU), + QU( 8174466846138590962LLU), QU( 2675777452363449006LLU), + QU( 9090685420572474281LLU), QU( 6659652652765562060LLU), + QU(12923120304018106621LLU), QU(11117480560334526775LLU), + QU( 937910473424587511LLU), QU( 1838692113502346645LLU), + QU(11133914074648726180LLU), QU( 7922600945143884053LLU), + QU(13435287702700959550LLU), QU( 5287964921251123332LLU), + QU(11354875374575318947LLU), QU(17955724760748238133LLU), + QU(13728617396297106512LLU), QU( 4107449660118101255LLU), + QU( 1210269794886589623LLU), QU(11408687205733456282LLU), + QU( 4538354710392677887LLU), QU(13566803319341319267LLU), + QU(17870798107734050771LLU), QU( 3354318982568089135LLU), + QU( 9034450839405133651LLU), QU(13087431795753424314LLU), + QU( 950333102820688239LLU), QU( 1968360654535604116LLU), + QU(16840551645563314995LLU), QU( 8867501803892924995LLU), + QU(11395388644490626845LLU), QU( 1529815836300732204LLU), + QU(13330848522996608842LLU), QU( 1813432878817504265LLU), + QU( 2336867432693429560LLU), QU(15192805445973385902LLU), + QU( 2528593071076407877LLU), QU( 128459777936689248LLU), + QU( 9976345382867214866LLU), QU( 6208885766767996043LLU), + QU(14982349522273141706LLU), QU( 3099654362410737822LLU), + QU(13776700761947297661LLU), QU( 8806185470684925550LLU), + QU( 8151717890410585321LLU), QU( 640860591588072925LLU), + QU(14592096303937307465LLU), QU( 9056472419613564846LLU), + QU(14861544647742266352LLU), QU(12703771500398470216LLU), + QU( 3142372800384138465LLU), QU( 6201105606917248196LLU), + QU(18337516409359270184LLU), QU(15042268695665115339LLU), + QU(15188246541383283846LLU), QU(12800028693090114519LLU), + QU( 5992859621101493472LLU), QU(18278043971816803521LLU), + QU( 9002773075219424560LLU), QU( 7325707116943598353LLU), + QU( 7930571931248040822LLU), QU( 5645275869617023448LLU), + QU( 7266107455295958487LLU), QU( 4363664528273524411LLU), + QU(14313875763787479809LLU), QU(17059695613553486802LLU), + QU( 9247761425889940932LLU), QU(13704726459237593128LLU), + QU( 2701312427328909832LLU), QU(17235532008287243115LLU), + QU(14093147761491729538LLU), QU( 6247352273768386516LLU), + QU( 8268710048153268415LLU), QU( 7985295214477182083LLU), + QU(15624495190888896807LLU), QU( 3772753430045262788LLU), + QU( 9133991620474991698LLU), QU( 5665791943316256028LLU), + QU( 7551996832462193473LLU), QU(13163729206798953877LLU), + QU( 9263532074153846374LLU), QU( 1015460703698618353LLU), + QU(17929874696989519390LLU), QU(18257884721466153847LLU), + QU(16271867543011222991LLU), QU( 3905971519021791941LLU), + QU(16814488397137052085LLU), QU( 1321197685504621613LLU), + QU( 2870359191894002181LLU), QU(14317282970323395450LLU), + QU(13663920845511074366LLU), QU( 2052463995796539594LLU), + QU(14126345686431444337LLU), QU( 1727572121947022534LLU), + QU(17793552254485594241LLU), QU( 6738857418849205750LLU), + QU( 1282987123157442952LLU), QU(16655480021581159251LLU), + QU( 6784587032080183866LLU), QU(14726758805359965162LLU), + QU( 7577995933961987349LLU), QU(12539609320311114036LLU), + QU(10789773033385439494LLU), QU( 8517001497411158227LLU), + QU(10075543932136339710LLU), QU(14838152340938811081LLU), + QU( 9560840631794044194LLU), QU(17445736541454117475LLU), + QU(10633026464336393186LLU), QU(15705729708242246293LLU), + QU( 1117517596891411098LLU), QU( 4305657943415886942LLU), + QU( 4948856840533979263LLU), QU(16071681989041789593LLU), + QU(13723031429272486527LLU), QU( 7639567622306509462LLU), + QU(12670424537483090390LLU), QU( 9715223453097197134LLU), + QU( 5457173389992686394LLU), QU( 289857129276135145LLU), + QU(17048610270521972512LLU), QU( 692768013309835485LLU), + QU(14823232360546632057LLU), QU(18218002361317895936LLU), + QU( 3281724260212650204LLU), QU(16453957266549513795LLU), + QU( 8592711109774511881LLU), QU( 929825123473369579LLU), + QU(15966784769764367791LLU), QU( 9627344291450607588LLU), + QU(10849555504977813287LLU), QU( 9234566913936339275LLU), + QU( 6413807690366911210LLU), QU(10862389016184219267LLU), + QU(13842504799335374048LLU), QU( 1531994113376881174LLU), + QU( 2081314867544364459LLU), QU(16430628791616959932LLU), + QU( 8314714038654394368LLU), QU( 9155473892098431813LLU), + QU(12577843786670475704LLU), QU( 4399161106452401017LLU), + QU( 1668083091682623186LLU), QU( 1741383777203714216LLU), + QU( 2162597285417794374LLU), QU(15841980159165218736LLU), + QU( 1971354603551467079LLU), QU( 1206714764913205968LLU), + QU( 4790860439591272330LLU), QU(14699375615594055799LLU), + QU( 8374423871657449988LLU), QU(10950685736472937738LLU), + QU( 697344331343267176LLU), QU(10084998763118059810LLU), + QU(12897369539795983124LLU), QU(12351260292144383605LLU), + QU( 1268810970176811234LLU), QU( 7406287800414582768LLU), + QU( 516169557043807831LLU), QU( 5077568278710520380LLU), + QU( 3828791738309039304LLU), QU( 7721974069946943610LLU), + QU( 3534670260981096460LLU), QU( 4865792189600584891LLU), + QU(16892578493734337298LLU), QU( 9161499464278042590LLU), + QU(11976149624067055931LLU), QU(13219479887277343990LLU), + QU(14161556738111500680LLU), QU(14670715255011223056LLU), + QU( 4671205678403576558LLU), QU(12633022931454259781LLU), + QU(14821376219869187646LLU), QU( 751181776484317028LLU), + QU( 2192211308839047070LLU), QU(11787306362361245189LLU), + QU(10672375120744095707LLU), QU( 4601972328345244467LLU), + QU(15457217788831125879LLU), QU( 8464345256775460809LLU), + QU(10191938789487159478LLU), QU( 6184348739615197613LLU), + QU(11425436778806882100LLU), QU( 2739227089124319793LLU), + QU( 461464518456000551LLU), QU( 4689850170029177442LLU), + QU( 6120307814374078625LLU), QU(11153579230681708671LLU), + QU( 7891721473905347926LLU), QU(10281646937824872400LLU), + QU( 3026099648191332248LLU), QU( 8666750296953273818LLU), + QU(14978499698844363232LLU), QU(13303395102890132065LLU), + QU( 8182358205292864080LLU), QU(10560547713972971291LLU), + QU(11981635489418959093LLU), QU( 3134621354935288409LLU), + QU(11580681977404383968LLU), QU(14205530317404088650LLU), + QU( 5997789011854923157LLU), QU(13659151593432238041LLU), + QU(11664332114338865086LLU), QU( 7490351383220929386LLU), + QU( 7189290499881530378LLU), QU(15039262734271020220LLU), + QU( 2057217285976980055LLU), QU( 555570804905355739LLU), + QU(11235311968348555110LLU), QU(13824557146269603217LLU), + QU(16906788840653099693LLU), QU( 7222878245455661677LLU), + QU( 5245139444332423756LLU), QU( 4723748462805674292LLU), + QU(12216509815698568612LLU), QU(17402362976648951187LLU), + QU(17389614836810366768LLU), QU( 4880936484146667711LLU), + QU( 9085007839292639880LLU), QU(13837353458498535449LLU), + QU(11914419854360366677LLU), QU(16595890135313864103LLU), + QU( 6313969847197627222LLU), QU(18296909792163910431LLU), + QU(10041780113382084042LLU), QU( 2499478551172884794LLU), + QU(11057894246241189489LLU), QU( 9742243032389068555LLU), + QU(12838934582673196228LLU), QU(13437023235248490367LLU), + QU(13372420669446163240LLU), QU( 6752564244716909224LLU), + QU( 7157333073400313737LLU), QU(12230281516370654308LLU), + QU( 1182884552219419117LLU), QU( 2955125381312499218LLU), + QU(10308827097079443249LLU), QU( 1337648572986534958LLU), + QU(16378788590020343939LLU), QU( 108619126514420935LLU), + QU( 3990981009621629188LLU), QU( 5460953070230946410LLU), + QU( 9703328329366531883LLU), QU(13166631489188077236LLU), + QU( 1104768831213675170LLU), QU( 3447930458553877908LLU), + QU( 8067172487769945676LLU), QU( 5445802098190775347LLU), + QU( 3244840981648973873LLU), QU(17314668322981950060LLU), + QU( 5006812527827763807LLU), QU(18158695070225526260LLU), + QU( 2824536478852417853LLU), QU(13974775809127519886LLU), + QU( 9814362769074067392LLU), QU(17276205156374862128LLU), + QU(11361680725379306967LLU), QU( 3422581970382012542LLU), + QU(11003189603753241266LLU), QU(11194292945277862261LLU), + QU( 6839623313908521348LLU), QU(11935326462707324634LLU), + QU( 1611456788685878444LLU), QU(13112620989475558907LLU), + QU( 517659108904450427LLU), QU(13558114318574407624LLU), + QU(15699089742731633077LLU), QU( 4988979278862685458LLU), + QU( 8111373583056521297LLU), QU( 3891258746615399627LLU), + QU( 8137298251469718086LLU), QU(12748663295624701649LLU), + QU( 4389835683495292062LLU), QU( 5775217872128831729LLU), + QU( 9462091896405534927LLU), QU( 8498124108820263989LLU), + QU( 8059131278842839525LLU), QU(10503167994254090892LLU), + QU(11613153541070396656LLU), QU(18069248738504647790LLU), + QU( 570657419109768508LLU), QU( 3950574167771159665LLU), + QU( 5514655599604313077LLU), QU( 2908460854428484165LLU), + QU(10777722615935663114LLU), QU(12007363304839279486LLU), + QU( 9800646187569484767LLU), QU( 8795423564889864287LLU), + QU(14257396680131028419LLU), QU( 6405465117315096498LLU), + QU( 7939411072208774878LLU), QU(17577572378528990006LLU), + QU(14785873806715994850LLU), QU(16770572680854747390LLU), + QU(18127549474419396481LLU), QU(11637013449455757750LLU), + QU(14371851933996761086LLU), QU( 3601181063650110280LLU), + QU( 4126442845019316144LLU), QU(10198287239244320669LLU), + QU(18000169628555379659LLU), QU(18392482400739978269LLU), + QU( 6219919037686919957LLU), QU( 3610085377719446052LLU), + QU( 2513925039981776336LLU), QU(16679413537926716955LLU), + QU(12903302131714909434LLU), QU( 5581145789762985009LLU), + QU(12325955044293303233LLU), QU(17216111180742141204LLU), + QU( 6321919595276545740LLU), QU( 3507521147216174501LLU), + QU( 9659194593319481840LLU), QU(11473976005975358326LLU), + QU(14742730101435987026LLU), QU( 492845897709954780LLU), + QU(16976371186162599676LLU), QU(17712703422837648655LLU), + QU( 9881254778587061697LLU), QU( 8413223156302299551LLU), + QU( 1563841828254089168LLU), QU( 9996032758786671975LLU), + QU( 138877700583772667LLU), QU(13003043368574995989LLU), + QU( 4390573668650456587LLU), QU( 8610287390568126755LLU), + QU(15126904974266642199LLU), QU( 6703637238986057662LLU), + QU( 2873075592956810157LLU), QU( 6035080933946049418LLU), + QU(13382846581202353014LLU), QU( 7303971031814642463LLU), + QU(18418024405307444267LLU), QU( 5847096731675404647LLU), + QU( 4035880699639842500LLU), QU(11525348625112218478LLU), + QU( 3041162365459574102LLU), QU( 2604734487727986558LLU), + QU(15526341771636983145LLU), QU(14556052310697370254LLU), + QU(12997787077930808155LLU), QU( 9601806501755554499LLU), + QU(11349677952521423389LLU), QU(14956777807644899350LLU), + QU(16559736957742852721LLU), QU(12360828274778140726LLU), + QU( 6685373272009662513LLU), QU(16932258748055324130LLU), + QU(15918051131954158508LLU), QU( 1692312913140790144LLU), + QU( 546653826801637367LLU), QU( 5341587076045986652LLU), + QU(14975057236342585662LLU), QU(12374976357340622412LLU), + QU(10328833995181940552LLU), QU(12831807101710443149LLU), + QU(10548514914382545716LLU), QU( 2217806727199715993LLU), + QU(12627067369242845138LLU), QU( 4598965364035438158LLU), + QU( 150923352751318171LLU), QU(14274109544442257283LLU), + QU( 4696661475093863031LLU), QU( 1505764114384654516LLU), + QU(10699185831891495147LLU), QU( 2392353847713620519LLU), + QU( 3652870166711788383LLU), QU( 8640653276221911108LLU), + QU( 3894077592275889704LLU), QU( 4918592872135964845LLU), + QU(16379121273281400789LLU), QU(12058465483591683656LLU), + QU(11250106829302924945LLU), QU( 1147537556296983005LLU), + QU( 6376342756004613268LLU), QU(14967128191709280506LLU), + QU(18007449949790627628LLU), QU( 9497178279316537841LLU), + QU( 7920174844809394893LLU), QU(10037752595255719907LLU), + QU(15875342784985217697LLU), QU(15311615921712850696LLU), + QU( 9552902652110992950LLU), QU(14054979450099721140LLU), + QU( 5998709773566417349LLU), QU(18027910339276320187LLU), + QU( 8223099053868585554LLU), QU( 7842270354824999767LLU), + QU( 4896315688770080292LLU), QU(12969320296569787895LLU), + QU( 2674321489185759961LLU), QU( 4053615936864718439LLU), + QU(11349775270588617578LLU), QU( 4743019256284553975LLU), + QU( 5602100217469723769LLU), QU(14398995691411527813LLU), + QU( 7412170493796825470LLU), QU( 836262406131744846LLU), + QU( 8231086633845153022LLU), QU( 5161377920438552287LLU), + QU( 8828731196169924949LLU), QU(16211142246465502680LLU), + QU( 3307990879253687818LLU), QU( 5193405406899782022LLU), + QU( 8510842117467566693LLU), QU( 6070955181022405365LLU), + QU(14482950231361409799LLU), QU(12585159371331138077LLU), + QU( 3511537678933588148LLU), QU( 2041849474531116417LLU), + QU(10944936685095345792LLU), QU(18303116923079107729LLU), + QU( 2720566371239725320LLU), QU( 4958672473562397622LLU), + QU( 3032326668253243412LLU), QU(13689418691726908338LLU), + QU( 1895205511728843996LLU), QU( 8146303515271990527LLU), + QU(16507343500056113480LLU), QU( 473996939105902919LLU), + QU( 9897686885246881481LLU), QU(14606433762712790575LLU), + QU( 6732796251605566368LLU), QU( 1399778120855368916LLU), + QU( 935023885182833777LLU), QU(16066282816186753477LLU), + QU( 7291270991820612055LLU), QU(17530230393129853844LLU), + QU(10223493623477451366LLU), QU(15841725630495676683LLU), + QU(17379567246435515824LLU), QU( 8588251429375561971LLU), + QU(18339511210887206423LLU), QU(17349587430725976100LLU), + QU(12244876521394838088LLU), QU( 6382187714147161259LLU), + QU(12335807181848950831LLU), QU(16948885622305460665LLU), + QU(13755097796371520506LLU), QU(14806740373324947801LLU), + QU( 4828699633859287703LLU), QU( 8209879281452301604LLU), + QU(12435716669553736437LLU), QU(13970976859588452131LLU), + QU( 6233960842566773148LLU), QU(12507096267900505759LLU), + QU( 1198713114381279421LLU), QU(14989862731124149015LLU), + QU(15932189508707978949LLU), QU( 2526406641432708722LLU), + QU( 29187427817271982LLU), QU( 1499802773054556353LLU), + QU(10816638187021897173LLU), QU( 5436139270839738132LLU), + QU( 6659882287036010082LLU), QU( 2154048955317173697LLU), + QU(10887317019333757642LLU), QU(16281091802634424955LLU), + QU(10754549879915384901LLU), QU(10760611745769249815LLU), + QU( 2161505946972504002LLU), QU( 5243132808986265107LLU), + QU(10129852179873415416LLU), QU( 710339480008649081LLU), + QU( 7802129453068808528LLU), QU(17967213567178907213LLU), + QU(15730859124668605599LLU), QU(13058356168962376502LLU), + QU( 3701224985413645909LLU), QU(14464065869149109264LLU), + QU( 9959272418844311646LLU), QU(10157426099515958752LLU), + QU(14013736814538268528LLU), QU(17797456992065653951LLU), + QU(17418878140257344806LLU), QU(15457429073540561521LLU), + QU( 2184426881360949378LLU), QU( 2062193041154712416LLU), + QU( 8553463347406931661LLU), QU( 4913057625202871854LLU), + QU( 2668943682126618425LLU), QU(17064444737891172288LLU), + QU( 4997115903913298637LLU), QU(12019402608892327416LLU), + QU(17603584559765897352LLU), QU(11367529582073647975LLU), + QU( 8211476043518436050LLU), QU( 8676849804070323674LLU), + QU(18431829230394475730LLU), QU(10490177861361247904LLU), + QU( 9508720602025651349LLU), QU( 7409627448555722700LLU), + QU( 5804047018862729008LLU), QU(11943858176893142594LLU), + QU(11908095418933847092LLU), QU( 5415449345715887652LLU), + QU( 1554022699166156407LLU), QU( 9073322106406017161LLU), + QU( 7080630967969047082LLU), QU(18049736940860732943LLU), + QU(12748714242594196794LLU), QU( 1226992415735156741LLU), + QU(17900981019609531193LLU), QU(11720739744008710999LLU), + QU( 3006400683394775434LLU), QU(11347974011751996028LLU), + QU( 3316999628257954608LLU), QU( 8384484563557639101LLU), + QU(18117794685961729767LLU), QU( 1900145025596618194LLU), + QU(17459527840632892676LLU), QU( 5634784101865710994LLU), + QU( 7918619300292897158LLU), QU( 3146577625026301350LLU), + QU( 9955212856499068767LLU), QU( 1873995843681746975LLU), + QU( 1561487759967972194LLU), QU( 8322718804375878474LLU), + QU(11300284215327028366LLU), QU( 4667391032508998982LLU), + QU( 9820104494306625580LLU), QU(17922397968599970610LLU), + QU( 1784690461886786712LLU), QU(14940365084341346821LLU), + QU( 5348719575594186181LLU), QU(10720419084507855261LLU), + QU(14210394354145143274LLU), QU( 2426468692164000131LLU), + QU(16271062114607059202LLU), QU(14851904092357070247LLU), + QU( 6524493015693121897LLU), QU( 9825473835127138531LLU), + QU(14222500616268569578LLU), QU(15521484052007487468LLU), + QU(14462579404124614699LLU), QU(11012375590820665520LLU), + QU(11625327350536084927LLU), QU(14452017765243785417LLU), + QU( 9989342263518766305LLU), QU( 3640105471101803790LLU), + QU( 4749866455897513242LLU), QU(13963064946736312044LLU), + QU(10007416591973223791LLU), QU(18314132234717431115LLU), + QU( 3286596588617483450LLU), QU( 7726163455370818765LLU), + QU( 7575454721115379328LLU), QU( 5308331576437663422LLU), + QU(18288821894903530934LLU), QU( 8028405805410554106LLU), + QU(15744019832103296628LLU), QU( 149765559630932100LLU), + QU( 6137705557200071977LLU), QU(14513416315434803615LLU), + QU(11665702820128984473LLU), QU( 218926670505601386LLU), + QU( 6868675028717769519LLU), QU(15282016569441512302LLU), + QU( 5707000497782960236LLU), QU( 6671120586555079567LLU), + QU( 2194098052618985448LLU), QU(16849577895477330978LLU), + QU(12957148471017466283LLU), QU( 1997805535404859393LLU), + QU( 1180721060263860490LLU), QU(13206391310193756958LLU), + QU(12980208674461861797LLU), QU( 3825967775058875366LLU), + QU(17543433670782042631LLU), QU( 1518339070120322730LLU), + QU(16344584340890991669LLU), QU( 2611327165318529819LLU), + QU(11265022723283422529LLU), QU( 4001552800373196817LLU), + QU(14509595890079346161LLU), QU( 3528717165416234562LLU), + QU(18153222571501914072LLU), QU( 9387182977209744425LLU), + QU(10064342315985580021LLU), QU(11373678413215253977LLU), + QU( 2308457853228798099LLU), QU( 9729042942839545302LLU), + QU( 7833785471140127746LLU), QU( 6351049900319844436LLU), + QU(14454610627133496067LLU), QU(12533175683634819111LLU), + QU(15570163926716513029LLU), QU(13356980519185762498LLU) +}; + +TEST_BEGIN(test_gen_rand_32) +{ + uint32_t array32[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + uint32_t array32_2[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + int i; + uint32_t r32; + sfmt_t *ctx; + + assert_d_le(get_min_array_size32(), BLOCK_SIZE, + "Array size too small"); + ctx = init_gen_rand(1234); + fill_array32(ctx, array32, BLOCK_SIZE); + fill_array32(ctx, array32_2, BLOCK_SIZE); + fini_gen_rand(ctx); + + ctx = init_gen_rand(1234); + for (i = 0; i < BLOCK_SIZE; i++) { + if (i < COUNT_1) { + assert_u32_eq(array32[i], init_gen_rand_32_expected[i], + "Output mismatch for i=%d", i); + } + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32[i], + "Mismatch at array32[%d]=%x, gen=%x", i, array32[i], r32); + } + for (i = 0; i < COUNT_2; i++) { + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32_2[i], + "Mismatch at array32_2[%d]=%x, gen=%x", i, array32_2[i], + r32); + } + fini_gen_rand(ctx); +} +TEST_END + +TEST_BEGIN(test_by_array_32) +{ + uint32_t array32[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + uint32_t array32_2[BLOCK_SIZE] JEMALLOC_ATTR(aligned(16)); + int i; + uint32_t ini[4] = {0x1234, 0x5678, 0x9abc, 0xdef0}; + uint32_t r32; + sfmt_t *ctx; + + assert_d_le(get_min_array_size32(), BLOCK_SIZE, + "Array size too small"); + ctx = init_by_array(ini, 4); + fill_array32(ctx, array32, BLOCK_SIZE); + fill_array32(ctx, array32_2, BLOCK_SIZE); + fini_gen_rand(ctx); + + ctx = init_by_array(ini, 4); + for (i = 0; i < BLOCK_SIZE; i++) { + if (i < COUNT_1) { + assert_u32_eq(array32[i], init_by_array_32_expected[i], + "Output mismatch for i=%d", i); + } + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32[i], + "Mismatch at array32[%d]=%x, gen=%x", i, array32[i], r32); + } + for (i = 0; i < COUNT_2; i++) { + r32 = gen_rand32(ctx); + assert_u32_eq(r32, array32_2[i], + "Mismatch at array32_2[%d]=%x, gen=%x", i, array32_2[i], + r32); + } + fini_gen_rand(ctx); +} +TEST_END + +TEST_BEGIN(test_gen_rand_64) +{ + uint64_t array64[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + uint64_t array64_2[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + int i; + uint64_t r; + sfmt_t *ctx; + + assert_d_le(get_min_array_size64(), BLOCK_SIZE64, + "Array size too small"); + ctx = init_gen_rand(4321); + fill_array64(ctx, array64, BLOCK_SIZE64); + fill_array64(ctx, array64_2, BLOCK_SIZE64); + fini_gen_rand(ctx); + + ctx = init_gen_rand(4321); + for (i = 0; i < BLOCK_SIZE64; i++) { + if (i < COUNT_1) { + assert_u64_eq(array64[i], init_gen_rand_64_expected[i], + "Output mismatch for i=%d", i); + } + r = gen_rand64(ctx); + assert_u64_eq(r, array64[i], + "Mismatch at array64[%d]=%"PRIx64", gen=%"PRIx64, i, + array64[i], r); + } + for (i = 0; i < COUNT_2; i++) { + r = gen_rand64(ctx); + assert_u64_eq(r, array64_2[i], + "Mismatch at array64_2[%d]=%"PRIx64" gen=%"PRIx64"", i, + array64_2[i], r); + } + fini_gen_rand(ctx); +} +TEST_END + +TEST_BEGIN(test_by_array_64) +{ + uint64_t array64[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + uint64_t array64_2[BLOCK_SIZE64] JEMALLOC_ATTR(aligned(16)); + int i; + uint64_t r; + uint32_t ini[] = {5, 4, 3, 2, 1}; + sfmt_t *ctx; + + assert_d_le(get_min_array_size64(), BLOCK_SIZE64, + "Array size too small"); + ctx = init_by_array(ini, 5); + fill_array64(ctx, array64, BLOCK_SIZE64); + fill_array64(ctx, array64_2, BLOCK_SIZE64); + fini_gen_rand(ctx); + + ctx = init_by_array(ini, 5); + for (i = 0; i < BLOCK_SIZE64; i++) { + if (i < COUNT_1) { + assert_u64_eq(array64[i], init_by_array_64_expected[i], + "Output mismatch for i=%d", i); + } + r = gen_rand64(ctx); + assert_u64_eq(r, array64[i], + "Mismatch at array64[%d]=%"PRIx64" gen=%"PRIx64, i, + array64[i], r); + } + for (i = 0; i < COUNT_2; i++) { + r = gen_rand64(ctx); + assert_u64_eq(r, array64_2[i], + "Mismatch at array64_2[%d]=%"PRIx64" gen=%"PRIx64, i, + array64_2[i], r); + } + fini_gen_rand(ctx); +} +TEST_END + +int +main(void) +{ + + return (test( + test_gen_rand_32, + test_by_array_32, + test_gen_rand_64, + test_by_array_64)); +} diff --git a/deps/jemalloc/test/bitmap.c b/deps/jemalloc/test/unit/bitmap.c similarity index 57% rename from deps/jemalloc/test/bitmap.c rename to deps/jemalloc/test/unit/bitmap.c index b2cb63004bc..8086b8885fc 100644 --- a/deps/jemalloc/test/bitmap.c +++ b/deps/jemalloc/test/unit/bitmap.c @@ -1,5 +1,4 @@ -#define JEMALLOC_MANGLE -#include "jemalloc_test.h" +#include "test/jemalloc_test.h" #if (LG_BITMAP_MAXBITS > 12) # define MAXBITS 4500 @@ -7,21 +6,21 @@ # define MAXBITS (1U << LG_BITMAP_MAXBITS) #endif -static void -test_bitmap_size(void) +TEST_BEGIN(test_bitmap_size) { size_t i, prev_size; prev_size = 0; for (i = 1; i <= MAXBITS; i++) { size_t size = bitmap_size(i); - assert(size >= prev_size); + assert_true(size >= prev_size, + "Bitmap size is smaller than expected"); prev_size = size; } } +TEST_END -static void -test_bitmap_init(void) +TEST_BEGIN(test_bitmap_init) { size_t i; @@ -34,16 +33,17 @@ test_bitmap_init(void) bitmap_info_ngroups(&binfo)); bitmap_init(bitmap, &binfo); - for (j = 0; j < i; j++) - assert(bitmap_get(bitmap, &binfo, j) == false); + for (j = 0; j < i; j++) { + assert_false(bitmap_get(bitmap, &binfo, j), + "Bit should be unset"); + } free(bitmap); - } } } +TEST_END -static void -test_bitmap_set(void) +TEST_BEGIN(test_bitmap_set) { size_t i; @@ -58,14 +58,15 @@ test_bitmap_set(void) for (j = 0; j < i; j++) bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); free(bitmap); } } } +TEST_END -static void -test_bitmap_unset(void) +TEST_BEGIN(test_bitmap_unset) { size_t i; @@ -80,19 +81,21 @@ test_bitmap_unset(void) for (j = 0; j < i; j++) bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); for (j = 0; j < i; j++) bitmap_unset(bitmap, &binfo, j); for (j = 0; j < i; j++) bitmap_set(bitmap, &binfo, j); - assert(bitmap_full(bitmap, &binfo)); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); free(bitmap); } } } +TEST_END -static void -test_bitmap_sfu(void) +TEST_BEGIN(test_bitmap_sfu) { size_t i; @@ -106,9 +109,13 @@ test_bitmap_sfu(void) bitmap_init(bitmap, &binfo); /* Iteratively set bits starting at the beginning. */ - for (j = 0; j < i; j++) - assert(bitmap_sfu(bitmap, &binfo) == j); - assert(bitmap_full(bitmap, &binfo)); + for (j = 0; j < i; j++) { + assert_zd_eq(bitmap_sfu(bitmap, &binfo), j, + "First unset bit should be just after " + "previous first unset bit"); + } + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); /* * Iteratively unset bits starting at the end, and @@ -116,10 +123,13 @@ test_bitmap_sfu(void) */ for (j = i - 1; j >= 0; j--) { bitmap_unset(bitmap, &binfo, j); - assert(bitmap_sfu(bitmap, &binfo) == j); + assert_zd_eq(bitmap_sfu(bitmap, &binfo), j, + "First unset bit should the bit previously " + "unset"); bitmap_unset(bitmap, &binfo, j); } - assert(bitmap_get(bitmap, &binfo, 0) == false); + assert_false(bitmap_get(bitmap, &binfo, 0), + "Bit should be unset"); /* * Iteratively set bits starting at the beginning, and @@ -127,27 +137,29 @@ test_bitmap_sfu(void) */ for (j = 1; j < i; j++) { bitmap_set(bitmap, &binfo, j - 1); - assert(bitmap_sfu(bitmap, &binfo) == j); + assert_zd_eq(bitmap_sfu(bitmap, &binfo), j, + "First unset bit should be just after the " + "bit previously set"); bitmap_unset(bitmap, &binfo, j); } - assert(bitmap_sfu(bitmap, &binfo) == i - 1); - assert(bitmap_full(bitmap, &binfo)); + assert_zd_eq(bitmap_sfu(bitmap, &binfo), i - 1, + "First unset bit should be the last bit"); + assert_true(bitmap_full(bitmap, &binfo), + "All bits should be set"); free(bitmap); } } } +TEST_END int main(void) { - malloc_printf("Test begin\n"); - - test_bitmap_size(); - test_bitmap_init(); - test_bitmap_set(); - test_bitmap_unset(); - test_bitmap_sfu(); - malloc_printf("Test end\n"); - return (0); + return (test( + test_bitmap_size, + test_bitmap_init, + test_bitmap_set, + test_bitmap_unset, + test_bitmap_sfu)); } diff --git a/deps/jemalloc/test/unit/ckh.c b/deps/jemalloc/test/unit/ckh.c new file mode 100644 index 00000000000..b214c279a5e --- /dev/null +++ b/deps/jemalloc/test/unit/ckh.c @@ -0,0 +1,206 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_new_delete) +{ + ckh_t ckh; + + assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp), + "Unexpected ckh_new() error"); + ckh_delete(&ckh); + + assert_false(ckh_new(&ckh, 3, ckh_pointer_hash, ckh_pointer_keycomp), + "Unexpected ckh_new() error"); + ckh_delete(&ckh); +} +TEST_END + +TEST_BEGIN(test_count_insert_search_remove) +{ + ckh_t ckh; + const char *strs[] = { + "a string", + "A string", + "a string.", + "A string." + }; + const char *missing = "A string not in the hash table."; + size_t i; + + assert_false(ckh_new(&ckh, 2, ckh_string_hash, ckh_string_keycomp), + "Unexpected ckh_new() error"); + assert_zu_eq(ckh_count(&ckh), 0, + "ckh_count() should return %zu, but it returned %zu", ZU(0), + ckh_count(&ckh)); + + /* Insert. */ + for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { + ckh_insert(&ckh, strs[i], strs[i]); + assert_zu_eq(ckh_count(&ckh), i+1, + "ckh_count() should return %zu, but it returned %zu", i+1, + ckh_count(&ckh)); + } + + /* Search. */ + for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { + union { + void *p; + const char *s; + } k, v; + void **kp, **vp; + const char *ks, *vs; + + kp = (i & 1) ? &k.p : NULL; + vp = (i & 2) ? &v.p : NULL; + k.p = NULL; + v.p = NULL; + assert_false(ckh_search(&ckh, strs[i], kp, vp), + "Unexpected ckh_search() error"); + + ks = (i & 1) ? strs[i] : (const char *)NULL; + vs = (i & 2) ? strs[i] : (const char *)NULL; + assert_ptr_eq((void *)ks, (void *)k.s, + "Key mismatch, i=%zu", i); + assert_ptr_eq((void *)vs, (void *)v.s, + "Value mismatch, i=%zu", i); + } + assert_true(ckh_search(&ckh, missing, NULL, NULL), + "Unexpected ckh_search() success"); + + /* Remove. */ + for (i = 0; i < sizeof(strs)/sizeof(const char *); i++) { + union { + void *p; + const char *s; + } k, v; + void **kp, **vp; + const char *ks, *vs; + + kp = (i & 1) ? &k.p : NULL; + vp = (i & 2) ? &v.p : NULL; + k.p = NULL; + v.p = NULL; + assert_false(ckh_remove(&ckh, strs[i], kp, vp), + "Unexpected ckh_remove() error"); + + ks = (i & 1) ? strs[i] : (const char *)NULL; + vs = (i & 2) ? strs[i] : (const char *)NULL; + assert_ptr_eq((void *)ks, (void *)k.s, + "Key mismatch, i=%zu", i); + assert_ptr_eq((void *)vs, (void *)v.s, + "Value mismatch, i=%zu", i); + assert_zu_eq(ckh_count(&ckh), + sizeof(strs)/sizeof(const char *) - i - 1, + "ckh_count() should return %zu, but it returned %zu", + sizeof(strs)/sizeof(const char *) - i - 1, + ckh_count(&ckh)); + } + + ckh_delete(&ckh); +} +TEST_END + +TEST_BEGIN(test_insert_iter_remove) +{ +#define NITEMS ZU(1000) + ckh_t ckh; + void **p[NITEMS]; + void *q, *r; + size_t i; + + assert_false(ckh_new(&ckh, 2, ckh_pointer_hash, ckh_pointer_keycomp), + "Unexpected ckh_new() error"); + + for (i = 0; i < NITEMS; i++) { + p[i] = mallocx(i+1, 0); + assert_ptr_not_null(p[i], "Unexpected mallocx() failure"); + } + + for (i = 0; i < NITEMS; i++) { + size_t j; + + for (j = i; j < NITEMS; j++) { + assert_false(ckh_insert(&ckh, p[j], p[j]), + "Unexpected ckh_insert() failure"); + assert_false(ckh_search(&ckh, p[j], &q, &r), + "Unexpected ckh_search() failure"); + assert_ptr_eq(p[j], q, "Key pointer mismatch"); + assert_ptr_eq(p[j], r, "Value pointer mismatch"); + } + + assert_zu_eq(ckh_count(&ckh), NITEMS, + "ckh_count() should return %zu, but it returned %zu", + NITEMS, ckh_count(&ckh)); + + for (j = i + 1; j < NITEMS; j++) { + assert_false(ckh_search(&ckh, p[j], NULL, NULL), + "Unexpected ckh_search() failure"); + assert_false(ckh_remove(&ckh, p[j], &q, &r), + "Unexpected ckh_remove() failure"); + assert_ptr_eq(p[j], q, "Key pointer mismatch"); + assert_ptr_eq(p[j], r, "Value pointer mismatch"); + assert_true(ckh_search(&ckh, p[j], NULL, NULL), + "Unexpected ckh_search() success"); + assert_true(ckh_remove(&ckh, p[j], &q, &r), + "Unexpected ckh_remove() success"); + } + + { + bool seen[NITEMS]; + size_t tabind; + + memset(seen, 0, sizeof(seen)); + + for (tabind = 0; ckh_iter(&ckh, &tabind, &q, &r) == + false;) { + size_t k; + + assert_ptr_eq(q, r, "Key and val not equal"); + + for (k = 0; k < NITEMS; k++) { + if (p[k] == q) { + assert_false(seen[k], + "Item %zu already seen", k); + seen[k] = true; + break; + } + } + } + + for (j = 0; j < i + 1; j++) + assert_true(seen[j], "Item %zu not seen", j); + for (; j < NITEMS; j++) + assert_false(seen[j], "Item %zu seen", j); + } + } + + for (i = 0; i < NITEMS; i++) { + assert_false(ckh_search(&ckh, p[i], NULL, NULL), + "Unexpected ckh_search() failure"); + assert_false(ckh_remove(&ckh, p[i], &q, &r), + "Unexpected ckh_remove() failure"); + assert_ptr_eq(p[i], q, "Key pointer mismatch"); + assert_ptr_eq(p[i], r, "Value pointer mismatch"); + assert_true(ckh_search(&ckh, p[i], NULL, NULL), + "Unexpected ckh_search() success"); + assert_true(ckh_remove(&ckh, p[i], &q, &r), + "Unexpected ckh_remove() success"); + dallocx(p[i], 0); + } + + assert_zu_eq(ckh_count(&ckh), 0, + "ckh_count() should return %zu, but it returned %zu", ZU(0), + ckh_count(&ckh)); + ckh_delete(&ckh); +#undef NITEMS +} +TEST_END + +int +main(void) +{ + + return (test( + test_new_delete, + test_count_insert_search_remove, + test_insert_iter_remove)); +} diff --git a/deps/jemalloc/test/unit/hash.c b/deps/jemalloc/test/unit/hash.c new file mode 100644 index 00000000000..abb394ac077 --- /dev/null +++ b/deps/jemalloc/test/unit/hash.c @@ -0,0 +1,171 @@ +/* + * This file is based on code that is part of SMHasher + * (https://code.google.com/p/smhasher/), and is subject to the MIT license + * (http://www.opensource.org/licenses/mit-license.php). Both email addresses + * associated with the source code's revision history belong to Austin Appleby, + * and the revision history ranges from 2010 to 2012. Therefore the copyright + * and license are here taken to be: + * + * Copyright (c) 2010-2012 Austin Appleby + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "test/jemalloc_test.h" + +typedef enum { + hash_variant_x86_32, + hash_variant_x86_128, + hash_variant_x64_128 +} hash_variant_t; + +static size_t +hash_variant_bits(hash_variant_t variant) +{ + + switch (variant) { + case hash_variant_x86_32: return (32); + case hash_variant_x86_128: return (128); + case hash_variant_x64_128: return (128); + default: not_reached(); + } +} + +static const char * +hash_variant_string(hash_variant_t variant) +{ + + switch (variant) { + case hash_variant_x86_32: return ("hash_x86_32"); + case hash_variant_x86_128: return ("hash_x86_128"); + case hash_variant_x64_128: return ("hash_x64_128"); + default: not_reached(); + } +} + +static void +hash_variant_verify(hash_variant_t variant) +{ + const size_t hashbytes = hash_variant_bits(variant) / 8; + uint8_t key[256]; + uint8_t hashes[hashbytes * 256]; + uint8_t final[hashbytes]; + unsigned i; + uint32_t computed, expected; + + memset(key, 0, sizeof(key)); + memset(hashes, 0, sizeof(hashes)); + memset(final, 0, sizeof(final)); + + /* + * Hash keys of the form {0}, {0,1}, {0,1,2}, ..., {0,1,...,255} as the + * seed. + */ + for (i = 0; i < 256; i++) { + key[i] = (uint8_t)i; + switch (variant) { + case hash_variant_x86_32: { + uint32_t out; + out = hash_x86_32(key, i, 256-i); + memcpy(&hashes[i*hashbytes], &out, hashbytes); + break; + } case hash_variant_x86_128: { + uint64_t out[2]; + hash_x86_128(key, i, 256-i, out); + memcpy(&hashes[i*hashbytes], out, hashbytes); + break; + } case hash_variant_x64_128: { + uint64_t out[2]; + hash_x64_128(key, i, 256-i, out); + memcpy(&hashes[i*hashbytes], out, hashbytes); + break; + } default: not_reached(); + } + } + + /* Hash the result array. */ + switch (variant) { + case hash_variant_x86_32: { + uint32_t out = hash_x86_32(hashes, hashbytes*256, 0); + memcpy(final, &out, sizeof(out)); + break; + } case hash_variant_x86_128: { + uint64_t out[2]; + hash_x86_128(hashes, hashbytes*256, 0, out); + memcpy(final, out, sizeof(out)); + break; + } case hash_variant_x64_128: { + uint64_t out[2]; + hash_x64_128(hashes, hashbytes*256, 0, out); + memcpy(final, out, sizeof(out)); + break; + } default: not_reached(); + } + + computed = (final[0] << 0) | (final[1] << 8) | (final[2] << 16) | + (final[3] << 24); + + switch (variant) { +#ifdef JEMALLOC_BIG_ENDIAN + case hash_variant_x86_32: expected = 0x6213303eU; break; + case hash_variant_x86_128: expected = 0x266820caU; break; + case hash_variant_x64_128: expected = 0xcc622b6fU; break; +#else + case hash_variant_x86_32: expected = 0xb0f57ee3U; break; + case hash_variant_x86_128: expected = 0xb3ece62aU; break; + case hash_variant_x64_128: expected = 0x6384ba69U; break; +#endif + default: not_reached(); + } + + assert_u32_eq(computed, expected, + "Hash mismatch for %s(): expected %#x but got %#x", + hash_variant_string(variant), expected, computed); +} + +TEST_BEGIN(test_hash_x86_32) +{ + + hash_variant_verify(hash_variant_x86_32); +} +TEST_END + +TEST_BEGIN(test_hash_x86_128) +{ + + hash_variant_verify(hash_variant_x86_128); +} +TEST_END + +TEST_BEGIN(test_hash_x64_128) +{ + + hash_variant_verify(hash_variant_x64_128); +} +TEST_END + +int +main(void) +{ + + return (test( + test_hash_x86_32, + test_hash_x86_128, + test_hash_x64_128)); +} diff --git a/deps/jemalloc/test/unit/junk.c b/deps/jemalloc/test/unit/junk.c new file mode 100644 index 00000000000..85bbf9e2bd3 --- /dev/null +++ b/deps/jemalloc/test/unit/junk.c @@ -0,0 +1,222 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_FILL +const char *malloc_conf = + "abort:false,junk:true,zero:false,redzone:true,quarantine:0"; +#endif + +static arena_dalloc_junk_small_t *arena_dalloc_junk_small_orig; +static arena_dalloc_junk_large_t *arena_dalloc_junk_large_orig; +static huge_dalloc_junk_t *huge_dalloc_junk_orig; +static void *most_recently_junked; + +static void +arena_dalloc_junk_small_intercept(void *ptr, arena_bin_info_t *bin_info) +{ + size_t i; + + arena_dalloc_junk_small_orig(ptr, bin_info); + for (i = 0; i < bin_info->reg_size; i++) { + assert_c_eq(((char *)ptr)[i], 0x5a, + "Missing junk fill for byte %zu/%zu of deallocated region", + i, bin_info->reg_size); + } + most_recently_junked = ptr; +} + +static void +arena_dalloc_junk_large_intercept(void *ptr, size_t usize) +{ + size_t i; + + arena_dalloc_junk_large_orig(ptr, usize); + for (i = 0; i < usize; i++) { + assert_c_eq(((char *)ptr)[i], 0x5a, + "Missing junk fill for byte %zu/%zu of deallocated region", + i, usize); + } + most_recently_junked = ptr; +} + +static void +huge_dalloc_junk_intercept(void *ptr, size_t usize) +{ + + huge_dalloc_junk_orig(ptr, usize); + /* + * The conditions under which junk filling actually occurs are nuanced + * enough that it doesn't make sense to duplicate the decision logic in + * test code, so don't actually check that the region is junk-filled. + */ + most_recently_junked = ptr; +} + +static void +test_junk(size_t sz_min, size_t sz_max) +{ + char *s; + size_t sz_prev, sz, i; + + arena_dalloc_junk_small_orig = arena_dalloc_junk_small; + arena_dalloc_junk_small = arena_dalloc_junk_small_intercept; + arena_dalloc_junk_large_orig = arena_dalloc_junk_large; + arena_dalloc_junk_large = arena_dalloc_junk_large_intercept; + huge_dalloc_junk_orig = huge_dalloc_junk; + huge_dalloc_junk = huge_dalloc_junk_intercept; + + sz_prev = 0; + s = (char *)mallocx(sz_min, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + + for (sz = sallocx(s, 0); sz <= sz_max; + sz_prev = sz, sz = sallocx(s, 0)) { + if (sz_prev > 0) { + assert_c_eq(s[0], 'a', + "Previously allocated byte %zu/%zu is corrupted", + ZU(0), sz_prev); + assert_c_eq(s[sz_prev-1], 'a', + "Previously allocated byte %zu/%zu is corrupted", + sz_prev-1, sz_prev); + } + + for (i = sz_prev; i < sz; i++) { + assert_c_eq(s[i], 0xa5, + "Newly allocated byte %zu/%zu isn't junk-filled", + i, sz); + s[i] = 'a'; + } + + if (xallocx(s, sz+1, 0, 0) == sz) { + void *junked = (void *)s; + + s = (char *)rallocx(s, sz+1, 0); + assert_ptr_not_null((void *)s, + "Unexpected rallocx() failure"); + if (!config_mremap || sz+1 <= arena_maxclass) { + assert_ptr_eq(most_recently_junked, junked, + "Expected region of size %zu to be " + "junk-filled", + sz); + } + } + } + + dallocx(s, 0); + assert_ptr_eq(most_recently_junked, (void *)s, + "Expected region of size %zu to be junk-filled", sz); + + arena_dalloc_junk_small = arena_dalloc_junk_small_orig; + arena_dalloc_junk_large = arena_dalloc_junk_large_orig; + huge_dalloc_junk = huge_dalloc_junk_orig; +} + +TEST_BEGIN(test_junk_small) +{ + + test_skip_if(!config_fill); + test_junk(1, SMALL_MAXCLASS-1); +} +TEST_END + +TEST_BEGIN(test_junk_large) +{ + + test_skip_if(!config_fill); + test_junk(SMALL_MAXCLASS+1, arena_maxclass); +} +TEST_END + +TEST_BEGIN(test_junk_huge) +{ + + test_skip_if(!config_fill); + test_junk(arena_maxclass+1, chunksize*2); +} +TEST_END + +arena_ralloc_junk_large_t *arena_ralloc_junk_large_orig; +static void *most_recently_trimmed; + +static void +arena_ralloc_junk_large_intercept(void *ptr, size_t old_usize, size_t usize) +{ + + arena_ralloc_junk_large_orig(ptr, old_usize, usize); + assert_zu_eq(old_usize, arena_maxclass, "Unexpected old_usize"); + assert_zu_eq(usize, arena_maxclass-PAGE, "Unexpected usize"); + most_recently_trimmed = ptr; +} + +TEST_BEGIN(test_junk_large_ralloc_shrink) +{ + void *p1, *p2; + + p1 = mallocx(arena_maxclass, 0); + assert_ptr_not_null(p1, "Unexpected mallocx() failure"); + + arena_ralloc_junk_large_orig = arena_ralloc_junk_large; + arena_ralloc_junk_large = arena_ralloc_junk_large_intercept; + + p2 = rallocx(p1, arena_maxclass-PAGE, 0); + assert_ptr_eq(p1, p2, "Unexpected move during shrink"); + + arena_ralloc_junk_large = arena_ralloc_junk_large_orig; + + assert_ptr_eq(most_recently_trimmed, p1, + "Expected trimmed portion of region to be junk-filled"); +} +TEST_END + +static bool detected_redzone_corruption; + +static void +arena_redzone_corruption_replacement(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + detected_redzone_corruption = true; +} + +TEST_BEGIN(test_junk_redzone) +{ + char *s; + arena_redzone_corruption_t *arena_redzone_corruption_orig; + + test_skip_if(!config_fill); + + arena_redzone_corruption_orig = arena_redzone_corruption; + arena_redzone_corruption = arena_redzone_corruption_replacement; + + /* Test underflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[-1] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + /* Test overflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[sallocx(s, 0)] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + arena_redzone_corruption = arena_redzone_corruption_orig; +} +TEST_END + +int +main(void) +{ + + return (test( + test_junk_small, + test_junk_large, + test_junk_huge, + test_junk_large_ralloc_shrink, + test_junk_redzone)); +} diff --git a/deps/jemalloc/test/unit/mallctl.c b/deps/jemalloc/test/unit/mallctl.c new file mode 100644 index 00000000000..31fb8105764 --- /dev/null +++ b/deps/jemalloc/test/unit/mallctl.c @@ -0,0 +1,415 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_mallctl_errors) +{ + uint64_t epoch; + size_t sz; + + assert_d_eq(mallctl("no_such_name", NULL, NULL, NULL, 0), ENOENT, + "mallctl() should return ENOENT for non-existent names"); + + assert_d_eq(mallctl("version", NULL, NULL, "0.0.0", strlen("0.0.0")), + EPERM, "mallctl() should return EPERM on attempt to write " + "read-only value"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)-1), + EINVAL, "mallctl() should return EINVAL for input size mismatch"); + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)+1), + EINVAL, "mallctl() should return EINVAL for input size mismatch"); + + sz = sizeof(epoch)-1; + assert_d_eq(mallctl("epoch", &epoch, &sz, NULL, 0), EINVAL, + "mallctl() should return EINVAL for output size mismatch"); + sz = sizeof(epoch)+1; + assert_d_eq(mallctl("epoch", &epoch, &sz, NULL, 0), EINVAL, + "mallctl() should return EINVAL for output size mismatch"); +} +TEST_END + +TEST_BEGIN(test_mallctlnametomib_errors) +{ + size_t mib[1]; + size_t miblen; + + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("no_such_name", mib, &miblen), ENOENT, + "mallctlnametomib() should return ENOENT for non-existent names"); +} +TEST_END + +TEST_BEGIN(test_mallctlbymib_errors) +{ + uint64_t epoch; + size_t sz; + size_t mib[1]; + size_t miblen; + + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("version", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, "0.0.0", + strlen("0.0.0")), EPERM, "mallctl() should return EPERM on " + "attempt to write read-only value"); + + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("epoch", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &epoch, + sizeof(epoch)-1), EINVAL, + "mallctlbymib() should return EINVAL for input size mismatch"); + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &epoch, + sizeof(epoch)+1), EINVAL, + "mallctlbymib() should return EINVAL for input size mismatch"); + + sz = sizeof(epoch)-1; + assert_d_eq(mallctlbymib(mib, miblen, &epoch, &sz, NULL, 0), EINVAL, + "mallctlbymib() should return EINVAL for output size mismatch"); + sz = sizeof(epoch)+1; + assert_d_eq(mallctlbymib(mib, miblen, &epoch, &sz, NULL, 0), EINVAL, + "mallctlbymib() should return EINVAL for output size mismatch"); +} +TEST_END + +TEST_BEGIN(test_mallctl_read_write) +{ + uint64_t old_epoch, new_epoch; + size_t sz = sizeof(old_epoch); + + /* Blind. */ + assert_d_eq(mallctl("epoch", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); + + /* Read. */ + assert_d_eq(mallctl("epoch", &old_epoch, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); + + /* Write. */ + assert_d_eq(mallctl("epoch", NULL, NULL, &new_epoch, sizeof(new_epoch)), + 0, "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); + + /* Read+write. */ + assert_d_eq(mallctl("epoch", &old_epoch, &sz, &new_epoch, + sizeof(new_epoch)), 0, "Unexpected mallctl() failure"); + assert_zu_eq(sz, sizeof(old_epoch), "Unexpected output size"); +} +TEST_END + +TEST_BEGIN(test_mallctlnametomib_short_mib) +{ + size_t mib[4]; + size_t miblen; + + miblen = 3; + mib[3] = 42; + assert_d_eq(mallctlnametomib("arenas.bin.0.nregs", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + assert_zu_eq(miblen, 3, "Unexpected mib output length"); + assert_zu_eq(mib[3], 42, + "mallctlnametomib() wrote past the end of the input mib"); +} +TEST_END + +TEST_BEGIN(test_mallctl_config) +{ + +#define TEST_MALLCTL_CONFIG(config) do { \ + bool oldval; \ + size_t sz = sizeof(oldval); \ + assert_d_eq(mallctl("config."#config, &oldval, &sz, NULL, 0), \ + 0, "Unexpected mallctl() failure"); \ + assert_b_eq(oldval, config_##config, "Incorrect config value"); \ + assert_zu_eq(sz, sizeof(oldval), "Unexpected output size"); \ +} while (0) + + TEST_MALLCTL_CONFIG(debug); + TEST_MALLCTL_CONFIG(dss); + TEST_MALLCTL_CONFIG(fill); + TEST_MALLCTL_CONFIG(lazy_lock); + TEST_MALLCTL_CONFIG(mremap); + TEST_MALLCTL_CONFIG(munmap); + TEST_MALLCTL_CONFIG(prof); + TEST_MALLCTL_CONFIG(prof_libgcc); + TEST_MALLCTL_CONFIG(prof_libunwind); + TEST_MALLCTL_CONFIG(stats); + TEST_MALLCTL_CONFIG(tcache); + TEST_MALLCTL_CONFIG(tls); + TEST_MALLCTL_CONFIG(utrace); + TEST_MALLCTL_CONFIG(valgrind); + TEST_MALLCTL_CONFIG(xmalloc); + +#undef TEST_MALLCTL_CONFIG +} +TEST_END + +TEST_BEGIN(test_mallctl_opt) +{ + bool config_always = true; + +#define TEST_MALLCTL_OPT(t, opt, config) do { \ + t oldval; \ + size_t sz = sizeof(oldval); \ + int expected = config_##config ? 0 : ENOENT; \ + int result = mallctl("opt."#opt, &oldval, &sz, NULL, 0); \ + assert_d_eq(result, expected, \ + "Unexpected mallctl() result for opt."#opt); \ + assert_zu_eq(sz, sizeof(oldval), "Unexpected output size"); \ +} while (0) + + TEST_MALLCTL_OPT(bool, abort, always); + TEST_MALLCTL_OPT(size_t, lg_chunk, always); + TEST_MALLCTL_OPT(const char *, dss, always); + TEST_MALLCTL_OPT(size_t, narenas, always); + TEST_MALLCTL_OPT(ssize_t, lg_dirty_mult, always); + TEST_MALLCTL_OPT(bool, stats_print, always); + TEST_MALLCTL_OPT(bool, junk, fill); + TEST_MALLCTL_OPT(size_t, quarantine, fill); + TEST_MALLCTL_OPT(bool, redzone, fill); + TEST_MALLCTL_OPT(bool, zero, fill); + TEST_MALLCTL_OPT(bool, utrace, utrace); + TEST_MALLCTL_OPT(bool, valgrind, valgrind); + TEST_MALLCTL_OPT(bool, xmalloc, xmalloc); + TEST_MALLCTL_OPT(bool, tcache, tcache); + TEST_MALLCTL_OPT(size_t, lg_tcache_max, tcache); + TEST_MALLCTL_OPT(bool, prof, prof); + TEST_MALLCTL_OPT(const char *, prof_prefix, prof); + TEST_MALLCTL_OPT(bool, prof_active, prof); + TEST_MALLCTL_OPT(ssize_t, lg_prof_sample, prof); + TEST_MALLCTL_OPT(bool, prof_accum, prof); + TEST_MALLCTL_OPT(ssize_t, lg_prof_interval, prof); + TEST_MALLCTL_OPT(bool, prof_gdump, prof); + TEST_MALLCTL_OPT(bool, prof_final, prof); + TEST_MALLCTL_OPT(bool, prof_leak, prof); + +#undef TEST_MALLCTL_OPT +} +TEST_END + +TEST_BEGIN(test_manpage_example) +{ + unsigned nbins, i; + size_t mib[4]; + size_t len, miblen; + + len = sizeof(nbins); + assert_d_eq(mallctl("arenas.nbins", &nbins, &len, NULL, 0), 0, + "Unexpected mallctl() failure"); + + miblen = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.size", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + for (i = 0; i < nbins; i++) { + size_t bin_size; + + mib[2] = i; + len = sizeof(bin_size); + assert_d_eq(mallctlbymib(mib, miblen, &bin_size, &len, NULL, 0), + 0, "Unexpected mallctlbymib() failure"); + /* Do something with bin_size... */ + } +} +TEST_END + +TEST_BEGIN(test_thread_arena) +{ + unsigned arena_old, arena_new, narenas; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect"); + arena_new = narenas - 1; + assert_d_eq(mallctl("thread.arena", &arena_old, &sz, &arena_new, + sizeof(unsigned)), 0, "Unexpected mallctl() failure"); + arena_new = 0; + assert_d_eq(mallctl("thread.arena", &arena_old, &sz, &arena_new, + sizeof(unsigned)), 0, "Unexpected mallctl() failure"); +} +TEST_END + +TEST_BEGIN(test_arena_i_purge) +{ + unsigned narenas; + size_t sz = sizeof(unsigned); + size_t mib[3]; + size_t miblen = 3; + + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); + + assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_d_eq(mallctlnametomib("arena.0.purge", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + mib[1] = narenas; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, NULL, 0), 0, + "Unexpected mallctlbymib() failure"); +} +TEST_END + +TEST_BEGIN(test_arena_i_dss) +{ + const char *dss_prec_old, *dss_prec_new; + size_t sz = sizeof(dss_prec_old); + + dss_prec_new = "primary"; + assert_d_eq(mallctl("arena.0.dss", &dss_prec_old, &sz, &dss_prec_new, + sizeof(dss_prec_new)), 0, "Unexpected mallctl() failure"); + assert_str_ne(dss_prec_old, "primary", + "Unexpected default for dss precedence"); + + assert_d_eq(mallctl("arena.0.dss", &dss_prec_new, &sz, &dss_prec_old, + sizeof(dss_prec_old)), 0, "Unexpected mallctl() failure"); +} +TEST_END + +TEST_BEGIN(test_arenas_purge) +{ + unsigned arena = 0; + + assert_d_eq(mallctl("arenas.purge", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + assert_d_eq(mallctl("arenas.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); +} +TEST_END + +TEST_BEGIN(test_arenas_initialized) +{ + unsigned narenas; + size_t sz = sizeof(narenas); + + assert_d_eq(mallctl("arenas.narenas", &narenas, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + { + bool initialized[narenas]; + + sz = narenas * sizeof(bool); + assert_d_eq(mallctl("arenas.initialized", initialized, &sz, + NULL, 0), 0, "Unexpected mallctl() failure"); + } +} +TEST_END + +TEST_BEGIN(test_arenas_constants) +{ + +#define TEST_ARENAS_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas."#name, &name, &sz, NULL, 0), 0, \ + "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); + TEST_ARENAS_CONSTANT(size_t, page, PAGE); + TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); + TEST_ARENAS_CONSTANT(size_t, nlruns, nlclasses); + +#undef TEST_ARENAS_CONSTANT +} +TEST_END + +TEST_BEGIN(test_arenas_bin_constants) +{ + +#define TEST_ARENAS_BIN_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas.bin.0."#name, &name, &sz, NULL, 0), \ + 0, "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_BIN_CONSTANT(size_t, size, arena_bin_info[0].reg_size); + TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, arena_bin_info[0].nregs); + TEST_ARENAS_BIN_CONSTANT(size_t, run_size, arena_bin_info[0].run_size); + +#undef TEST_ARENAS_BIN_CONSTANT +} +TEST_END + +TEST_BEGIN(test_arenas_lrun_constants) +{ + +#define TEST_ARENAS_LRUN_CONSTANT(t, name, expected) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("arenas.lrun.0."#name, &name, &sz, NULL, \ + 0), 0, "Unexpected mallctl() failure"); \ + assert_zu_eq(name, expected, "Incorrect "#name" size"); \ +} while (0) + + TEST_ARENAS_LRUN_CONSTANT(size_t, size, (1 << LG_PAGE)); + +#undef TEST_ARENAS_LRUN_CONSTANT +} +TEST_END + +TEST_BEGIN(test_arenas_extend) +{ + unsigned narenas_before, arena, narenas_after; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.narenas", &narenas_before, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_d_eq(mallctl("arenas.extend", &arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_d_eq(mallctl("arenas.narenas", &narenas_after, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + + assert_u_eq(narenas_before+1, narenas_after, + "Unexpected number of arenas before versus after extension"); + assert_u_eq(arena, narenas_after-1, "Unexpected arena index"); +} +TEST_END + +TEST_BEGIN(test_stats_arenas) +{ + +#define TEST_STATS_ARENAS(t, name) do { \ + t name; \ + size_t sz = sizeof(t); \ + assert_d_eq(mallctl("stats.arenas.0."#name, &name, &sz, NULL, \ + 0), 0, "Unexpected mallctl() failure"); \ +} while (0) + + TEST_STATS_ARENAS(const char *, dss); + TEST_STATS_ARENAS(unsigned, nthreads); + TEST_STATS_ARENAS(size_t, pactive); + TEST_STATS_ARENAS(size_t, pdirty); + +#undef TEST_STATS_ARENAS +} +TEST_END + +int +main(void) +{ + + return (test( + test_mallctl_errors, + test_mallctlnametomib_errors, + test_mallctlbymib_errors, + test_mallctl_read_write, + test_mallctlnametomib_short_mib, + test_mallctl_config, + test_mallctl_opt, + test_manpage_example, + test_thread_arena, + test_arena_i_purge, + test_arena_i_dss, + test_arenas_purge, + test_arenas_initialized, + test_arenas_constants, + test_arenas_bin_constants, + test_arenas_lrun_constants, + test_arenas_extend, + test_stats_arenas)); +} diff --git a/deps/jemalloc/test/unit/math.c b/deps/jemalloc/test/unit/math.c new file mode 100644 index 00000000000..a1b288ea19c --- /dev/null +++ b/deps/jemalloc/test/unit/math.c @@ -0,0 +1,388 @@ +#include "test/jemalloc_test.h" + +#define MAX_REL_ERR 1.0e-9 +#define MAX_ABS_ERR 1.0e-9 + +static bool +double_eq_rel(double a, double b, double max_rel_err, double max_abs_err) +{ + double rel_err; + + if (fabs(a - b) < max_abs_err) + return (true); + rel_err = (fabs(b) > fabs(a)) ? fabs((a-b)/b) : fabs((a-b)/a); + return (rel_err < max_rel_err); +} + +static uint64_t +factorial(unsigned x) +{ + uint64_t ret = 1; + unsigned i; + + for (i = 2; i <= x; i++) + ret *= (uint64_t)i; + + return (ret); +} + +TEST_BEGIN(test_ln_gamma_factorial) +{ + unsigned x; + + /* exp(ln_gamma(x)) == (x-1)! for integer x. */ + for (x = 1; x <= 21; x++) { + assert_true(double_eq_rel(exp(ln_gamma(x)), + (double)factorial(x-1), MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect factorial result for x=%u", x); + } +} +TEST_END + +/* Expected ln_gamma([0.0..100.0] increment=0.25). */ +static const double ln_gamma_misc_expected[] = { + INFINITY, + 1.28802252469807743, 0.57236494292470008, 0.20328095143129538, + 0.00000000000000000, -0.09827183642181320, -0.12078223763524518, + -0.08440112102048555, 0.00000000000000000, 0.12487171489239651, + 0.28468287047291918, 0.47521466691493719, 0.69314718055994529, + 0.93580193110872523, 1.20097360234707429, 1.48681557859341718, + 1.79175946922805496, 2.11445692745037128, 2.45373657084244234, + 2.80857141857573644, 3.17805383034794575, 3.56137591038669710, + 3.95781396761871651, 4.36671603662228680, 4.78749174278204581, + 5.21960398699022932, 5.66256205985714178, 6.11591589143154568, + 6.57925121201010121, 7.05218545073853953, 7.53436423675873268, + 8.02545839631598312, 8.52516136106541467, 9.03318691960512332, + 9.54926725730099690, 10.07315123968123949, 10.60460290274525086, + 11.14340011995171231, 11.68933342079726856, 12.24220494005076176, + 12.80182748008146909, 13.36802367147604720, 13.94062521940376342, + 14.51947222506051816, 15.10441257307551943, 15.69530137706046524, + 16.29200047656724237, 16.89437797963419285, 17.50230784587389010, + 18.11566950571089407, 18.73434751193644843, 19.35823122022435427, + 19.98721449566188468, 20.62119544270163018, 21.26007615624470048, + 21.90376249182879320, 22.55216385312342098, 23.20519299513386002, + 23.86276584168908954, 24.52480131594137802, 25.19122118273868338, + 25.86194990184851861, 26.53691449111561340, 27.21604439872720604, + 27.89927138384089389, 28.58652940490193828, 29.27775451504081516, + 29.97288476399884871, 30.67186010608067548, 31.37462231367769050, + 32.08111489594735843, 32.79128302226991565, 33.50507345013689076, + 34.22243445715505317, 34.94331577687681545, 35.66766853819134298, + 36.39544520803305261, 37.12659953718355865, 37.86108650896109395, + 38.59886229060776230, 39.33988418719949465, 40.08411059791735198, + 40.83150097453079752, 41.58201578195490100, 42.33561646075348506, + 43.09226539146988699, 43.85192586067515208, 44.61456202863158893, + 45.38013889847690052, 46.14862228684032885, 46.91997879580877395, + 47.69417578616628361, 48.47118135183522014, 49.25096429545256882, + 50.03349410501914463, 50.81874093156324790, 51.60667556776436982, + 52.39726942748592364, 53.19049452616926743, 53.98632346204390586, + 54.78472939811231157, 55.58568604486942633, 56.38916764371992940, + 57.19514895105859864, 58.00360522298051080, 58.81451220059079787, + 59.62784609588432261, 60.44358357816834371, 61.26170176100199427, + 62.08217818962842927, 62.90499082887649962, 63.73011805151035958, + 64.55753862700632340, 65.38723171073768015, 66.21917683354901385, + 67.05335389170279825, 67.88974313718154008, 68.72832516833013017, + 69.56908092082363737, 70.41199165894616385, 71.25703896716800045, + 72.10420474200799390, 72.95347118416940191, 73.80482079093779646, + 74.65823634883015814, 75.51370092648485866, 76.37119786778275454, + 77.23071078519033961, 78.09222355331530707, 78.95572030266725960, + 79.82118541361435859, 80.68860351052903468, 81.55795945611502873, + 82.42923834590904164, 83.30242550295004378, 84.17750647261028973, + 85.05446701758152983, 85.93329311301090456, 86.81397094178107920, + 87.69648688992882057, 88.58082754219766741, 89.46697967771913795, + 90.35493026581838194, 91.24466646193963015, 92.13617560368709292, + 93.02944520697742803, 93.92446296229978486, 94.82121673107967297, + 95.71969454214321615, 96.61988458827809723, 97.52177522288820910, + 98.42535495673848800, 99.33061245478741341, 100.23753653310367895, + 101.14611615586458981, 102.05634043243354370, 102.96819861451382394, + 103.88168009337621811, 104.79677439715833032, 105.71347118823287303, + 106.63176026064346047, 107.55163153760463501, 108.47307506906540198, + 109.39608102933323153, 110.32063971475740516, 111.24674154146920557, + 112.17437704317786995, 113.10353686902013237, 114.03421178146170689, + 114.96639265424990128, 115.90007047041454769, 116.83523632031698014, + 117.77188139974506953, 118.70999700805310795, 119.64957454634490830, + 120.59060551569974962, 121.53308151543865279, 122.47699424143097247, + 123.42233548443955726, 124.36909712850338394, 125.31727114935689826, + 126.26684961288492559, 127.21782467361175861, 128.17018857322420899, + 129.12393363912724453, 130.07905228303084755, 131.03553699956862033, + 131.99338036494577864, 132.95257503561629164, 133.91311374698926784, + 134.87498931216194364, 135.83819462068046846, 136.80272263732638294, + 137.76856640092901785, 138.73571902320256299, 139.70417368760718091, + 140.67392364823425055, 141.64496222871400732, 142.61728282114600574, + 143.59087888505104047, 144.56574394634486680, 145.54187159633210058, + 146.51925549072063859, 147.49788934865566148, 148.47776695177302031, + 149.45888214327129617, 150.44122882700193600, 151.42480096657754984, + 152.40959258449737490, 153.39559776128982094, 154.38281063467164245, + 155.37122539872302696, 156.36083630307879844, 157.35163765213474107, + 158.34362380426921391, 159.33678917107920370, 160.33112821663092973, + 161.32663545672428995, 162.32330545817117695, 163.32113283808695314, + 164.32011226319519892, 165.32023844914485267, 166.32150615984036790, + 167.32391020678358018, 168.32744544842768164, 169.33210678954270634, + 170.33788918059275375, 171.34478761712384198, 172.35279713916281707, + 173.36191283062726143, 174.37212981874515094, 175.38344327348534080, + 176.39584840699734514, 177.40934047306160437, 178.42391476654847793, + 179.43956662288721304, 180.45629141754378111, 181.47408456550741107, + 182.49294152078630304, 183.51285777591152737, 184.53382886144947861, + 185.55585034552262869, 186.57891783333786861, 187.60302696672312095, + 188.62817342367162610, 189.65435291789341932, 190.68156119837468054, + 191.70979404894376330, 192.73904728784492590, 193.76931676731820176, + 194.80059837318714244, 195.83288802445184729, 196.86618167288995096, + 197.90047530266301123, 198.93576492992946214, 199.97204660246373464, + 201.00931639928148797, 202.04757043027063901, 203.08680483582807597, + 204.12701578650228385, 205.16819948264117102, 206.21035215404597807, + 207.25347005962987623, 208.29754948708190909, 209.34258675253678916, + 210.38857820024875878, 211.43552020227099320, 212.48340915813977858, + 213.53224149456323744, 214.58201366511514152, 215.63272214993284592, + 216.68436345542014010, 217.73693411395422004, 218.79043068359703739, + 219.84484974781133815, 220.90018791517996988, 221.95644181913033322, + 223.01360811766215875, 224.07168349307951871, 225.13066465172661879, + 226.19054832372759734, 227.25133126272962159, 228.31301024565024704, + 229.37558207242807384, 230.43904356577689896, 231.50339157094342113, + 232.56862295546847008, 233.63473460895144740, 234.70172344281823484, + 235.76958639009222907, 236.83832040516844586, 237.90792246359117712, + 238.97838956183431947, 240.04971871708477238, 241.12190696702904802, + 242.19495136964280846, 243.26884900298270509, 244.34359696498191283, + 245.41919237324782443, 246.49563236486270057, 247.57291409618682110, + 248.65103474266476269, 249.72999149863338175, 250.80978157713354904, + 251.89040220972316320, 252.97185064629374551, 254.05412415488834199, + 255.13722002152300661, 256.22113555000953511, 257.30586806178126835, + 258.39141489572085675, 259.47777340799029844, 260.56494097186322279, + 261.65291497755913497, 262.74169283208021852, 263.83127195904967266, + 264.92164979855277807, 266.01282380697938379, 267.10479145686849733, + 268.19755023675537586, 269.29109765101975427, 270.38543121973674488, + 271.48054847852881721, 272.57644697842033565, 273.67312428569374561, + 274.77057798174683967, 275.86880566295326389, 276.96780494052313770, + 278.06757344036617496, 279.16810880295668085, 280.26940868320008349, + 281.37147075030043197, 282.47429268763045229, 283.57787219260217171, + 284.68220697654078322, 285.78729476455760050, 286.89313329542699194, + 287.99972032146268930, 289.10705360839756395, 290.21513093526289140, + 291.32395009427028754, 292.43350889069523646, 293.54380514276073200, + 294.65483668152336350, 295.76660135076059532, 296.87909700685889902, + 297.99232151870342022, 299.10627276756946458, 300.22094864701409733, + 301.33634706277030091, 302.45246593264130297, 303.56930318639643929, + 304.68685676566872189, 305.80512462385280514, 306.92410472600477078, + 308.04379504874236773, 309.16419358014690033, 310.28529831966631036, + 311.40710727801865687, 312.52961847709792664, 313.65282994987899201, + 314.77673974032603610, 315.90134590329950015, 317.02664650446632777, + 318.15263962020929966, 319.27932333753892635, 320.40669575400545455, + 321.53475497761127144, 322.66349912672620803, 323.79292633000159185, + 324.92303472628691452, 326.05382246454587403, 327.18528770377525916, + 328.31742861292224234, 329.45024337080525356, 330.58373016603343331, + 331.71788719692847280, 332.85271267144611329, 333.98820480709991898, + 335.12436183088397001, 336.26118197919845443, 337.39866349777429377, + 338.53680464159958774, 339.67560367484657036, 340.81505887079896411, + 341.95516851178109619, 343.09593088908627578, 344.23734430290727460, + 345.37940706226686416, 346.52211748494903532, 347.66547389743118401, + 348.80947463481720661, 349.95411804077025408, 351.09940246744753267, + 352.24532627543504759, 353.39188783368263103, 354.53908551944078908, + 355.68691771819692349, 356.83538282361303118, 357.98447923746385868, + 359.13420536957539753 +}; + +TEST_BEGIN(test_ln_gamma_misc) +{ + unsigned i; + + for (i = 1; i < sizeof(ln_gamma_misc_expected)/sizeof(double); i++) { + double x = (double)i * 0.25; + assert_true(double_eq_rel(ln_gamma(x), + ln_gamma_misc_expected[i], MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect ln_gamma result for i=%u", i); + } +} +TEST_END + +/* Expected pt_norm([0.01..0.99] increment=0.01). */ +static const double pt_norm_expected[] = { + -INFINITY, + -2.32634787404084076, -2.05374891063182252, -1.88079360815125085, + -1.75068607125216946, -1.64485362695147264, -1.55477359459685305, + -1.47579102817917063, -1.40507156030963221, -1.34075503369021654, + -1.28155156554460081, -1.22652812003661049, -1.17498679206608991, + -1.12639112903880045, -1.08031934081495606, -1.03643338949378938, + -0.99445788320975281, -0.95416525314619416, -0.91536508784281390, + -0.87789629505122846, -0.84162123357291418, -0.80642124701824025, + -0.77219321418868492, -0.73884684918521371, -0.70630256284008752, + -0.67448975019608171, -0.64334540539291685, -0.61281299101662701, + -0.58284150727121620, -0.55338471955567281, -0.52440051270804067, + -0.49585034734745320, -0.46769879911450812, -0.43991316567323380, + -0.41246312944140462, -0.38532046640756751, -0.35845879325119373, + -0.33185334643681652, -0.30548078809939738, -0.27931903444745404, + -0.25334710313579978, -0.22754497664114931, -0.20189347914185077, + -0.17637416478086135, -0.15096921549677725, -0.12566134685507399, + -0.10043372051146975, -0.07526986209982976, -0.05015358346473352, + -0.02506890825871106, 0.00000000000000000, 0.02506890825871106, + 0.05015358346473366, 0.07526986209982990, 0.10043372051146990, + 0.12566134685507413, 0.15096921549677739, 0.17637416478086146, + 0.20189347914185105, 0.22754497664114931, 0.25334710313579978, + 0.27931903444745404, 0.30548078809939738, 0.33185334643681652, + 0.35845879325119373, 0.38532046640756762, 0.41246312944140484, + 0.43991316567323391, 0.46769879911450835, 0.49585034734745348, + 0.52440051270804111, 0.55338471955567303, 0.58284150727121620, + 0.61281299101662701, 0.64334540539291685, 0.67448975019608171, + 0.70630256284008752, 0.73884684918521371, 0.77219321418868492, + 0.80642124701824036, 0.84162123357291441, 0.87789629505122879, + 0.91536508784281423, 0.95416525314619460, 0.99445788320975348, + 1.03643338949378938, 1.08031934081495606, 1.12639112903880045, + 1.17498679206608991, 1.22652812003661049, 1.28155156554460081, + 1.34075503369021654, 1.40507156030963265, 1.47579102817917085, + 1.55477359459685394, 1.64485362695147308, 1.75068607125217102, + 1.88079360815125041, 2.05374891063182208, 2.32634787404084076 +}; + +TEST_BEGIN(test_pt_norm) +{ + unsigned i; + + for (i = 1; i < sizeof(pt_norm_expected)/sizeof(double); i++) { + double p = (double)i * 0.01; + assert_true(double_eq_rel(pt_norm(p), pt_norm_expected[i], + MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect pt_norm result for i=%u", i); + } +} +TEST_END + +/* + * Expected pt_chi2(p=[0.01..0.99] increment=0.07, + * df={0.1, 1.1, 10.1, 100.1, 1000.1}). + */ +static const double pt_chi2_df[] = {0.1, 1.1, 10.1, 100.1, 1000.1}; +static const double pt_chi2_expected[] = { + 1.168926411457320e-40, 1.347680397072034e-22, 3.886980416666260e-17, + 8.245951724356564e-14, 2.068936347497604e-11, 1.562561743309233e-09, + 5.459543043426564e-08, 1.114775688149252e-06, 1.532101202364371e-05, + 1.553884683726585e-04, 1.239396954915939e-03, 8.153872320255721e-03, + 4.631183739647523e-02, 2.473187311701327e-01, 2.175254800183617e+00, + + 0.0003729887888876379, 0.0164409238228929513, 0.0521523015190650113, + 0.1064701372271216612, 0.1800913735793082115, 0.2748704281195626931, + 0.3939246282787986497, 0.5420727552260817816, 0.7267265822221973259, + 0.9596554296000253670, 1.2607440376386165326, 1.6671185084541604304, + 2.2604828984738705167, 3.2868613342148607082, 6.9298574921692139839, + + 2.606673548632508, 4.602913725294877, 5.646152813924212, + 6.488971315540869, 7.249823275816285, 7.977314231410841, + 8.700354939944047, 9.441728024225892, 10.224338321374127, + 11.076435368801061, 12.039320937038386, 13.183878752697167, + 14.657791935084575, 16.885728216339373, 23.361991680031817, + + 70.14844087392152, 80.92379498849355, 85.53325420085891, + 88.94433120715347, 91.83732712857017, 94.46719943606301, + 96.96896479994635, 99.43412843510363, 101.94074719829733, + 104.57228644307247, 107.43900093448734, 110.71844673417287, + 114.76616819871325, 120.57422505959563, 135.92318818757556, + + 899.0072447849649, 937.9271278858220, 953.8117189560207, + 965.3079371501154, 974.8974061207954, 983.4936235182347, + 991.5691170518946, 999.4334123954690, 1007.3391826856553, + 1015.5445154999951, 1024.3777075619569, 1034.3538789836223, + 1046.4872561869577, 1063.5717461999654, 1107.0741966053859 +}; + +TEST_BEGIN(test_pt_chi2) +{ + unsigned i, j; + unsigned e = 0; + + for (i = 0; i < sizeof(pt_chi2_df)/sizeof(double); i++) { + double df = pt_chi2_df[i]; + double ln_gamma_df = ln_gamma(df * 0.5); + for (j = 1; j < 100; j += 7) { + double p = (double)j * 0.01; + assert_true(double_eq_rel(pt_chi2(p, df, ln_gamma_df), + pt_chi2_expected[e], MAX_REL_ERR, MAX_ABS_ERR), + "Incorrect pt_chi2 result for i=%u, j=%u", i, j); + e++; + } + } +} +TEST_END + +/* + * Expected pt_gamma(p=[0.1..0.99] increment=0.07, + * shape=[0.5..3.0] increment=0.5). + */ +static const double pt_gamma_shape[] = {0.5, 1.0, 1.5, 2.0, 2.5, 3.0}; +static const double pt_gamma_expected[] = { + 7.854392895485103e-05, 5.043466107888016e-03, 1.788288957794883e-02, + 3.900956150232906e-02, 6.913847560638034e-02, 1.093710833465766e-01, + 1.613412523825817e-01, 2.274682115597864e-01, 3.114117323127083e-01, + 4.189466220207417e-01, 5.598106789059246e-01, 7.521856146202706e-01, + 1.036125427911119e+00, 1.532450860038180e+00, 3.317448300510606e+00, + + 0.01005033585350144, 0.08338160893905107, 0.16251892949777497, + 0.24846135929849966, 0.34249030894677596, 0.44628710262841947, + 0.56211891815354142, 0.69314718055994529, 0.84397007029452920, + 1.02165124753198167, 1.23787435600161766, 1.51412773262977574, + 1.89711998488588196, 2.52572864430825783, 4.60517018598809091, + + 0.05741590094955853, 0.24747378084860744, 0.39888572212236084, + 0.54394139997444901, 0.69048812513915159, 0.84311389861296104, + 1.00580622221479898, 1.18298694218766931, 1.38038096305861213, + 1.60627736383027453, 1.87396970522337947, 2.20749220408081070, + 2.65852391865854942, 3.37934630984842244, 5.67243336507218476, + + 0.1485547402532659, 0.4657458011640391, 0.6832386130709406, + 0.8794297834672100, 1.0700752852474524, 1.2629614217350744, + 1.4638400448580779, 1.6783469900166610, 1.9132338090606940, + 2.1778589228618777, 2.4868823970010991, 2.8664695666264195, + 3.3724415436062114, 4.1682658512758071, 6.6383520679938108, + + 0.2771490383641385, 0.7195001279643727, 0.9969081732265243, + 1.2383497880608061, 1.4675206597269927, 1.6953064251816552, + 1.9291243435606809, 2.1757300955477641, 2.4428032131216391, + 2.7406534569230616, 3.0851445039665513, 3.5043101122033367, + 4.0575997065264637, 4.9182956424675286, 7.5431362346944937, + + 0.4360451650782932, 0.9983600902486267, 1.3306365880734528, + 1.6129750834753802, 1.8767241606994294, 2.1357032436097660, + 2.3988853336865565, 2.6740603137235603, 2.9697561737517959, + 3.2971457713883265, 3.6731795898504660, 4.1275751617770631, + 4.7230515633946677, 5.6417477865306020, 8.4059469148854635 +}; + +TEST_BEGIN(test_pt_gamma_shape) +{ + unsigned i, j; + unsigned e = 0; + + for (i = 0; i < sizeof(pt_gamma_shape)/sizeof(double); i++) { + double shape = pt_gamma_shape[i]; + double ln_gamma_shape = ln_gamma(shape); + for (j = 1; j < 100; j += 7) { + double p = (double)j * 0.01; + assert_true(double_eq_rel(pt_gamma(p, shape, 1.0, + ln_gamma_shape), pt_gamma_expected[e], MAX_REL_ERR, + MAX_ABS_ERR), + "Incorrect pt_gamma result for i=%u, j=%u", i, j); + e++; + } + } +} +TEST_END + +TEST_BEGIN(test_pt_gamma_scale) +{ + double shape = 1.0; + double ln_gamma_shape = ln_gamma(shape); + + assert_true(double_eq_rel( + pt_gamma(0.5, shape, 1.0, ln_gamma_shape) * 10.0, + pt_gamma(0.5, shape, 10.0, ln_gamma_shape), MAX_REL_ERR, + MAX_ABS_ERR), + "Scale should be trivially equivalent to external multiplication"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_ln_gamma_factorial, + test_ln_gamma_misc, + test_pt_norm, + test_pt_chi2, + test_pt_gamma_shape, + test_pt_gamma_scale)); +} diff --git a/deps/jemalloc/test/unit/mq.c b/deps/jemalloc/test/unit/mq.c new file mode 100644 index 00000000000..f57e96af1cb --- /dev/null +++ b/deps/jemalloc/test/unit/mq.c @@ -0,0 +1,92 @@ +#include "test/jemalloc_test.h" + +#define NSENDERS 3 +#define NMSGS 100000 + +typedef struct mq_msg_s mq_msg_t; +struct mq_msg_s { + mq_msg(mq_msg_t) link; +}; +mq_gen(static, mq_, mq_t, mq_msg_t, link) + +TEST_BEGIN(test_mq_basic) +{ + mq_t mq; + mq_msg_t msg; + + assert_false(mq_init(&mq), "Unexpected mq_init() failure"); + assert_u_eq(mq_count(&mq), 0, "mq should be empty"); + assert_ptr_null(mq_tryget(&mq), + "mq_tryget() should fail when the queue is empty"); + + mq_put(&mq, &msg); + assert_u_eq(mq_count(&mq), 1, "mq should contain one message"); + assert_ptr_eq(mq_tryget(&mq), &msg, "mq_tryget() should return msg"); + + mq_put(&mq, &msg); + assert_ptr_eq(mq_get(&mq), &msg, "mq_get() should return msg"); + + mq_fini(&mq); +} +TEST_END + +static void * +thd_receiver_start(void *arg) +{ + mq_t *mq = (mq_t *)arg; + unsigned i; + + for (i = 0; i < (NSENDERS * NMSGS); i++) { + mq_msg_t *msg = mq_get(mq); + assert_ptr_not_null(msg, "mq_get() should never return NULL"); + dallocx(msg, 0); + } + return (NULL); +} + +static void * +thd_sender_start(void *arg) +{ + mq_t *mq = (mq_t *)arg; + unsigned i; + + for (i = 0; i < NMSGS; i++) { + mq_msg_t *msg; + void *p; + p = mallocx(sizeof(mq_msg_t), 0); + assert_ptr_not_null(p, "Unexpected allocm() failure"); + msg = (mq_msg_t *)p; + mq_put(mq, msg); + } + return (NULL); +} + +TEST_BEGIN(test_mq_threaded) +{ + mq_t mq; + thd_t receiver; + thd_t senders[NSENDERS]; + unsigned i; + + assert_false(mq_init(&mq), "Unexpected mq_init() failure"); + + thd_create(&receiver, thd_receiver_start, (void *)&mq); + for (i = 0; i < NSENDERS; i++) + thd_create(&senders[i], thd_sender_start, (void *)&mq); + + thd_join(receiver, NULL); + for (i = 0; i < NSENDERS; i++) + thd_join(senders[i], NULL); + + mq_fini(&mq); +} +TEST_END + +int +main(void) +{ + return (test( + test_mq_basic, + test_mq_threaded)); +} + diff --git a/deps/jemalloc/test/unit/mtx.c b/deps/jemalloc/test/unit/mtx.c new file mode 100644 index 00000000000..96ff69486ea --- /dev/null +++ b/deps/jemalloc/test/unit/mtx.c @@ -0,0 +1,60 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 2 +#define NINCRS 2000000 + +TEST_BEGIN(test_mtx_basic) +{ + mtx_t mtx; + + assert_false(mtx_init(&mtx), "Unexpected mtx_init() failure"); + mtx_lock(&mtx); + mtx_unlock(&mtx); + mtx_fini(&mtx); +} +TEST_END + +typedef struct { + mtx_t mtx; + unsigned x; +} thd_start_arg_t; + +static void * +thd_start(void *varg) +{ + thd_start_arg_t *arg = (thd_start_arg_t *)varg; + unsigned i; + + for (i = 0; i < NINCRS; i++) { + mtx_lock(&arg->mtx); + arg->x++; + mtx_unlock(&arg->mtx); + } + return (NULL); +} + +TEST_BEGIN(test_mtx_race) +{ + thd_start_arg_t arg; + thd_t thds[NTHREADS]; + unsigned i; + + assert_false(mtx_init(&arg.mtx), "Unexpected mtx_init() failure"); + arg.x = 0; + for (i = 0; i < NTHREADS; i++) + thd_create(&thds[i], thd_start, (void *)&arg); + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); + assert_u_eq(arg.x, NTHREADS * NINCRS, + "Race-related counter corruption"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_mtx_basic, + test_mtx_race)); +} diff --git a/deps/jemalloc/test/unit/prof_accum.c b/deps/jemalloc/test/unit/prof_accum.c new file mode 100644 index 00000000000..050a8a7ee5d --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum.c @@ -0,0 +1,86 @@ +#include "prof_accum.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0"; +#endif + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +static void * +alloc_from_permuted_backtrace(unsigned thd_ind, unsigned iteration) +{ + + return (alloc_0(thd_ind*NALLOCS_PER_THREAD + iteration)); +} + +static void * +thd_start(void *varg) +{ + unsigned thd_ind = *(unsigned *)varg; + size_t bt_count_prev, bt_count; + unsigned i_prev, i; + + i_prev = 0; + bt_count_prev = 0; + for (i = 0; i < NALLOCS_PER_THREAD; i++) { + void *p = alloc_from_permuted_backtrace(thd_ind, i); + dallocx(p, 0); + if (i % DUMP_INTERVAL == 0) { + assert_d_eq(mallctl("prof.dump", NULL, NULL, NULL, 0), + 0, "Unexpected error while dumping heap profile"); + } + + if (i % BT_COUNT_CHECK_INTERVAL == 0 || + i+1 == NALLOCS_PER_THREAD) { + bt_count = prof_bt_count(); + assert_zu_le(bt_count_prev+(i-i_prev), bt_count, + "Expected larger backtrace count increase"); + i_prev = i; + bt_count_prev = bt_count; + } + } + + return (NULL); +} + +TEST_BEGIN(test_idump) +{ + bool active; + thd_t thds[NTHREADS]; + unsigned thd_args[NTHREADS]; + unsigned i; + + test_skip_if(!config_prof); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + prof_dump_open = prof_dump_open_intercept; + + for (i = 0; i < NTHREADS; i++) { + thd_args[i] = i; + thd_create(&thds[i], thd_start, (void *)&thd_args[i]); + } + for (i = 0; i < NTHREADS; i++) + thd_join(thds[i], NULL); +} +TEST_END + +int +main(void) +{ + + return (test( + test_idump)); +} diff --git a/deps/jemalloc/test/unit/prof_accum.h b/deps/jemalloc/test/unit/prof_accum.h new file mode 100644 index 00000000000..109d86b598d --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum.h @@ -0,0 +1,35 @@ +#include "test/jemalloc_test.h" + +#define NTHREADS 4 +#define NALLOCS_PER_THREAD 50 +#define DUMP_INTERVAL 1 +#define BT_COUNT_CHECK_INTERVAL 5 + +#define alloc_n_proto(n) \ +void *alloc_##n(unsigned bits); +alloc_n_proto(0) +alloc_n_proto(1) + +#define alloc_n_gen(n) \ +void * \ +alloc_##n(unsigned bits) \ +{ \ + void *p; \ + \ + if (bits == 0) \ + p = mallocx(1, 0); \ + else { \ + switch (bits & 0x1U) { \ + case 0: \ + p = (alloc_0(bits >> 1)); \ + break; \ + case 1: \ + p = (alloc_1(bits >> 1)); \ + break; \ + default: not_reached(); \ + } \ + } \ + /* Intentionally sabotage tail call optimization. */ \ + assert_ptr_not_null(p, "Unexpected mallocx() failure"); \ + return (p); \ +} diff --git a/deps/jemalloc/test/unit/prof_accum_a.c b/deps/jemalloc/test/unit/prof_accum_a.c new file mode 100644 index 00000000000..42ad521d8df --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum_a.c @@ -0,0 +1,3 @@ +#include "prof_accum.h" + +alloc_n_gen(0) diff --git a/deps/jemalloc/test/unit/prof_accum_b.c b/deps/jemalloc/test/unit/prof_accum_b.c new file mode 100644 index 00000000000..60d9dab6a8e --- /dev/null +++ b/deps/jemalloc/test/unit/prof_accum_b.c @@ -0,0 +1,3 @@ +#include "prof_accum.h" + +alloc_n_gen(1) diff --git a/deps/jemalloc/test/unit/prof_gdump.c b/deps/jemalloc/test/unit/prof_gdump.c new file mode 100644 index 00000000000..a00b1054f1d --- /dev/null +++ b/deps/jemalloc/test/unit/prof_gdump.c @@ -0,0 +1,56 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = "prof:true,prof_active:false,prof_gdump:true"; +#endif + +static bool did_prof_dump_open; + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + did_prof_dump_open = true; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +TEST_BEGIN(test_gdump) +{ + bool active; + void *p, *q; + + test_skip_if(!config_prof); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + prof_dump_open = prof_dump_open_intercept; + + did_prof_dump_open = false; + p = mallocx(chunksize, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + assert_true(did_prof_dump_open, "Expected a profile dump"); + + did_prof_dump_open = false; + q = mallocx(chunksize, 0); + assert_ptr_not_null(q, "Unexpected mallocx() failure"); + assert_true(did_prof_dump_open, "Expected a profile dump"); + + dallocx(p, 0); + dallocx(q, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_gdump)); +} diff --git a/deps/jemalloc/test/unit/prof_idump.c b/deps/jemalloc/test/unit/prof_idump.c new file mode 100644 index 00000000000..bdea53ecdd9 --- /dev/null +++ b/deps/jemalloc/test/unit/prof_idump.c @@ -0,0 +1,51 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_PROF +const char *malloc_conf = + "prof:true,prof_accum:true,prof_active:false,lg_prof_sample:0," + "lg_prof_interval:0"; +#endif + +static bool did_prof_dump_open; + +static int +prof_dump_open_intercept(bool propagate_err, const char *filename) +{ + int fd; + + did_prof_dump_open = true; + + fd = open("/dev/null", O_WRONLY); + assert_d_ne(fd, -1, "Unexpected open() failure"); + + return (fd); +} + +TEST_BEGIN(test_idump) +{ + bool active; + void *p; + + test_skip_if(!config_prof); + + active = true; + assert_d_eq(mallctl("prof.active", NULL, NULL, &active, sizeof(active)), + 0, "Unexpected mallctl failure while activating profiling"); + + prof_dump_open = prof_dump_open_intercept; + + did_prof_dump_open = false; + p = mallocx(1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + dallocx(p, 0); + assert_true(did_prof_dump_open, "Expected a profile dump"); +} +TEST_END + +int +main(void) +{ + + return (test( + test_idump)); +} diff --git a/deps/jemalloc/test/unit/ql.c b/deps/jemalloc/test/unit/ql.c new file mode 100644 index 00000000000..05fad450fc0 --- /dev/null +++ b/deps/jemalloc/test/unit/ql.c @@ -0,0 +1,209 @@ +#include "test/jemalloc_test.h" + +/* Number of ring entries, in [2..26]. */ +#define NENTRIES 9 + +typedef struct list_s list_t; +typedef ql_head(list_t) list_head_t; + +struct list_s { + ql_elm(list_t) link; + char id; +}; + +static void +test_empty_list(list_head_t *head) +{ + list_t *t; + unsigned i; + + assert_ptr_null(ql_first(head), "Unexpected element for empty list"); + assert_ptr_null(ql_last(head, link), + "Unexpected element for empty list"); + + i = 0; + ql_foreach(t, head, link) { + i++; + } + assert_u_eq(i, 0, "Unexpected element for empty list"); + + i = 0; + ql_reverse_foreach(t, head, link) { + i++; + } + assert_u_eq(i, 0, "Unexpected element for empty list"); +} + +TEST_BEGIN(test_ql_empty) +{ + list_head_t head; + + ql_new(&head); + test_empty_list(&head); +} +TEST_END + +static void +init_entries(list_t *entries, unsigned nentries) +{ + unsigned i; + + for (i = 0; i < nentries; i++) { + entries[i].id = 'a' + i; + ql_elm_new(&entries[i], link); + } +} + +static void +test_entries_list(list_head_t *head, list_t *entries, unsigned nentries) +{ + list_t *t; + unsigned i; + + assert_c_eq(ql_first(head)->id, entries[0].id, "Element id mismatch"); + assert_c_eq(ql_last(head, link)->id, entries[nentries-1].id, + "Element id mismatch"); + + i = 0; + ql_foreach(t, head, link) { + assert_c_eq(t->id, entries[i].id, "Element id mismatch"); + i++; + } + + i = 0; + ql_reverse_foreach(t, head, link) { + assert_c_eq(t->id, entries[nentries-i-1].id, + "Element id mismatch"); + i++; + } + + for (i = 0; i < nentries-1; i++) { + t = ql_next(head, &entries[i], link); + assert_c_eq(t->id, entries[i+1].id, "Element id mismatch"); + } + assert_ptr_null(ql_next(head, &entries[nentries-1], link), + "Unexpected element"); + + assert_ptr_null(ql_prev(head, &entries[0], link), "Unexpected element"); + for (i = 1; i < nentries; i++) { + t = ql_prev(head, &entries[i], link); + assert_c_eq(t->id, entries[i-1].id, "Element id mismatch"); + } +} + +TEST_BEGIN(test_ql_tail_insert) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_tail_insert(&head, &entries[i], link); + + test_entries_list(&head, entries, NENTRIES); +} +TEST_END + +TEST_BEGIN(test_ql_tail_remove) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_tail_insert(&head, &entries[i], link); + + for (i = 0; i < NENTRIES; i++) { + test_entries_list(&head, entries, NENTRIES-i); + ql_tail_remove(&head, list_t, link); + } + test_empty_list(&head); +} +TEST_END + +TEST_BEGIN(test_ql_head_insert) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_head_insert(&head, &entries[NENTRIES-i-1], link); + + test_entries_list(&head, entries, NENTRIES); +} +TEST_END + +TEST_BEGIN(test_ql_head_remove) +{ + list_head_t head; + list_t entries[NENTRIES]; + unsigned i; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + for (i = 0; i < NENTRIES; i++) + ql_head_insert(&head, &entries[NENTRIES-i-1], link); + + for (i = 0; i < NENTRIES; i++) { + test_entries_list(&head, &entries[i], NENTRIES-i); + ql_head_remove(&head, list_t, link); + } + test_empty_list(&head); +} +TEST_END + +TEST_BEGIN(test_ql_insert) +{ + list_head_t head; + list_t entries[8]; + list_t *a, *b, *c, *d, *e, *f, *g, *h; + + ql_new(&head); + init_entries(entries, sizeof(entries)/sizeof(list_t)); + a = &entries[0]; + b = &entries[1]; + c = &entries[2]; + d = &entries[3]; + e = &entries[4]; + f = &entries[5]; + g = &entries[6]; + h = &entries[7]; + + /* + * ql_remove(), ql_before_insert(), and ql_after_insert() are used + * internally by other macros that are already tested, so there's no + * need to test them completely. However, insertion/deletion from the + * middle of lists is not otherwise tested; do so here. + */ + ql_tail_insert(&head, f, link); + ql_before_insert(&head, f, b, link); + ql_before_insert(&head, f, c, link); + ql_after_insert(f, h, link); + ql_after_insert(f, g, link); + ql_before_insert(&head, b, a, link); + ql_after_insert(c, d, link); + ql_before_insert(&head, f, e, link); + + test_entries_list(&head, entries, sizeof(entries)/sizeof(list_t)); +} +TEST_END + +int +main(void) +{ + + return (test( + test_ql_empty, + test_ql_tail_insert, + test_ql_tail_remove, + test_ql_head_insert, + test_ql_head_remove, + test_ql_insert)); +} diff --git a/deps/jemalloc/test/unit/qr.c b/deps/jemalloc/test/unit/qr.c new file mode 100644 index 00000000000..a2a2d902b58 --- /dev/null +++ b/deps/jemalloc/test/unit/qr.c @@ -0,0 +1,248 @@ +#include "test/jemalloc_test.h" + +/* Number of ring entries, in [2..26]. */ +#define NENTRIES 9 +/* Split index, in [1..NENTRIES). */ +#define SPLIT_INDEX 5 + +typedef struct ring_s ring_t; + +struct ring_s { + qr(ring_t) link; + char id; +}; + +static void +init_entries(ring_t *entries) +{ + unsigned i; + + for (i = 0; i < NENTRIES; i++) { + qr_new(&entries[i], link); + entries[i].id = 'a' + i; + } +} + +static void +test_independent_entries(ring_t *entries) +{ + ring_t *t; + unsigned i, j; + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + j++; + } + assert_u_eq(j, 1, + "Iteration over single-element ring should visit precisely " + "one element"); + } + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + j++; + } + assert_u_eq(j, 1, + "Iteration over single-element ring should visit precisely " + "one element"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_next(&entries[i], link); + assert_ptr_eq(t, &entries[i], + "Next element in single-element ring should be same as " + "current element"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_prev(&entries[i], link); + assert_ptr_eq(t, &entries[i], + "Previous element in single-element ring should be same as " + "current element"); + } +} + +TEST_BEGIN(test_qr_one) +{ + ring_t entries[NENTRIES]; + + init_entries(entries); + test_independent_entries(entries); +} +TEST_END + +static void +test_entries_ring(ring_t *entries) +{ + ring_t *t; + unsigned i, j; + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(i+j) % NENTRIES].id, + "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(NENTRIES+i-j-1) % + NENTRIES].id, "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + t = qr_next(&entries[i], link); + assert_c_eq(t->id, entries[(i+1) % NENTRIES].id, + "Element id mismatch"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_prev(&entries[i], link); + assert_c_eq(t->id, entries[(NENTRIES+i-1) % NENTRIES].id, + "Element id mismatch"); + } +} + +TEST_BEGIN(test_qr_after_insert) +{ + ring_t entries[NENTRIES]; + unsigned i; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_after_insert(&entries[i - 1], &entries[i], link); + test_entries_ring(entries); +} +TEST_END + +TEST_BEGIN(test_qr_remove) +{ + ring_t entries[NENTRIES]; + ring_t *t; + unsigned i, j; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_after_insert(&entries[i - 1], &entries[i], link); + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[i+j].id, + "Element id mismatch"); + j++; + } + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[NENTRIES - 1 - j].id, + "Element id mismatch"); + j++; + } + qr_remove(&entries[i], link); + } + test_independent_entries(entries); +} +TEST_END + +TEST_BEGIN(test_qr_before_insert) +{ + ring_t entries[NENTRIES]; + ring_t *t; + unsigned i, j; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_before_insert(&entries[i - 1], &entries[i], link); + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(NENTRIES+i-j) % + NENTRIES].id, "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_reverse_foreach(t, &entries[i], link) { + assert_c_eq(t->id, entries[(i+j+1) % NENTRIES].id, + "Element id mismatch"); + j++; + } + } + for (i = 0; i < NENTRIES; i++) { + t = qr_next(&entries[i], link); + assert_c_eq(t->id, entries[(NENTRIES+i-1) % NENTRIES].id, + "Element id mismatch"); + } + for (i = 0; i < NENTRIES; i++) { + t = qr_prev(&entries[i], link); + assert_c_eq(t->id, entries[(i+1) % NENTRIES].id, + "Element id mismatch"); + } +} +TEST_END + +static void +test_split_entries(ring_t *entries) +{ + ring_t *t; + unsigned i, j; + + for (i = 0; i < NENTRIES; i++) { + j = 0; + qr_foreach(t, &entries[i], link) { + if (i < SPLIT_INDEX) { + assert_c_eq(t->id, + entries[(i+j) % SPLIT_INDEX].id, + "Element id mismatch"); + } else { + assert_c_eq(t->id, entries[(i+j-SPLIT_INDEX) % + (NENTRIES-SPLIT_INDEX) + SPLIT_INDEX].id, + "Element id mismatch"); + } + j++; + } + } +} + +TEST_BEGIN(test_qr_meld_split) +{ + ring_t entries[NENTRIES]; + unsigned i; + + init_entries(entries); + for (i = 1; i < NENTRIES; i++) + qr_after_insert(&entries[i - 1], &entries[i], link); + + qr_split(&entries[0], &entries[SPLIT_INDEX], link); + test_split_entries(entries); + + qr_meld(&entries[0], &entries[SPLIT_INDEX], link); + test_entries_ring(entries); + + qr_meld(&entries[0], &entries[SPLIT_INDEX], link); + test_split_entries(entries); + + qr_split(&entries[0], &entries[SPLIT_INDEX], link); + test_entries_ring(entries); + + qr_split(&entries[0], &entries[0], link); + test_entries_ring(entries); + + qr_meld(&entries[0], &entries[0], link); + test_entries_ring(entries); +} +TEST_END + +int +main(void) +{ + + return (test( + test_qr_one, + test_qr_after_insert, + test_qr_remove, + test_qr_before_insert, + test_qr_meld_split)); +} diff --git a/deps/jemalloc/test/unit/quarantine.c b/deps/jemalloc/test/unit/quarantine.c new file mode 100644 index 00000000000..bbd48a51ddb --- /dev/null +++ b/deps/jemalloc/test/unit/quarantine.c @@ -0,0 +1,108 @@ +#include "test/jemalloc_test.h" + +#define QUARANTINE_SIZE 8192 +#define STRINGIFY_HELPER(x) #x +#define STRINGIFY(x) STRINGIFY_HELPER(x) + +#ifdef JEMALLOC_FILL +const char *malloc_conf = "abort:false,junk:true,redzone:true,quarantine:" + STRINGIFY(QUARANTINE_SIZE); +#endif + +void +quarantine_clear(void) +{ + void *p; + + p = mallocx(QUARANTINE_SIZE*2, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + dallocx(p, 0); +} + +TEST_BEGIN(test_quarantine) +{ +#define SZ ZU(256) +#define NQUARANTINED (QUARANTINE_SIZE/SZ) + void *quarantined[NQUARANTINED+1]; + size_t i, j; + + test_skip_if(!config_fill); + + assert_zu_eq(nallocx(SZ, 0), SZ, + "SZ=%zu does not precisely equal a size class", SZ); + + quarantine_clear(); + + /* + * Allocate enough regions to completely fill the quarantine, plus one + * more. The last iteration occurs with a completely full quarantine, + * but no regions should be drained from the quarantine until the last + * deallocation occurs. Therefore no region recycling should occur + * until after this loop completes. + */ + for (i = 0; i < NQUARANTINED+1; i++) { + void *p = mallocx(SZ, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + quarantined[i] = p; + dallocx(p, 0); + for (j = 0; j < i; j++) { + assert_ptr_ne(p, quarantined[j], + "Quarantined region recycled too early; " + "i=%zu, j=%zu", i, j); + } + } +#undef NQUARANTINED +#undef SZ +} +TEST_END + +static bool detected_redzone_corruption; + +static void +arena_redzone_corruption_replacement(void *ptr, size_t usize, bool after, + size_t offset, uint8_t byte) +{ + + detected_redzone_corruption = true; +} + +TEST_BEGIN(test_quarantine_redzone) +{ + char *s; + arena_redzone_corruption_t *arena_redzone_corruption_orig; + + test_skip_if(!config_fill); + + arena_redzone_corruption_orig = arena_redzone_corruption; + arena_redzone_corruption = arena_redzone_corruption_replacement; + + /* Test underflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[-1] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + /* Test overflow. */ + detected_redzone_corruption = false; + s = (char *)mallocx(1, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + s[sallocx(s, 0)] = 0xbb; + dallocx(s, 0); + assert_true(detected_redzone_corruption, + "Did not detect redzone corruption"); + + arena_redzone_corruption = arena_redzone_corruption_orig; +} +TEST_END + +int +main(void) +{ + + return (test( + test_quarantine, + test_quarantine_redzone)); +} diff --git a/deps/jemalloc/test/unit/rb.c b/deps/jemalloc/test/unit/rb.c new file mode 100644 index 00000000000..b737485a7c6 --- /dev/null +++ b/deps/jemalloc/test/unit/rb.c @@ -0,0 +1,333 @@ +#include "test/jemalloc_test.h" + +#define rbtn_black_height(a_type, a_field, a_rbt, r_height) do { \ + a_type *rbp_bh_t; \ + for (rbp_bh_t = (a_rbt)->rbt_root, (r_height) = 0; \ + rbp_bh_t != &(a_rbt)->rbt_nil; \ + rbp_bh_t = rbtn_left_get(a_type, a_field, rbp_bh_t)) { \ + if (rbtn_red_get(a_type, a_field, rbp_bh_t) == false) { \ + (r_height)++; \ + } \ + } \ +} while (0) + +typedef struct node_s node_t; + +struct node_s { +#define NODE_MAGIC 0x9823af7e + uint32_t magic; + rb_node(node_t) link; + uint64_t key; +}; + +static int +node_cmp(node_t *a, node_t *b) { + int ret; + + assert_u32_eq(a->magic, NODE_MAGIC, "Bad magic"); + assert_u32_eq(b->magic, NODE_MAGIC, "Bad magic"); + + ret = (a->key > b->key) - (a->key < b->key); + if (ret == 0) { + /* + * Duplicates are not allowed in the tree, so force an + * arbitrary ordering for non-identical items with equal keys. + */ + ret = (((uintptr_t)a) > ((uintptr_t)b)) + - (((uintptr_t)a) < ((uintptr_t)b)); + } + return (ret); +} + +typedef rb_tree(node_t) tree_t; +rb_gen(static, tree_, tree_t, node_t, link, node_cmp); + +TEST_BEGIN(test_rb_empty) +{ + tree_t tree; + node_t key; + + tree_new(&tree); + + assert_ptr_null(tree_first(&tree), "Unexpected node"); + assert_ptr_null(tree_last(&tree), "Unexpected node"); + + key.key = 0; + key.magic = NODE_MAGIC; + assert_ptr_null(tree_search(&tree, &key), "Unexpected node"); + + key.key = 0; + key.magic = NODE_MAGIC; + assert_ptr_null(tree_nsearch(&tree, &key), "Unexpected node"); + + key.key = 0; + key.magic = NODE_MAGIC; + assert_ptr_null(tree_psearch(&tree, &key), "Unexpected node"); +} +TEST_END + +static unsigned +tree_recurse(node_t *node, unsigned black_height, unsigned black_depth, + node_t *nil) +{ + unsigned ret = 0; + node_t *left_node = rbtn_left_get(node_t, link, node); + node_t *right_node = rbtn_right_get(node_t, link, node); + + if (rbtn_red_get(node_t, link, node) == false) + black_depth++; + + /* Red nodes must be interleaved with black nodes. */ + if (rbtn_red_get(node_t, link, node)) { + assert_false(rbtn_red_get(node_t, link, left_node), + "Node should be black"); + assert_false(rbtn_red_get(node_t, link, right_node), + "Node should be black"); + } + + if (node == nil) + return (ret); + /* Self. */ + assert_u32_eq(node->magic, NODE_MAGIC, "Bad magic"); + + /* Left subtree. */ + if (left_node != nil) + ret += tree_recurse(left_node, black_height, black_depth, nil); + else + ret += (black_depth != black_height); + + /* Right subtree. */ + if (right_node != nil) + ret += tree_recurse(right_node, black_height, black_depth, nil); + else + ret += (black_depth != black_height); + + return (ret); +} + +static node_t * +tree_iterate_cb(tree_t *tree, node_t *node, void *data) +{ + unsigned *i = (unsigned *)data; + node_t *search_node; + + assert_u32_eq(node->magic, NODE_MAGIC, "Bad magic"); + + /* Test rb_search(). */ + search_node = tree_search(tree, node); + assert_ptr_eq(search_node, node, + "tree_search() returned unexpected node"); + + /* Test rb_nsearch(). */ + search_node = tree_nsearch(tree, node); + assert_ptr_eq(search_node, node, + "tree_nsearch() returned unexpected node"); + + /* Test rb_psearch(). */ + search_node = tree_psearch(tree, node); + assert_ptr_eq(search_node, node, + "tree_psearch() returned unexpected node"); + + (*i)++; + + return (NULL); +} + +static unsigned +tree_iterate(tree_t *tree) +{ + unsigned i; + + i = 0; + tree_iter(tree, NULL, tree_iterate_cb, (void *)&i); + + return (i); +} + +static unsigned +tree_iterate_reverse(tree_t *tree) +{ + unsigned i; + + i = 0; + tree_reverse_iter(tree, NULL, tree_iterate_cb, (void *)&i); + + return (i); +} + +static void +node_remove(tree_t *tree, node_t *node, unsigned nnodes) +{ + node_t *search_node; + unsigned black_height, imbalances; + + tree_remove(tree, node); + + /* Test rb_nsearch(). */ + search_node = tree_nsearch(tree, node); + if (search_node != NULL) { + assert_u64_ge(search_node->key, node->key, + "Key ordering error"); + } + + /* Test rb_psearch(). */ + search_node = tree_psearch(tree, node); + if (search_node != NULL) { + assert_u64_le(search_node->key, node->key, + "Key ordering error"); + } + + node->magic = 0; + + rbtn_black_height(node_t, link, tree, black_height); + imbalances = tree_recurse(tree->rbt_root, black_height, 0, + &(tree->rbt_nil)); + assert_u_eq(imbalances, 0, "Tree is unbalanced"); + assert_u_eq(tree_iterate(tree), nnodes-1, + "Unexpected node iteration count"); + assert_u_eq(tree_iterate_reverse(tree), nnodes-1, + "Unexpected node iteration count"); +} + +static node_t * +remove_iterate_cb(tree_t *tree, node_t *node, void *data) +{ + unsigned *nnodes = (unsigned *)data; + node_t *ret = tree_next(tree, node); + + node_remove(tree, node, *nnodes); + + return (ret); +} + +static node_t * +remove_reverse_iterate_cb(tree_t *tree, node_t *node, void *data) +{ + unsigned *nnodes = (unsigned *)data; + node_t *ret = tree_prev(tree, node); + + node_remove(tree, node, *nnodes); + + return (ret); +} + +TEST_BEGIN(test_rb_random) +{ +#define NNODES 25 +#define NBAGS 250 +#define SEED 42 + sfmt_t *sfmt; + uint64_t bag[NNODES]; + tree_t tree; + node_t nodes[NNODES]; + unsigned i, j, k, black_height, imbalances; + + sfmt = init_gen_rand(SEED); + for (i = 0; i < NBAGS; i++) { + switch (i) { + case 0: + /* Insert in order. */ + for (j = 0; j < NNODES; j++) + bag[j] = j; + break; + case 1: + /* Insert in reverse order. */ + for (j = 0; j < NNODES; j++) + bag[j] = NNODES - j - 1; + break; + default: + for (j = 0; j < NNODES; j++) + bag[j] = gen_rand64_range(sfmt, NNODES); + } + + for (j = 1; j <= NNODES; j++) { + /* Initialize tree and nodes. */ + tree_new(&tree); + tree.rbt_nil.magic = 0; + for (k = 0; k < j; k++) { + nodes[k].magic = NODE_MAGIC; + nodes[k].key = bag[k]; + } + + /* Insert nodes. */ + for (k = 0; k < j; k++) { + tree_insert(&tree, &nodes[k]); + + rbtn_black_height(node_t, link, &tree, + black_height); + imbalances = tree_recurse(tree.rbt_root, + black_height, 0, &(tree.rbt_nil)); + assert_u_eq(imbalances, 0, + "Tree is unbalanced"); + + assert_u_eq(tree_iterate(&tree), k+1, + "Unexpected node iteration count"); + assert_u_eq(tree_iterate_reverse(&tree), k+1, + "Unexpected node iteration count"); + + assert_ptr_not_null(tree_first(&tree), + "Tree should not be empty"); + assert_ptr_not_null(tree_last(&tree), + "Tree should not be empty"); + + tree_next(&tree, &nodes[k]); + tree_prev(&tree, &nodes[k]); + } + + /* Remove nodes. */ + switch (i % 4) { + case 0: + for (k = 0; k < j; k++) + node_remove(&tree, &nodes[k], j - k); + break; + case 1: + for (k = j; k > 0; k--) + node_remove(&tree, &nodes[k-1], k); + break; + case 2: { + node_t *start; + unsigned nnodes = j; + + start = NULL; + do { + start = tree_iter(&tree, start, + remove_iterate_cb, (void *)&nnodes); + nnodes--; + } while (start != NULL); + assert_u_eq(nnodes, 0, + "Removal terminated early"); + break; + } case 3: { + node_t *start; + unsigned nnodes = j; + + start = NULL; + do { + start = tree_reverse_iter(&tree, start, + remove_reverse_iterate_cb, + (void *)&nnodes); + nnodes--; + } while (start != NULL); + assert_u_eq(nnodes, 0, + "Removal terminated early"); + break; + } default: + not_reached(); + } + } + } + fini_gen_rand(sfmt); +#undef NNODES +#undef NBAGS +#undef SEED +} +TEST_END + +int +main(void) +{ + + return (test( + test_rb_empty, + test_rb_random)); +} diff --git a/deps/jemalloc/test/unit/rtree.c b/deps/jemalloc/test/unit/rtree.c new file mode 100644 index 00000000000..5463055fe92 --- /dev/null +++ b/deps/jemalloc/test/unit/rtree.c @@ -0,0 +1,118 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_rtree_get_empty) +{ + unsigned i; + + for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + assert_u_eq(rtree_get(rtree, 0), 0, + "rtree_get() should return NULL for empty tree"); + rtree_delete(rtree); + } +} +TEST_END + +TEST_BEGIN(test_rtree_extrema) +{ + unsigned i; + + for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + + rtree_set(rtree, 0, 1); + assert_u_eq(rtree_get(rtree, 0), 1, + "rtree_get() should return previously set value"); + + rtree_set(rtree, ~((uintptr_t)0), 1); + assert_u_eq(rtree_get(rtree, ~((uintptr_t)0)), 1, + "rtree_get() should return previously set value"); + + rtree_delete(rtree); + } +} +TEST_END + +TEST_BEGIN(test_rtree_bits) +{ + unsigned i, j, k; + + for (i = 1; i < (sizeof(uintptr_t) << 3); i++) { + uintptr_t keys[] = {0, 1, + (((uintptr_t)1) << (sizeof(uintptr_t)*8-i)) - 1}; + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + + for (j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { + rtree_set(rtree, keys[j], 1); + for (k = 0; k < sizeof(keys)/sizeof(uintptr_t); k++) { + assert_u_eq(rtree_get(rtree, keys[k]), 1, + "rtree_get() should return previously set " + "value and ignore insignificant key bits; " + "i=%u, j=%u, k=%u, set key=%#"PRIxPTR", " + "get key=%#"PRIxPTR, i, j, k, keys[j], + keys[k]); + } + assert_u_eq(rtree_get(rtree, + (((uintptr_t)1) << (sizeof(uintptr_t)*8-i))), 0, + "Only leftmost rtree leaf should be set; " + "i=%u, j=%u", i, j); + rtree_set(rtree, keys[j], 0); + } + + rtree_delete(rtree); + } +} +TEST_END + +TEST_BEGIN(test_rtree_random) +{ + unsigned i; + sfmt_t *sfmt; +#define NSET 100 +#define SEED 42 + + sfmt = init_gen_rand(SEED); + for (i = 1; i <= (sizeof(uintptr_t) << 3); i++) { + rtree_t *rtree = rtree_new(i, imalloc, idalloc); + uintptr_t keys[NSET]; + unsigned j; + + for (j = 0; j < NSET; j++) { + keys[j] = (uintptr_t)gen_rand64(sfmt); + rtree_set(rtree, keys[j], 1); + assert_u_eq(rtree_get(rtree, keys[j]), 1, + "rtree_get() should return previously set value"); + } + for (j = 0; j < NSET; j++) { + assert_u_eq(rtree_get(rtree, keys[j]), 1, + "rtree_get() should return previously set value"); + } + + for (j = 0; j < NSET; j++) { + rtree_set(rtree, keys[j], 0); + assert_u_eq(rtree_get(rtree, keys[j]), 0, + "rtree_get() should return previously set value"); + } + for (j = 0; j < NSET; j++) { + assert_u_eq(rtree_get(rtree, keys[j]), 0, + "rtree_get() should return previously set value"); + } + + rtree_delete(rtree); + } + fini_gen_rand(sfmt); +#undef NSET +#undef SEED +} +TEST_END + +int +main(void) +{ + + return (test( + test_rtree_get_empty, + test_rtree_extrema, + test_rtree_bits, + test_rtree_random)); +} diff --git a/deps/jemalloc/test/unit/stats.c b/deps/jemalloc/test/unit/stats.c new file mode 100644 index 00000000000..03a55c7fdce --- /dev/null +++ b/deps/jemalloc/test/unit/stats.c @@ -0,0 +1,380 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_stats_summary) +{ + size_t *cactive; + size_t sz, allocated, active, mapped; + int expected = config_stats ? 0 : ENOENT; + + sz = sizeof(cactive); + assert_d_eq(mallctl("stats.cactive", &cactive, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.allocated", &allocated, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.active", &active, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.mapped", &mapped, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_le(active, *cactive, + "active should be no larger than cactive"); + assert_zu_le(allocated, active, + "allocated should be no larger than active"); + assert_zu_le(active, mapped, + "active should be no larger than mapped"); + } +} +TEST_END + +TEST_BEGIN(test_stats_chunks) +{ + size_t current, high; + uint64_t total; + size_t sz; + int expected = config_stats ? 0 : ENOENT; + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.chunks.current", ¤t, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.chunks.total", &total, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.chunks.high", &high, &sz, NULL, 0), expected, + "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_le(current, high, + "current should be no larger than high"); + assert_u64_le((uint64_t)high, total, + "high should be no larger than total"); + } +} +TEST_END + +TEST_BEGIN(test_stats_huge) +{ + void *p; + uint64_t epoch; + size_t allocated; + uint64_t nmalloc, ndalloc; + size_t sz; + int expected = config_stats ? 0 : ENOENT; + + p = mallocx(arena_maxclass+1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.huge.allocated", &allocated, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.huge.nmalloc", &nmalloc, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.huge.ndalloc", &ndalloc, &sz, NULL, 0), + expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_summary) +{ + unsigned arena; + void *small, *large; + uint64_t epoch; + size_t sz; + int expected = config_stats ? 0 : ENOENT; + size_t mapped; + uint64_t npurge, nmadvise, purged; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + small = mallocx(SMALL_MAXCLASS, 0); + assert_ptr_not_null(small, "Unexpected mallocx() failure"); + large = mallocx(arena_maxclass, 0); + assert_ptr_not_null(large, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.mapped", &mapped, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.npurge", &npurge, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.nmadvise", &nmadvise, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.purged", &purged, &sz, NULL, 0), + expected, "Unexepected mallctl() result"); + + if (config_stats) { + assert_u64_gt(npurge, 0, + "At least one purge should have occurred"); + assert_u64_le(nmadvise, purged, + "nmadvise should be no greater than purged"); + } + + dallocx(small, 0); + dallocx(large, 0); +} +TEST_END + +void * +thd_start(void *arg) +{ + + return (NULL); +} + +static void +no_lazy_lock(void) +{ + thd_t thd; + + thd_create(&thd, thd_start, NULL); + thd_join(thd, NULL); +} + +TEST_BEGIN(test_stats_arenas_small) +{ + unsigned arena; + void *p; + size_t sz, allocated; + uint64_t epoch, nmalloc, ndalloc, nrequests; + int expected = config_stats ? 0 : ENOENT; + + no_lazy_lock(); /* Lazy locking would dodge tcache testing. */ + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(SMALL_MAXCLASS, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), + config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.small.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.small.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.small.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.small.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_u64_gt(nmalloc, 0, + "nmalloc should be no greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(nrequests, 0, + "nrequests should be greater than zero"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_large) +{ + unsigned arena; + void *p; + size_t sz, allocated; + uint64_t epoch, nmalloc, ndalloc, nrequests; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(arena_maxclass, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.large.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.large.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.large.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.large.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_zu_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_zu_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_zu_gt(nrequests, 0, + "nrequests should be greater than zero"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_bins) +{ + unsigned arena; + void *p; + size_t sz, allocated, curruns; + uint64_t epoch, nmalloc, ndalloc, nrequests, nfills, nflushes; + uint64_t nruns, nreruns; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(arena_bin_info[0].reg_size, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), + config_tcache ? 0 : ENOENT, "Unexpected mallctl() result"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.allocated", &allocated, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + assert_d_eq(mallctl("stats.arenas.0.bins.0.nfills", &nfills, &sz, + NULL, 0), config_tcache ? expected : ENOENT, + "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nflushes", &nflushes, &sz, + NULL, 0), config_tcache ? expected : ENOENT, + "Unexpected mallctl() result"); + + assert_d_eq(mallctl("stats.arenas.0.bins.0.nruns", &nruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.bins.0.nreruns", &nreruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.bins.0.curruns", &curruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_zu_gt(allocated, 0, + "allocated should be greater than zero"); + assert_u64_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(nrequests, 0, + "nrequests should be greater than zero"); + if (config_tcache) { + assert_u64_gt(nfills, 0, + "At least one fill should have occurred"); + assert_u64_gt(nflushes, 0, + "At least one flush should have occurred"); + } + assert_u64_gt(nruns, 0, + "At least one run should have been allocated"); + assert_zu_gt(curruns, 0, + "At least one run should be currently allocated"); + } + + dallocx(p, 0); +} +TEST_END + +TEST_BEGIN(test_stats_arenas_lruns) +{ + unsigned arena; + void *p; + uint64_t epoch, nmalloc, ndalloc, nrequests; + size_t curruns, sz; + int expected = config_stats ? 0 : ENOENT; + + arena = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena, sizeof(arena)), + 0, "Unexpected mallctl() failure"); + + p = mallocx(SMALL_MAXCLASS+1, 0); + assert_ptr_not_null(p, "Unexpected mallocx() failure"); + + assert_d_eq(mallctl("epoch", NULL, NULL, &epoch, sizeof(epoch)), 0, + "Unexpected mallctl() failure"); + + sz = sizeof(uint64_t); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.nmalloc", &nmalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.ndalloc", &ndalloc, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.nrequests", &nrequests, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + sz = sizeof(size_t); + assert_d_eq(mallctl("stats.arenas.0.lruns.0.curruns", &curruns, &sz, + NULL, 0), expected, "Unexpected mallctl() result"); + + if (config_stats) { + assert_u64_gt(nmalloc, 0, + "nmalloc should be greater than zero"); + assert_u64_ge(nmalloc, ndalloc, + "nmalloc should be at least as large as ndalloc"); + assert_u64_gt(nrequests, 0, + "nrequests should be greater than zero"); + assert_u64_gt(curruns, 0, + "At least one run should be currently allocated"); + } + + dallocx(p, 0); +} +TEST_END + +int +main(void) +{ + + return (test( + test_stats_summary, + test_stats_chunks, + test_stats_huge, + test_stats_arenas_summary, + test_stats_arenas_small, + test_stats_arenas_large, + test_stats_arenas_bins, + test_stats_arenas_lruns)); +} diff --git a/deps/jemalloc/test/unit/tsd.c b/deps/jemalloc/test/unit/tsd.c new file mode 100644 index 00000000000..f421c1a3cff --- /dev/null +++ b/deps/jemalloc/test/unit/tsd.c @@ -0,0 +1,71 @@ +#include "test/jemalloc_test.h" + +#define THREAD_DATA 0x72b65c10 + +typedef unsigned int data_t; + +static bool data_cleanup_executed; + +void +data_cleanup(void *arg) +{ + data_t *data = (data_t *)arg; + + assert_x_eq(*data, THREAD_DATA, + "Argument passed into cleanup function should match tsd value"); + data_cleanup_executed = true; +} + +malloc_tsd_protos(, data, data_t) +malloc_tsd_externs(data, data_t) +#define DATA_INIT 0x12345678 +malloc_tsd_data(, data, data_t, DATA_INIT) +malloc_tsd_funcs(, data, data_t, DATA_INIT, data_cleanup) + +static void * +thd_start(void *arg) +{ + data_t d = (data_t)(uintptr_t)arg; + assert_x_eq(*data_tsd_get(), DATA_INIT, + "Initial tsd get should return initialization value"); + + data_tsd_set(&d); + assert_x_eq(*data_tsd_get(), d, + "After tsd set, tsd get should return value that was set"); + + d = 0; + assert_x_eq(*data_tsd_get(), (data_t)(uintptr_t)arg, + "Resetting local data should have no effect on tsd"); + + return (NULL); +} + +TEST_BEGIN(test_tsd_main_thread) +{ + + thd_start((void *) 0xa5f3e329); +} +TEST_END + +TEST_BEGIN(test_tsd_sub_thread) +{ + thd_t thd; + + data_cleanup_executed = false; + thd_create(&thd, thd_start, (void *)THREAD_DATA); + thd_join(thd, NULL); + assert_true(data_cleanup_executed, + "Cleanup function should have executed"); +} +TEST_END + +int +main(void) +{ + + data_tsd_boot(); + + return (test( + test_tsd_main_thread, + test_tsd_sub_thread)); +} diff --git a/deps/jemalloc/test/unit/util.c b/deps/jemalloc/test/unit/util.c new file mode 100644 index 00000000000..dc3cfe8a9db --- /dev/null +++ b/deps/jemalloc/test/unit/util.c @@ -0,0 +1,294 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_pow2_ceil) +{ + unsigned i, pow2; + size_t x; + + assert_zu_eq(pow2_ceil(0), 0, "Unexpected result"); + + for (i = 0; i < sizeof(size_t) * 8; i++) { + assert_zu_eq(pow2_ceil(ZU(1) << i), ZU(1) << i, + "Unexpected result"); + } + + for (i = 2; i < sizeof(size_t) * 8; i++) { + assert_zu_eq(pow2_ceil((ZU(1) << i) - 1), ZU(1) << i, + "Unexpected result"); + } + + for (i = 0; i < sizeof(size_t) * 8 - 1; i++) { + assert_zu_eq(pow2_ceil((ZU(1) << i) + 1), ZU(1) << (i+1), + "Unexpected result"); + } + + for (pow2 = 1; pow2 < 25; pow2++) { + for (x = (ZU(1) << (pow2-1)) + 1; x <= ZU(1) << pow2; x++) { + assert_zu_eq(pow2_ceil(x), ZU(1) << pow2, + "Unexpected result, x=%zu", x); + } + } +} +TEST_END + +TEST_BEGIN(test_malloc_strtoumax_no_endptr) +{ + int err; + + set_errno(0); + assert_ju_eq(malloc_strtoumax("0", NULL, 0), 0, "Unexpected result"); + err = get_errno(); + assert_d_eq(err, 0, "Unexpected failure"); +} +TEST_END + +TEST_BEGIN(test_malloc_strtoumax) +{ + struct test_s { + const char *input; + const char *expected_remainder; + int base; + int expected_errno; + const char *expected_errno_name; + uintmax_t expected_x; + }; +#define ERR(e) e, #e +#define UMAX(x) ((uintmax_t)x##ULL) + struct test_s tests[] = { + {"0", "0", -1, ERR(EINVAL), UINTMAX_MAX}, + {"0", "0", 1, ERR(EINVAL), UINTMAX_MAX}, + {"0", "0", 37, ERR(EINVAL), UINTMAX_MAX}, + + {"", "", 0, ERR(EINVAL), UINTMAX_MAX}, + {"+", "+", 0, ERR(EINVAL), UINTMAX_MAX}, + {"++3", "++3", 0, ERR(EINVAL), UINTMAX_MAX}, + {"-", "-", 0, ERR(EINVAL), UINTMAX_MAX}, + + {"42", "", 0, ERR(0), UMAX(42)}, + {"+42", "", 0, ERR(0), UMAX(42)}, + {"-42", "", 0, ERR(0), UMAX(-42)}, + {"042", "", 0, ERR(0), UMAX(042)}, + {"+042", "", 0, ERR(0), UMAX(042)}, + {"-042", "", 0, ERR(0), UMAX(-042)}, + {"0x42", "", 0, ERR(0), UMAX(0x42)}, + {"+0x42", "", 0, ERR(0), UMAX(0x42)}, + {"-0x42", "", 0, ERR(0), UMAX(-0x42)}, + + {"0", "", 0, ERR(0), UMAX(0)}, + {"1", "", 0, ERR(0), UMAX(1)}, + + {"42", "", 0, ERR(0), UMAX(42)}, + {" 42", "", 0, ERR(0), UMAX(42)}, + {"42 ", " ", 0, ERR(0), UMAX(42)}, + {"0x", "x", 0, ERR(0), UMAX(0)}, + {"42x", "x", 0, ERR(0), UMAX(42)}, + + {"07", "", 0, ERR(0), UMAX(7)}, + {"010", "", 0, ERR(0), UMAX(8)}, + {"08", "8", 0, ERR(0), UMAX(0)}, + {"0_", "_", 0, ERR(0), UMAX(0)}, + + {"0x", "x", 0, ERR(0), UMAX(0)}, + {"0X", "X", 0, ERR(0), UMAX(0)}, + {"0xg", "xg", 0, ERR(0), UMAX(0)}, + {"0XA", "", 0, ERR(0), UMAX(10)}, + + {"010", "", 10, ERR(0), UMAX(10)}, + {"0x3", "x3", 10, ERR(0), UMAX(0)}, + + {"12", "2", 2, ERR(0), UMAX(1)}, + {"78", "8", 8, ERR(0), UMAX(7)}, + {"9a", "a", 10, ERR(0), UMAX(9)}, + {"9A", "A", 10, ERR(0), UMAX(9)}, + {"fg", "g", 16, ERR(0), UMAX(15)}, + {"FG", "G", 16, ERR(0), UMAX(15)}, + {"0xfg", "g", 16, ERR(0), UMAX(15)}, + {"0XFG", "G", 16, ERR(0), UMAX(15)}, + {"z_", "_", 36, ERR(0), UMAX(35)}, + {"Z_", "_", 36, ERR(0), UMAX(35)} + }; +#undef ERR +#undef UMAX + unsigned i; + + for (i = 0; i < sizeof(tests)/sizeof(struct test_s); i++) { + struct test_s *test = &tests[i]; + int err; + uintmax_t result; + char *remainder; + + set_errno(0); + result = malloc_strtoumax(test->input, &remainder, test->base); + err = get_errno(); + assert_d_eq(err, test->expected_errno, + "Expected errno %s for \"%s\", base %d", + test->expected_errno_name, test->input, test->base); + assert_str_eq(remainder, test->expected_remainder, + "Unexpected remainder for \"%s\", base %d", + test->input, test->base); + if (err == 0) { + assert_ju_eq(result, test->expected_x, + "Unexpected result for \"%s\", base %d", + test->input, test->base); + } + } +} +TEST_END + +TEST_BEGIN(test_malloc_snprintf_truncated) +{ +#define BUFLEN 15 + char buf[BUFLEN]; + int result; + size_t len; +#define TEST(expected_str_untruncated, fmt...) do { \ + result = malloc_snprintf(buf, len, fmt); \ + assert_d_eq(strncmp(buf, expected_str_untruncated, len-1), 0, \ + "Unexpected string inequality (\"%s\" vs \"%s\")", \ + buf, expected_str_untruncated); \ + assert_d_eq(result, strlen(expected_str_untruncated), \ + "Unexpected result"); \ +} while (0) + + for (len = 1; len < BUFLEN; len++) { + TEST("012346789", "012346789"); + TEST("a0123b", "a%sb", "0123"); + TEST("a01234567", "a%s%s", "0123", "4567"); + TEST("a0123 ", "a%-6s", "0123"); + TEST("a 0123", "a%6s", "0123"); + TEST("a 012", "a%6.3s", "0123"); + TEST("a 012", "a%*.*s", 6, 3, "0123"); + TEST("a 123b", "a% db", 123); + TEST("a123b", "a%-db", 123); + TEST("a-123b", "a%-db", -123); + TEST("a+123b", "a%+db", 123); + } +#undef BUFLEN +#undef TEST +} +TEST_END + +TEST_BEGIN(test_malloc_snprintf) +{ +#define BUFLEN 128 + char buf[BUFLEN]; + int result; +#define TEST(expected_str, fmt...) do { \ + result = malloc_snprintf(buf, sizeof(buf), fmt); \ + assert_str_eq(buf, expected_str, "Unexpected output"); \ + assert_d_eq(result, strlen(expected_str), "Unexpected result"); \ +} while (0) + + TEST("hello", "hello"); + + TEST("50%, 100%", "50%%, %d%%", 100); + + TEST("a0123b", "a%sb", "0123"); + + TEST("a 0123b", "a%5sb", "0123"); + TEST("a 0123b", "a%*sb", 5, "0123"); + + TEST("a0123 b", "a%-5sb", "0123"); + TEST("a0123b", "a%*sb", -1, "0123"); + TEST("a0123 b", "a%*sb", -5, "0123"); + TEST("a0123 b", "a%-*sb", -5, "0123"); + + TEST("a012b", "a%.3sb", "0123"); + TEST("a012b", "a%.*sb", 3, "0123"); + TEST("a0123b", "a%.*sb", -3, "0123"); + + TEST("a 012b", "a%5.3sb", "0123"); + TEST("a 012b", "a%5.*sb", 3, "0123"); + TEST("a 012b", "a%*.3sb", 5, "0123"); + TEST("a 012b", "a%*.*sb", 5, 3, "0123"); + TEST("a 0123b", "a%*.*sb", 5, -3, "0123"); + + TEST("_abcd_", "_%x_", 0xabcd); + TEST("_0xabcd_", "_%#x_", 0xabcd); + TEST("_1234_", "_%o_", 01234); + TEST("_01234_", "_%#o_", 01234); + TEST("_1234_", "_%u_", 1234); + + TEST("_1234_", "_%d_", 1234); + TEST("_ 1234_", "_% d_", 1234); + TEST("_+1234_", "_%+d_", 1234); + TEST("_-1234_", "_%d_", -1234); + TEST("_-1234_", "_% d_", -1234); + TEST("_-1234_", "_%+d_", -1234); + + TEST("_-1234_", "_%d_", -1234); + TEST("_1234_", "_%d_", 1234); + TEST("_-1234_", "_%i_", -1234); + TEST("_1234_", "_%i_", 1234); + TEST("_01234_", "_%#o_", 01234); + TEST("_1234_", "_%u_", 1234); + TEST("_0x1234abc_", "_%#x_", 0x1234abc); + TEST("_0X1234ABC_", "_%#X_", 0x1234abc); + TEST("_c_", "_%c_", 'c'); + TEST("_string_", "_%s_", "string"); + TEST("_0x42_", "_%p_", ((void *)0x42)); + + TEST("_-1234_", "_%ld_", ((long)-1234)); + TEST("_1234_", "_%ld_", ((long)1234)); + TEST("_-1234_", "_%li_", ((long)-1234)); + TEST("_1234_", "_%li_", ((long)1234)); + TEST("_01234_", "_%#lo_", ((long)01234)); + TEST("_1234_", "_%lu_", ((long)1234)); + TEST("_0x1234abc_", "_%#lx_", ((long)0x1234abc)); + TEST("_0X1234ABC_", "_%#lX_", ((long)0x1234ABC)); + + TEST("_-1234_", "_%lld_", ((long long)-1234)); + TEST("_1234_", "_%lld_", ((long long)1234)); + TEST("_-1234_", "_%lli_", ((long long)-1234)); + TEST("_1234_", "_%lli_", ((long long)1234)); + TEST("_01234_", "_%#llo_", ((long long)01234)); + TEST("_1234_", "_%llu_", ((long long)1234)); + TEST("_0x1234abc_", "_%#llx_", ((long long)0x1234abc)); + TEST("_0X1234ABC_", "_%#llX_", ((long long)0x1234ABC)); + + TEST("_-1234_", "_%qd_", ((long long)-1234)); + TEST("_1234_", "_%qd_", ((long long)1234)); + TEST("_-1234_", "_%qi_", ((long long)-1234)); + TEST("_1234_", "_%qi_", ((long long)1234)); + TEST("_01234_", "_%#qo_", ((long long)01234)); + TEST("_1234_", "_%qu_", ((long long)1234)); + TEST("_0x1234abc_", "_%#qx_", ((long long)0x1234abc)); + TEST("_0X1234ABC_", "_%#qX_", ((long long)0x1234ABC)); + + TEST("_-1234_", "_%jd_", ((intmax_t)-1234)); + TEST("_1234_", "_%jd_", ((intmax_t)1234)); + TEST("_-1234_", "_%ji_", ((intmax_t)-1234)); + TEST("_1234_", "_%ji_", ((intmax_t)1234)); + TEST("_01234_", "_%#jo_", ((intmax_t)01234)); + TEST("_1234_", "_%ju_", ((intmax_t)1234)); + TEST("_0x1234abc_", "_%#jx_", ((intmax_t)0x1234abc)); + TEST("_0X1234ABC_", "_%#jX_", ((intmax_t)0x1234ABC)); + + TEST("_1234_", "_%td_", ((ptrdiff_t)1234)); + TEST("_-1234_", "_%td_", ((ptrdiff_t)-1234)); + TEST("_1234_", "_%ti_", ((ptrdiff_t)1234)); + TEST("_-1234_", "_%ti_", ((ptrdiff_t)-1234)); + + TEST("_-1234_", "_%zd_", ((ssize_t)-1234)); + TEST("_1234_", "_%zd_", ((ssize_t)1234)); + TEST("_-1234_", "_%zi_", ((ssize_t)-1234)); + TEST("_1234_", "_%zi_", ((ssize_t)1234)); + TEST("_01234_", "_%#zo_", ((ssize_t)01234)); + TEST("_1234_", "_%zu_", ((ssize_t)1234)); + TEST("_0x1234abc_", "_%#zx_", ((ssize_t)0x1234abc)); + TEST("_0X1234ABC_", "_%#zX_", ((ssize_t)0x1234ABC)); +#undef BUFLEN +} +TEST_END + +int +main(void) +{ + + return (test( + test_pow2_ceil, + test_malloc_strtoumax_no_endptr, + test_malloc_strtoumax, + test_malloc_snprintf_truncated, + test_malloc_snprintf)); +} diff --git a/deps/jemalloc/test/unit/zero.c b/deps/jemalloc/test/unit/zero.c new file mode 100644 index 00000000000..65a8f0c9c32 --- /dev/null +++ b/deps/jemalloc/test/unit/zero.c @@ -0,0 +1,78 @@ +#include "test/jemalloc_test.h" + +#ifdef JEMALLOC_FILL +const char *malloc_conf = + "abort:false,junk:false,zero:true,redzone:false,quarantine:0"; +#endif + +static void +test_zero(size_t sz_min, size_t sz_max) +{ + char *s; + size_t sz_prev, sz, i; + + sz_prev = 0; + s = (char *)mallocx(sz_min, 0); + assert_ptr_not_null((void *)s, "Unexpected mallocx() failure"); + + for (sz = sallocx(s, 0); sz <= sz_max; + sz_prev = sz, sz = sallocx(s, 0)) { + if (sz_prev > 0) { + assert_c_eq(s[0], 'a', + "Previously allocated byte %zu/%zu is corrupted", + ZU(0), sz_prev); + assert_c_eq(s[sz_prev-1], 'a', + "Previously allocated byte %zu/%zu is corrupted", + sz_prev-1, sz_prev); + } + + for (i = sz_prev; i < sz; i++) { + assert_c_eq(s[i], 0x0, + "Newly allocated byte %zu/%zu isn't zero-filled", + i, sz); + s[i] = 'a'; + } + + if (xallocx(s, sz+1, 0, 0) == sz) { + s = (char *)rallocx(s, sz+1, 0); + assert_ptr_not_null((void *)s, + "Unexpected rallocx() failure"); + } + } + + dallocx(s, 0); +} + +TEST_BEGIN(test_zero_small) +{ + + test_skip_if(!config_fill); + test_zero(1, SMALL_MAXCLASS-1); +} +TEST_END + +TEST_BEGIN(test_zero_large) +{ + + test_skip_if(!config_fill); + test_zero(SMALL_MAXCLASS+1, arena_maxclass); +} +TEST_END + +TEST_BEGIN(test_zero_huge) +{ + + test_skip_if(!config_fill); + test_zero(arena_maxclass+1, chunksize*2); +} +TEST_END + +int +main(void) +{ + + return (test( + test_zero_small, + test_zero_large, + test_zero_huge)); +} From 3a915ace446878869fb7c5c7dfbf8e7d605e12df Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Apr 2014 10:52:08 +0200 Subject: [PATCH 1327/1752] sdscatfmt() added to SDS library. sdscatprintf() relies on printf() family libc functions and is sometimes too slow in critical code paths. sdscatfmt() is an alternative which is: 1) Far less capable. 2) Format specifier uncompatible. 3) Faster. It is suitable to be used in those speed critical code paths such as CLIENT LIST output generation. --- src/sds.c | 192 ++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 165 insertions(+), 27 deletions(-) diff --git a/src/sds.c b/src/sds.c index f1ec5c9b4d2..096ac0496f7 100644 --- a/src/sds.c +++ b/src/sds.c @@ -289,6 +289,55 @@ sds sdscpy(sds s, const char *t) { return sdscpylen(s, t, strlen(t)); } +/* Helper for sdscatlonglong() doing the actual number -> string + * conversion. 's' must point to a string with room for at least + * SDS_LLSTR_SIZE bytes. + * + * The function returns the lenght of the null-terminated string + * representation stored at 's'. */ +#define SDS_LLSTR_SIZE 21 +int sdsll2str(char *s, long long value) { + char *p, aux; + unsigned long long v; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + v = (value < 0) ? -value : value; + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + if (value < 0) *p++ = '-'; + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + +/* Create an sds string from a long long value. It is much faster than: + * + * sdscatprintf(sdsempty(),"%lld\n", value); + */ +sds sdsfromlonglong(long long value) { + char buf[SDS_LLSTR_SIZE]; + int len = sdsll2str(buf,value); + + return sdsnewlen(buf,len); +} + /* Like sdscatpritf() but gets va_list instead of being variadic. */ sds sdscatvprintf(sds s, const char *fmt, va_list ap) { va_list cpy; @@ -351,6 +400,91 @@ sds sdscatprintf(sds s, const char *fmt, ...) { return t; } +/* This function is similar to sdscatprintf, but much faster as it does + * not rely on sprintf() family functions implemented by the libc that + * are often very slow. Moreover directly handling the sds string as + * new data is concatenated provides a performance improvement. + * + * However this function only handles an incompatible subset of printf-alike + * format specifiers: + * + * %s - C String + * %S - SDS string + * %i - signed integer + * %I - 64 bit signed integer (long long, int64_t) + * %% - Verbatim "%" character. + */ +sds sdscatfmt(sds s, char const *fmt, ...) { + struct sdshdr *sh = (void*) (s-(sizeof(struct sdshdr))); + size_t initlen = sdslen(s); + const char *f = fmt; + int i; + va_list ap; + + va_start(ap,fmt); + f = fmt; /* Next format specifier byte to process. */ + i = initlen; /* Position of the next byte to write to dest str. */ + while(*f) { + char next, *str; + size_t l; + long long num; + + /* Make sure there is always space for at least 1 char. */ + if (sh->free == 0) s = sdsMakeRoomFor(s,1); + + switch(*f) { + case '%': + next = *(f+1); + f++; + switch(next) { + case 's': + case 'S': + str = va_arg(ap,char*); + l = (next == 's') ? strlen(str) : sdslen(str); + if (sh->free < l) s = sdsMakeRoomFor(s,l); + memcpy(s+i,str,l); + sh->len += l; + sh->free -= l; + i += l; + break; + case 'i': + case 'I': + if (next == 'i') + num = va_arg(ap,int); + else + num = va_arg(ap,long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsll2str(buf,num); + if (sh->free < l) s = sdsMakeRoomFor(s,l); + memcpy(s+i,buf,l); + sh->len += l; + sh->free -= l; + i += l; + } + break; + default: /* Handle %% and generally %. */ + s[i++] = next; + sh->len += 1; + sh->free -= 1; + break; + } + break; + default: + s[i++] = *f; + sh->len += 1; + sh->free -= 1; + break; + } + f++; + } + va_end(ap); + + /* Add null-term */ + s[i] = '\0'; + return s; +} + /* Remove the part of the string from left and from right composed just of * contiguous characters found in 'cset', that is a null terminted C string. * @@ -538,25 +672,6 @@ void sdsfreesplitres(sds *tokens, int count) { zfree(tokens); } -/* Create an sds string from a long long value. It is much faster than: - * - * sdscatprintf(sdsempty(),"%lld\n", value); - */ -sds sdsfromlonglong(long long value) { - char buf[32], *p; - unsigned long long v; - - v = (value < 0) ? -value : value; - p = buf+31; /* point to the last character */ - do { - *p-- = '0'+(v%10); - v /= 10; - } while(v); - if (value < 0) *p-- = '-'; - p++; - return sdsnewlen(p,32-(p-buf)); -} - /* Append to the sds string "s" an escaped string representation where * all the non-printable characters (tested with isprint()) are turned into * escapes in the form "\n\r\a...." or "\x". @@ -787,6 +902,7 @@ sds sdsjoin(char **argv, int argc, char *sep) { #ifdef SDS_TEST_MAIN #include #include "testhelp.h" +#include "limits.h" int main(void) { { @@ -817,39 +933,54 @@ int main(void) { sdsfree(x); x = sdscatprintf(sdsempty(),"%d",123); test_cond("sdscatprintf() seems working in the base case", - sdslen(x) == 3 && memcmp(x,"123\0",4) ==0) + sdslen(x) == 3 && memcmp(x,"123\0",4) == 0) + + sdsfree(x); + x = sdsnew("--"); + x = sdscatfmt(x, "Hello %s World %I,%I--", "Hi!", LLONG_MIN,LLONG_MAX); + test_cond("sdscatfmt() seems working in the base case", + sdslen(x) == 60 && + memcmp(x,"--Hello Hi! World -9223372036854775808," + "9223372036854775807--",60) == 0) sdsfree(x); - x = sdstrim(sdsnew("xxciaoyyy"),"xy"); + x = sdsnew("xxciaoyyy"); + sdstrim(x,"xy"); test_cond("sdstrim() correctly trims characters", sdslen(x) == 4 && memcmp(x,"ciao\0",5) == 0) - y = sdsrange(sdsdup(x),1,1); + y = sdsdup(x); + sdsrange(y,1,1); test_cond("sdsrange(...,1,1)", sdslen(y) == 1 && memcmp(y,"i\0",2) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),1,-1); + y = sdsdup(x); + sdsrange(y,1,-1); test_cond("sdsrange(...,1,-1)", sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),-2,-1); + y = sdsdup(x); + sdsrange(y,-2,-1); test_cond("sdsrange(...,-2,-1)", sdslen(y) == 2 && memcmp(y,"ao\0",3) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),2,1); + y = sdsdup(x); + sdsrange(y,2,1); test_cond("sdsrange(...,2,1)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),1,100); + y = sdsdup(x); + sdsrange(y,1,100); test_cond("sdsrange(...,1,100)", sdslen(y) == 3 && memcmp(y,"iao\0",4) == 0) sdsfree(y); - y = sdsrange(sdsdup(x),100,100); + y = sdsdup(x); + sdsrange(y,100,100); test_cond("sdsrange(...,100,100)", sdslen(y) == 0 && memcmp(y,"\0",1) == 0) @@ -871,6 +1002,13 @@ int main(void) { y = sdsnew("bar"); test_cond("sdscmp(bar,bar)", sdscmp(x,y) < 0) + sdsfree(y); + sdsfree(x); + x = sdsnewlen("\a\n\0foo\r",7); + y = sdscatrepr(sdsempty(),x,sdslen(x)); + test_cond("sdscatrepr(...data...)", + memcmp(y,"\"\\a\\n\\x00foo\\r\"",15) == 0) + { int oldfree; From 4acc3daa92bb69d93857beab9095ef95850811f8 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Apr 2014 16:38:17 +0200 Subject: [PATCH 1328/1752] Added new sdscatfmt() %u and %U format specifiers. This commit also fixes a bug in the implementation of sdscatfmt() resulting from stale references to the SDS string header after sdsMakeRoomFor() calls. --- src/sds.c | 75 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 71 insertions(+), 4 deletions(-) diff --git a/src/sds.c b/src/sds.c index 096ac0496f7..b64ffb13a84 100644 --- a/src/sds.c +++ b/src/sds.c @@ -327,6 +327,35 @@ int sdsll2str(char *s, long long value) { return l; } +/* Identical sdsll2str(), but for unsigned long long type. */ +int sdsull2str(char *s, unsigned long long v) { + char *p, aux; + size_t l; + + /* Generate the string representation, this method produces + * an reversed string. */ + p = s; + do { + *p++ = '0'+(v%10); + v /= 10; + } while(v); + + /* Compute length and add null term. */ + l = p-s; + *p = '\0'; + + /* Reverse the string. */ + p--; + while(s < p) { + aux = *s; + *s = *p; + *p = aux; + s++; + p--; + } + return l; +} + /* Create an sds string from a long long value. It is much faster than: * * sdscatprintf(sdsempty(),"%lld\n", value); @@ -410,8 +439,10 @@ sds sdscatprintf(sds s, const char *fmt, ...) { * * %s - C String * %S - SDS string - * %i - signed integer + * %i - signed int * %I - 64 bit signed integer (long long, int64_t) + * %u - unsigned int + * %U - 64 bit unsigned integer (unsigned long long, uint64_t) * %% - Verbatim "%" character. */ sds sdscatfmt(sds s, char const *fmt, ...) { @@ -428,9 +459,13 @@ sds sdscatfmt(sds s, char const *fmt, ...) { char next, *str; size_t l; long long num; + unsigned long long unum; /* Make sure there is always space for at least 1 char. */ - if (sh->free == 0) s = sdsMakeRoomFor(s,1); + if (sh->free == 0) { + s = sdsMakeRoomFor(s,1); + sh = (void*) (s-(sizeof(struct sdshdr))); + } switch(*f) { case '%': @@ -441,7 +476,10 @@ sds sdscatfmt(sds s, char const *fmt, ...) { case 'S': str = va_arg(ap,char*); l = (next == 's') ? strlen(str) : sdslen(str); - if (sh->free < l) s = sdsMakeRoomFor(s,l); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } memcpy(s+i,str,l); sh->len += l; sh->free -= l; @@ -456,7 +494,29 @@ sds sdscatfmt(sds s, char const *fmt, ...) { { char buf[SDS_LLSTR_SIZE]; l = sdsll2str(buf,num); - if (sh->free < l) s = sdsMakeRoomFor(s,l); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } + memcpy(s+i,buf,l); + sh->len += l; + sh->free -= l; + i += l; + } + break; + case 'u': + case 'U': + if (next == 'u') + unum = va_arg(ap,unsigned int); + else + unum = va_arg(ap,unsigned long long); + { + char buf[SDS_LLSTR_SIZE]; + l = sdsull2str(buf,unum); + if (sh->free < l) { + s = sdsMakeRoomFor(s,l); + sh = (void*) (s-(sizeof(struct sdshdr))); + } memcpy(s+i,buf,l); sh->len += l; sh->free -= l; @@ -943,6 +1003,13 @@ int main(void) { memcmp(x,"--Hello Hi! World -9223372036854775808," "9223372036854775807--",60) == 0) + sdsfree(x); + x = sdsnew("--"); + x = sdscatfmt(x, "%u,%U--", UINT_MAX, ULLONG_MAX); + test_cond("sdscatfmt() seems working with unsigned numbers", + sdslen(x) == 35 && + memcmp(x,"--4294967295,18446744073709551615--",35) == 0) + sdsfree(x); x = sdsnew("xxciaoyyy"); sdstrim(x,"xy"); From 52189cb9c30a9b87400929a53c640d831d337141 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Apr 2014 16:41:38 +0200 Subject: [PATCH 1329/1752] Use sdscatfmt() in getClientInfoString() to make it faster. --- src/networking.c | 18 +++++++++--------- src/sds.h | 1 + 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/networking.c b/src/networking.c index f77f2a71500..95922ddec4a 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1258,23 +1258,23 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_READABLE) *p++ = 'r'; if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; - return sdscatprintf(sdsempty(), - "addr=%s fd=%d name=%s age=%ld idle=%ld flags=%s db=%d sub=%d psub=%d multi=%d qbuf=%lu qbuf-free=%lu obl=%lu oll=%lu omem=%lu events=%s cmd=%s", + return sdscatfmt(sdsempty(), + "addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", peerid, client->fd, client->name ? (char*)client->name->ptr : "", - (long)(server.unixtime - client->ctime), - (long)(server.unixtime - client->lastinteraction), + (long long)(server.unixtime - client->ctime), + (long long)(server.unixtime - client->lastinteraction), flags, client->db->id, (int) dictSize(client->pubsub_channels), (int) listLength(client->pubsub_patterns), (client->flags & REDIS_MULTI) ? client->mstate.count : -1, - (unsigned long) sdslen(client->querybuf), - (unsigned long) sdsavail(client->querybuf), - (unsigned long) client->bufpos, - (unsigned long) listLength(client->reply), - getClientOutputBufferMemoryUsage(client), + (unsigned long long) sdslen(client->querybuf), + (unsigned long long) sdsavail(client->querybuf), + (unsigned long long) client->bufpos, + (unsigned long long) listLength(client->reply), + (unsigned long long) getClientOutputBufferMemoryUsage(client), events, client->lastcmd ? client->lastcmd->name : "NULL"); } diff --git a/src/sds.h b/src/sds.h index 615c751cdcd..9a604021c27 100644 --- a/src/sds.h +++ b/src/sds.h @@ -76,6 +76,7 @@ sds sdscatprintf(sds s, const char *fmt, ...) sds sdscatprintf(sds s, const char *fmt, ...); #endif +sds sdscatfmt(sds s, char const *fmt, ...); sds sdstrim(sds s, const char *cset); void sdsrange(sds s, int start, int end); void sdsupdatelen(sds s); From d8d415e717415130b02ebe8fb07743a8ab69f3e8 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 28 Apr 2014 17:36:57 +0200 Subject: [PATCH 1330/1752] CLIENT LIST speedup via peerid caching + smart allocation. This commit adds peer ID caching in the client structure plus an API change and the use of sdsMakeRoomFor() in order to improve the reallocation pattern to generate the CLIENT LIST output. Both the changes account for a very significant speedup. --- src/aof.c | 1 + src/debug.c | 2 +- src/networking.c | 50 +++++++++++++++++++++++++++++------------------ src/redis.h | 5 +++-- src/replication.c | 10 +++++++--- 5 files changed, 43 insertions(+), 25 deletions(-) diff --git a/src/aof.c b/src/aof.c index 567d7ffcbf4..9409c49b948 100644 --- a/src/aof.c +++ b/src/aof.c @@ -506,6 +506,7 @@ struct redisClient *createFakeClient(void) { c->reply_bytes = 0; c->obuf_soft_limit_reached_time = 0; c->watched_keys = listCreate(); + c->peerid = NULL; listSetFreeMethod(c->reply,decrRefCountVoid); listSetDupMethod(c->reply,dupClientReplyValue); initClientMultiState(c); diff --git a/src/debug.c b/src/debug.c index 10ef2af6fee..b94959b7354 100644 --- a/src/debug.c +++ b/src/debug.c @@ -692,7 +692,7 @@ void logCurrentClient(void) { int j; redisLog(REDIS_WARNING, "--- CURRENT CLIENT INFO"); - client = getClientInfoString(cc); + client = catClientInfoString(sdsempty(),cc); redisLog(REDIS_WARNING,"client: %s", client); sdsfree(client); for (j = 0; j < cc->argc; j++) { diff --git a/src/networking.c b/src/networking.c index 95922ddec4a..d0b0ce0e5b7 100644 --- a/src/networking.c +++ b/src/networking.c @@ -103,6 +103,7 @@ redisClient *createClient(int fd) { c->watched_keys = listCreate(); c->pubsub_channels = dictCreate(&setDictType,NULL); c->pubsub_patterns = listCreate(); + c->peerid = NULL; listSetFreeMethod(c->pubsub_patterns,decrRefCountVoid); listSetMatchMethod(c->pubsub_patterns,listMatchObjects); if (fd != -1) listAddNodeTail(server.clients,c); @@ -745,6 +746,7 @@ void freeClient(redisClient *c) { if (c->name) decrRefCount(c->name); zfree(c->argv); freeClientMultiState(c); + sdsfree(c->peerid); zfree(c); } @@ -923,7 +925,7 @@ int processInlineBuffer(redisClient *c) { * multi bulk requests idempotent. */ static void setProtocolError(redisClient *c, int pos) { if (server.verbosity >= REDIS_VERBOSE) { - sds client = getClientInfoString(c); + sds client = catClientInfoString(sdsempty(),c); redisLog(REDIS_VERBOSE, "Protocol error from client: %s", client); sdsfree(client); @@ -1158,7 +1160,7 @@ void readQueryFromClient(aeEventLoop *el, int fd, void *privdata, int mask) { return; } if (sdslen(c->querybuf) > server.client_max_querybuf_len) { - sds ci = getClientInfoString(c), bytes = sdsempty(); + sds ci = catClientInfoString(sdsempty(),c), bytes = sdsempty(); bytes = sdscatrepr(bytes,c->querybuf,64); redisLog(REDIS_WARNING,"Closing client that reached max query buffer length: %s (qbuf initial bytes: %s)", ci, bytes); @@ -1189,7 +1191,7 @@ void getClientsMaxBuffers(unsigned long *longest_output_list, *biggest_input_buffer = bib; } -/* This is a helper function for getClientPeerId(). +/* This is a helper function for genClientPeerId(). * It writes the specified ip/port to "peerid" as a null termiated string * in the form ip:port if ip does not contain ":" itself, otherwise * [ip]:port format is used (for IPv6 addresses basically). */ @@ -1213,7 +1215,7 @@ void formatPeerId(char *peerid, size_t peerid_len, char *ip, int port) { * On failure the function still populates 'peerid' with the "?:0" string * in case you want to relax error checking or need to display something * anyway (see anetPeerToString implementation for more info). */ -int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { +int genClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { char ip[REDIS_IP_STR_LEN]; int port; @@ -1229,12 +1231,26 @@ int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len) { } } -/* Turn a Redis client into an sds string representing its state. */ -sds getClientInfoString(redisClient *client) { - char peerid[REDIS_PEER_ID_LEN], flags[16], events[3], *p; +/* This function returns the client peer id, by creating and caching it + * if client->perrid is NULL, otherwise returning the cached value. + * The Peer ID never changes during the life of the client, however it + * is expensive to compute. */ +char *getClientPeerId(redisClient *c) { + char peerid[REDIS_PEER_ID_LEN]; + + if (c->peerid == NULL) { + genClientPeerId(c,peerid,sizeof(peerid)); + c->peerid = sdsnew(peerid); + } + return c->peerid; +} + +/* Concatenate a string representing the state of a client in an human + * readable format, into the sds string 's'. */ +sds catClientInfoString(sds s, redisClient *client) { + char flags[16], events[3], *p; int emask; - getClientPeerId(client,peerid,sizeof(peerid)); p = flags; if (client->flags & REDIS_SLAVE) { if (client->flags & REDIS_MONITOR) @@ -1258,9 +1274,9 @@ sds getClientInfoString(redisClient *client) { if (emask & AE_READABLE) *p++ = 'r'; if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; - return sdscatfmt(sdsempty(), + return sdscatfmt(s, "addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", - peerid, + getClientPeerId(client), client->fd, client->name ? (char*)client->name->ptr : "", (long long)(server.unixtime - client->ctime), @@ -1285,14 +1301,11 @@ sds getAllClientsInfoString(void) { redisClient *client; sds o = sdsempty(); + o = sdsMakeRoomFor(o,200*listLength(server.clients)); listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - sds cs; - client = listNodeValue(ln); - cs = getClientInfoString(client); - o = sdscatsds(o,cs); - sdsfree(cs); + o = catClientInfoString(o,client); o = sdscatlen(o,"\n",1); } return o; @@ -1310,11 +1323,10 @@ void clientCommand(redisClient *c) { } else if (!strcasecmp(c->argv[1]->ptr,"kill") && c->argc == 3) { listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - char peerid[REDIS_PEER_ID_LEN]; + char *peerid; client = listNodeValue(ln); - if (getClientPeerId(client,peerid,sizeof(peerid)) == REDIS_ERR) - continue; + peerid = getClientPeerId(client); if (strcmp(peerid,c->argv[2]->ptr) == 0) { addReply(c,shared.ok); if (c == client) { @@ -1513,7 +1525,7 @@ void asyncCloseClientOnOutputBufferLimitReached(redisClient *c) { redisAssert(c->reply_bytes < ULONG_MAX-(1024*64)); if (c->reply_bytes == 0 || c->flags & REDIS_CLOSE_ASAP) return; if (checkClientOutputBufferLimits(c)) { - sds client = getClientInfoString(c); + sds client = catClientInfoString(sdsempty(),c); freeClientAsync(c); redisLog(REDIS_WARNING,"Client %s scheduled to be closed ASAP for overcoming of output buffer limits.", client); diff --git a/src/redis.h b/src/redis.h index f6deb797527..085638a44c6 100644 --- a/src/redis.h +++ b/src/redis.h @@ -486,6 +486,7 @@ typedef struct redisClient { list *watched_keys; /* Keys WATCHED for MULTI/EXEC CAS */ dict *pubsub_channels; /* channels a client is interested in (SUBSCRIBE) */ list *pubsub_patterns; /* patterns a client is interested in (SUBSCRIBE) */ + sds peerid; /* Cached peer ID. */ /* Response buffer */ int bufpos; @@ -928,8 +929,8 @@ void *dupClientReplyValue(void *o); void getClientsMaxBuffers(unsigned long *longest_output_list, unsigned long *biggest_input_buffer); void formatPeerId(char *peerid, size_t peerid_len, char *ip, int port); -int getClientPeerId(redisClient *client, char *peerid, size_t peerid_len); -sds getClientInfoString(redisClient *client); +char *getClientPeerId(redisClient *client); +sds catClientInfoString(sds s, redisClient *client); sds getAllClientsInfoString(void); void rewriteClientCommandVector(redisClient *c, int argc, ...); void rewriteClientCommandArgument(redisClient *c, int i, robj *newval); diff --git a/src/replication.c b/src/replication.c index 439dbba260f..ae128b0d98d 100644 --- a/src/replication.c +++ b/src/replication.c @@ -238,7 +238,6 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** int j; sds cmdrepr = sdsnew("+"); robj *cmdobj; - char peerid[REDIS_PEER_ID_LEN]; struct timeval tv; gettimeofday(&tv,NULL); @@ -248,8 +247,7 @@ void replicationFeedMonitors(redisClient *c, list *monitors, int dictid, robj ** } else if (c->flags & REDIS_UNIX_SOCKET) { cmdrepr = sdscatprintf(cmdrepr,"[%d unix:%s] ",dictid,server.unixsocket); } else { - getClientPeerId(c,peerid,sizeof(peerid)); - cmdrepr = sdscatprintf(cmdrepr,"[%d %s] ",dictid,peerid); + cmdrepr = sdscatprintf(cmdrepr,"[%d %s] ",dictid,getClientPeerId(c)); } for (j = 0; j < argc; j++) { @@ -1363,6 +1361,12 @@ void replicationCacheMaster(redisClient *c) { /* Set fd to -1 so that we can safely call freeClient(c) later. */ c->fd = -1; + /* Invalidate the Peer ID cache. */ + if (c->peerid) { + sdsfree(c->peerid); + c->peerid = NULL; + } + /* Caching the master happens instead of the actual freeClient() call, * so make sure to adjust the replication state. This function will * also set server.master to NULL. */ From 41a15205b67331c2caf98da124388c36cb528250 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Jun 2014 17:27:49 +0200 Subject: [PATCH 1331/1752] ROLE command added. The new ROLE command is designed in order to provide a client with informations about the replication in a fast and easy to use way compared to the INFO command where the same information is also available. --- src/redis.c | 1 + src/redis.h | 1 + src/replication.c | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+) diff --git a/src/redis.c b/src/redis.c index 5462067a412..8c675b3bdec 100644 --- a/src/redis.c +++ b/src/redis.c @@ -240,6 +240,7 @@ struct redisCommand redisCommandTable[] = { {"pttl",pttlCommand,2,"r",0,NULL,1,1,1,0,0}, {"persist",persistCommand,2,"w",0,NULL,1,1,1,0,0}, {"slaveof",slaveofCommand,3,"ast",0,NULL,0,0,0,0,0}, + {"role",roleCommand,1,"ast",0,NULL,0,0,0,0,0}, {"debug",debugCommand,-2,"as",0,NULL,0,0,0,0,0}, {"config",configCommand,-2,"art",0,NULL,0,0,0,0,0}, {"subscribe",subscribeCommand,-2,"rpslt",0,NULL,0,0,0,0,0}, diff --git a/src/redis.h b/src/redis.h index 085638a44c6..2e176912d17 100644 --- a/src/redis.h +++ b/src/redis.h @@ -1300,6 +1300,7 @@ void ttlCommand(redisClient *c); void pttlCommand(redisClient *c); void persistCommand(redisClient *c); void slaveofCommand(redisClient *c); +void roleCommand(redisClient *c); void debugCommand(redisClient *c); void msetCommand(redisClient *c); void msetnxCommand(redisClient *c); diff --git a/src/replication.c b/src/replication.c index ae128b0d98d..4f761698471 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1300,6 +1300,43 @@ void slaveofCommand(redisClient *c) { addReply(c,shared.ok); } +/* ROLE command: provide information about the role of the instance + * (master or slave) and additional information related to replication + * in an easy to process format. */ +void roleCommand(redisClient *c) { + if (server.masterhost == NULL) { + listIter li; + listNode *ln; + void *mbcount; + int slaves = 0; + + addReplyMultiBulkLen(c,3); + addReplyBulkCBuffer(c,"master",6); + addReplyLongLong(c,server.master_repl_offset); + mbcount = addDeferredMultiBulkLength(c); + listRewind(server.slaves,&li); + while((ln = listNext(&li))) { + redisClient *slave = ln->value; + char ip[REDIS_IP_STR_LEN]; + + if (anetPeerToString(slave->fd,ip,sizeof(ip),NULL) == -1) continue; + if (slave->replstate != REDIS_REPL_ONLINE) continue; + addReplyMultiBulkLen(c,3); + addReplyBulkCString(c,ip); + addReplyBulkLongLong(c,slave->slave_listening_port); + addReplyBulkLongLong(c,slave->repl_ack_off); + slaves++; + } + setDeferredMultiBulkLength(c,mbcount,slaves); + } else { + addReplyMultiBulkLen(c,4); + addReplyBulkCBuffer(c,"slave",5); + addReplyBulkCString(c,server.masterhost); + addReplyLongLong(c,server.masterport); + addReplyLongLong(c,server.master->reploff); + } +} + /* Send a REPLCONF ACK command to the master to inform it about the current * processed offset. If we are not connected with a master, the command has * no effects. */ From 8060de98f89e4114af2aed90650e397292ada937 Mon Sep 17 00:00:00 2001 From: antirez Date: Sat, 7 Jun 2014 17:38:16 +0200 Subject: [PATCH 1332/1752] ROLE output improved for slaves. Info about the replication state with the master added. --- src/replication.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/replication.c b/src/replication.c index 4f761698471..4e70efadc38 100644 --- a/src/replication.c +++ b/src/replication.c @@ -1329,11 +1329,23 @@ void roleCommand(redisClient *c) { } setDeferredMultiBulkLength(c,mbcount,slaves); } else { + char *slavestate = NULL; + addReplyMultiBulkLen(c,4); addReplyBulkCBuffer(c,"slave",5); addReplyBulkCString(c,server.masterhost); addReplyLongLong(c,server.masterport); - addReplyLongLong(c,server.master->reploff); + switch(server.repl_state) { + case REDIS_REPL_NONE: slavestate = "none"; break; + case REDIS_REPL_CONNECT: slavestate = "connect"; break; + case REDIS_REPL_CONNECTING: slavestate = "connecting"; break; + case REDIS_REPL_RECEIVE_PONG: /* see next */ + case REDIS_REPL_TRANSFER: slavestate = "sync"; break; + case REDIS_REPL_CONNECTED: slavestate = "connected"; break; + default: slavestate = "unknown"; break; + } + addReplyBulkCString(c,slavestate); + addReplyLongLong(c,server.master ? server.master->reploff : -1); } } From b6a26b52bf7887839957e4f1afb1cab7e5fc2d1a Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Jun 2014 10:43:05 +0200 Subject: [PATCH 1333/1752] Client types generalized. Because of output buffer limits Redis internals had this idea of type of clients: normal, pubsub, slave. It is possible to set different output buffer limits for the three kinds of clients. However all the macros and API were named after output buffer limit classes, while the idea of a client type is a generic one that can be reused. This commit does two things: 1) Rename the API and defines with more general names. 2) Change the class of clients executing the MONITOR command from "slave" to "normal". "2" is a good idea because you want to have very special settings for slaves, that are not a good idea for MONITOR clients that are instead normal clients even if they are conceptually slave-alike (since it is a push protocol). The backward-compatibility breakage resulting from "2" is considered to be minimal to care, since MONITOR is a debugging command, and because anyway this change is not going to break the format or the behavior, but just when a connection is closed on big output buffer issues. --- redis.conf | 4 ++-- src/config.c | 18 +++++++++--------- src/networking.c | 35 ++++++++++++++++++----------------- src/redis.c | 2 +- src/redis.h | 16 ++++++++-------- 5 files changed, 38 insertions(+), 37 deletions(-) diff --git a/redis.conf b/redis.conf index d5b51639c6c..ae85d5b2965 100644 --- a/redis.conf +++ b/redis.conf @@ -676,8 +676,8 @@ activerehashing yes # # The limit can be set differently for the three different classes of clients: # -# normal -> normal clients -# slave -> slave clients and MONITOR clients +# normal -> normal clients including MONITOR clients +# slave -> slave clients # pubsub -> clients subscribed to at least one pubsub channel or pattern # # The syntax of every client-output-buffer-limit directive is the following: diff --git a/src/config.c b/src/config.c index ece4a95f44b..6e08a41f201 100644 --- a/src/config.c +++ b/src/config.c @@ -49,7 +49,7 @@ static struct { {NULL, 0} }; -clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_LIMIT_NUM_CLASSES] = { +clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_TYPE_COUNT] = { {0, 0, 0}, /* normal */ {1024*1024*256, 1024*1024*64, 60}, /* slave */ {1024*1024*32, 1024*1024*8, 60} /* pubsub */ @@ -425,7 +425,7 @@ void loadServerConfigFromString(char *config) { } else if (!strcasecmp(argv[0],"client-output-buffer-limit") && argc == 5) { - int class = getClientLimitClassByName(argv[1]); + int class = getClientTypeByName(argv[1]); unsigned long long hard, soft; int soft_seconds; @@ -777,7 +777,7 @@ void configSetCommand(redisClient *c) { long val; if ((j % 4) == 0) { - if (getClientLimitClassByName(v[j]) == -1) { + if (getClientTypeByName(v[j]) == -1) { sdsfreesplitres(v,vlen); goto badfmt; } @@ -795,7 +795,7 @@ void configSetCommand(redisClient *c) { unsigned long long hard, soft; int soft_seconds; - class = getClientLimitClassByName(v[j]); + class = getClientTypeByName(v[j]); hard = strtoll(v[j+1],NULL,10); soft = strtoll(v[j+2],NULL,10); soft_seconds = strtoll(v[j+3],NULL,10); @@ -1058,13 +1058,13 @@ void configGetCommand(redisClient *c) { sds buf = sdsempty(); int j; - for (j = 0; j < REDIS_CLIENT_LIMIT_NUM_CLASSES; j++) { + for (j = 0; j < REDIS_CLIENT_TYPE_COUNT; j++) { buf = sdscatprintf(buf,"%s %llu %llu %ld", - getClientLimitClassName(j), + getClientTypeName(j), server.client_obuf_limits[j].hard_limit_bytes, server.client_obuf_limits[j].soft_limit_bytes, (long) server.client_obuf_limits[j].soft_limit_seconds); - if (j != REDIS_CLIENT_LIMIT_NUM_CLASSES-1) + if (j != REDIS_CLIENT_TYPE_COUNT-1) buf = sdscatlen(buf," ",1); } addReplyBulkCString(c,"client-output-buffer-limit"); @@ -1479,7 +1479,7 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state int j; char *option = "client-output-buffer-limit"; - for (j = 0; j < REDIS_CLIENT_LIMIT_NUM_CLASSES; j++) { + for (j = 0; j < REDIS_CLIENT_TYPE_COUNT; j++) { int force = (server.client_obuf_limits[j].hard_limit_bytes != clientBufferLimitsDefaults[j].hard_limit_bytes) || (server.client_obuf_limits[j].soft_limit_bytes != @@ -1495,7 +1495,7 @@ void rewriteConfigClientoutputbufferlimitOption(struct rewriteConfigState *state server.client_obuf_limits[j].soft_limit_bytes); line = sdscatprintf(sdsempty(),"%s %s %s %s %ld", - option, getClientLimitClassName(j), hard, soft, + option, getClientTypeName(j), hard, soft, (long) server.client_obuf_limits[j].soft_limit_seconds); rewriteConfigRewriteLine(state,option,line,force); } diff --git a/src/networking.c b/src/networking.c index d0b0ce0e5b7..960fd4605c6 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1447,30 +1447,31 @@ unsigned long getClientOutputBufferMemoryUsage(redisClient *c) { * classes of clients. * * The function will return one of the following: - * REDIS_CLIENT_LIMIT_CLASS_NORMAL -> Normal client - * REDIS_CLIENT_LIMIT_CLASS_SLAVE -> Slave or client executing MONITOR command - * REDIS_CLIENT_LIMIT_CLASS_PUBSUB -> Client subscribed to Pub/Sub channels + * REDIS_CLIENT_TYPE_NORMAL -> Normal client + * REDIS_CLIENT_TYPE_SLAVE -> Slave or client executing MONITOR command + * REDIS_CLIENT_TYPE_PUBSUB -> Client subscribed to Pub/Sub channels */ -int getClientLimitClass(redisClient *c) { - if (c->flags & REDIS_SLAVE) return REDIS_CLIENT_LIMIT_CLASS_SLAVE; +int getClientType(redisClient *c) { + if ((c->flags & REDIS_SLAVE) && !(c->flags & REDIS_MONITOR)) + return REDIS_CLIENT_TYPE_SLAVE; if (dictSize(c->pubsub_channels) || listLength(c->pubsub_patterns)) - return REDIS_CLIENT_LIMIT_CLASS_PUBSUB; - return REDIS_CLIENT_LIMIT_CLASS_NORMAL; + return REDIS_CLIENT_TYPE_PUBSUB; + return REDIS_CLIENT_TYPE_NORMAL; } -int getClientLimitClassByName(char *name) { - if (!strcasecmp(name,"normal")) return REDIS_CLIENT_LIMIT_CLASS_NORMAL; - else if (!strcasecmp(name,"slave")) return REDIS_CLIENT_LIMIT_CLASS_SLAVE; - else if (!strcasecmp(name,"pubsub")) return REDIS_CLIENT_LIMIT_CLASS_PUBSUB; +int getClientTypeByName(char *name) { + if (!strcasecmp(name,"normal")) return REDIS_CLIENT_TYPE_NORMAL; + else if (!strcasecmp(name,"slave")) return REDIS_CLIENT_TYPE_SLAVE; + else if (!strcasecmp(name,"pubsub")) return REDIS_CLIENT_TYPE_PUBSUB; else return -1; } -char *getClientLimitClassName(int class) { +char *getClientTypeName(int class) { switch(class) { - case REDIS_CLIENT_LIMIT_CLASS_NORMAL: return "normal"; - case REDIS_CLIENT_LIMIT_CLASS_SLAVE: return "slave"; - case REDIS_CLIENT_LIMIT_CLASS_PUBSUB: return "pubsub"; - default: return NULL; + case REDIS_CLIENT_TYPE_NORMAL: return "normal"; + case REDIS_CLIENT_TYPE_SLAVE: return "slave"; + case REDIS_CLIENT_TYPE_PUBSUB: return "pubsub"; + default: return NULL; } } @@ -1484,7 +1485,7 @@ int checkClientOutputBufferLimits(redisClient *c) { int soft = 0, hard = 0, class; unsigned long used_mem = getClientOutputBufferMemoryUsage(c); - class = getClientLimitClass(c); + class = getClientType(c); if (server.client_obuf_limits[class].hard_limit_bytes && used_mem >= server.client_obuf_limits[class].hard_limit_bytes) hard = 1; diff --git a/src/redis.c b/src/redis.c index 8c675b3bdec..9558938b8f0 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1400,7 +1400,7 @@ void initServerConfig() { server.repl_no_slaves_since = time(NULL); /* Client output buffer limits */ - for (j = 0; j < REDIS_CLIENT_LIMIT_NUM_CLASSES; j++) + for (j = 0; j < REDIS_CLIENT_TYPE_COUNT; j++) server.client_obuf_limits[j] = clientBufferLimitsDefaults[j]; /* Double constants initialization */ diff --git a/src/redis.h b/src/redis.h index 2e176912d17..5db01ac8573 100644 --- a/src/redis.h +++ b/src/redis.h @@ -239,10 +239,10 @@ /* Client classes for client limits, currently used only for * the max-client-output-buffer limit implementation. */ -#define REDIS_CLIENT_LIMIT_CLASS_NORMAL 0 -#define REDIS_CLIENT_LIMIT_CLASS_SLAVE 1 -#define REDIS_CLIENT_LIMIT_CLASS_PUBSUB 2 -#define REDIS_CLIENT_LIMIT_NUM_CLASSES 3 +#define REDIS_CLIENT_TYPE_NORMAL 0 /* Normal req-reply clients + MONITORs */ +#define REDIS_CLIENT_TYPE_SLAVE 1 /* Slaves. */ +#define REDIS_CLIENT_TYPE_PUBSUB 2 /* Clients subscribed to PubSub channels. */ +#define REDIS_CLIENT_TYPE_COUNT 3 /* Slave replication state - from the point of view of the slave. */ #define REDIS_REPL_NONE 0 /* No active replication */ @@ -541,7 +541,7 @@ typedef struct clientBufferLimitsConfig { time_t soft_limit_seconds; } clientBufferLimitsConfig; -extern clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_LIMIT_NUM_CLASSES]; +extern clientBufferLimitsConfig clientBufferLimitsDefaults[REDIS_CLIENT_TYPE_COUNT]; /* The redisOp structure defines a Redis Operation, that is an instance of * a command with an argument vector, database ID, propagation target @@ -645,7 +645,7 @@ struct redisServer { size_t client_max_querybuf_len; /* Limit for client query buffer length */ int dbnum; /* Total number of configured DBs */ int daemonize; /* True if running as a daemon */ - clientBufferLimitsConfig client_obuf_limits[REDIS_CLIENT_LIMIT_NUM_CLASSES]; + clientBufferLimitsConfig client_obuf_limits[REDIS_CLIENT_TYPE_COUNT]; /* AOF persistence */ int aof_state; /* REDIS_AOF_(ON|OFF|WAIT_REWRITE) */ int aof_fsync; /* Kind of fsync() policy */ @@ -937,8 +937,8 @@ void rewriteClientCommandArgument(redisClient *c, int i, robj *newval); unsigned long getClientOutputBufferMemoryUsage(redisClient *c); void freeClientsInAsyncFreeQueue(void); void asyncCloseClientOnOutputBufferLimitReached(redisClient *c); -int getClientLimitClassByName(char *name); -char *getClientLimitClassName(int class); +int getClientTypeByName(char *name); +char *getClientTypeName(int class); void flushSlavesOutputBuffers(void); void disconnectSlaves(void); int processEventsWhileBlocked(void); From cad13223f09d3c2541bdb6987ebdfa01b7edc33e Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Jun 2014 14:22:55 +0200 Subject: [PATCH 1334/1752] Assign an unique non-repeating ID to each new client. This will be used by CLIENT KILL and is also a good way to ensure a given client is still the same across CLIENT LIST calls. The output of CLIENT LIST was modified to include the new ID, but this change is considered to be backward compatible as the API does not imply you can do positional parsing, since each filed as a different name. --- src/networking.c | 4 +++- src/redis.c | 1 + src/redis.h | 4 +++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/networking.c b/src/networking.c index 960fd4605c6..736ca5cbffc 100644 --- a/src/networking.c +++ b/src/networking.c @@ -72,6 +72,7 @@ redisClient *createClient(int fd) { } selectDb(c,0); + c->id = server.next_client_id++; c->fd = fd; c->name = NULL; c->bufpos = 0; @@ -1275,7 +1276,8 @@ sds catClientInfoString(sds s, redisClient *client) { if (emask & AE_WRITABLE) *p++ = 'w'; *p = '\0'; return sdscatfmt(s, - "addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", + "id=%U addr=%s fd=%i name=%s age=%I idle=%I flags=%s db=%i sub=%i psub=%i multi=%i qbuf=%U qbuf-free=%U obl=%U oll=%U omem=%U events=%s cmd=%s", + (unsigned long long) client->id, getClientPeerId(client), client->fd, client->name ? (char*)client->name->ptr : "", diff --git a/src/redis.c b/src/redis.c index 9558938b8f0..1183f7bf7d5 100644 --- a/src/redis.c +++ b/src/redis.c @@ -1366,6 +1366,7 @@ void initServerConfig() { server.lua_time_limit = REDIS_LUA_TIME_LIMIT; server.lua_client = NULL; server.lua_timedout = 0; + server.next_client_id = 1; /* Client IDs, start from 1 .*/ server.loading_process_events_interval_bytes = (1024*1024*2); updateLRUClock(); diff --git a/src/redis.h b/src/redis.h index 5db01ac8573..4c9cf117cba 100644 --- a/src/redis.h +++ b/src/redis.h @@ -451,6 +451,7 @@ typedef struct readyList { /* With multiplexing we need to take per-client state. * Clients are taken in a liked list. */ typedef struct redisClient { + uint64_t id; /* Client incremental unique ID. */ int fd; redisDb *db; int dictid; @@ -602,7 +603,8 @@ struct redisServer { list *clients_to_close; /* Clients to close asynchronously */ list *slaves, *monitors; /* List of slaves and MONITORs */ redisClient *current_client; /* Current client, only used on crash report */ - char neterr[ANET_ERR_LEN]; /* Error buffer for anet.c */ + char neterr[ANET_ERR_LEN]; /* Error buffer for anet.c */ + uint64_t next_client_id; /* Next client unique ID. Incremental. */ /* RDB / AOF loading information */ int loading; /* We are loading data from disk if true */ off_t loading_total_bytes; From 09dc6dadc14ed0e358a19467ba5f3f821130b00d Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Jun 2014 14:24:28 +0200 Subject: [PATCH 1335/1752] New features for CLIENT KILL. --- src/networking.c | 71 +++++++++++++++++++++++++++++++++++++++--------- src/redis.h | 1 + 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/src/networking.c b/src/networking.c index 736ca5cbffc..8ced3d560c7 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1319,27 +1319,72 @@ void clientCommand(redisClient *c) { redisClient *client; if (!strcasecmp(c->argv[1]->ptr,"list") && c->argc == 2) { + /* CLIENT LIST */ sds o = getAllClientsInfoString(); addReplyBulkCBuffer(c,o,sdslen(o)); sdsfree(o); - } else if (!strcasecmp(c->argv[1]->ptr,"kill") && c->argc == 3) { + } else if (!strcasecmp(c->argv[1]->ptr,"kill")) { + /* CLIENT KILL + * CLIENT KILL */ + char *addr = NULL; + int type = -1; + uint64_t id = 0; + int killed = 0; + + /* Parse arguments. */ + if (c->argc == 3) { + addr = c->argv[2]->ptr; + } else if (c->argc == 4) { + if (!strcasecmp(c->argv[2]->ptr,"id")) { + long long tmp; + + if (getLongLongFromObjectOrReply(c,c->argv[3],&tmp,NULL) + != REDIS_OK) return; + id = tmp; + } else if (!strcasecmp(c->argv[2]->ptr,"type")) { + type = getClientTypeByName(c->argv[3]->ptr); + if (type == -1) { + addReplyErrorFormat(c,"Unknown client type '%s'", + (char*) c->argv[3]->ptr); + return; + } + } else if (!strcasecmp(c->argv[2]->ptr,"addr")) { + addr = c->argv[3]->ptr; + } else { + addReply(c,shared.syntaxerr); + return; + } + } else { + addReply(c,shared.syntaxerr); + return; + } + + /* Iterate clients killing all the matching clients. */ listRewind(server.clients,&li); while ((ln = listNext(&li)) != NULL) { - char *peerid; - client = listNodeValue(ln); - peerid = getClientPeerId(client); - if (strcmp(peerid,c->argv[2]->ptr) == 0) { - addReply(c,shared.ok); - if (c == client) { - client->flags |= REDIS_CLOSE_AFTER_REPLY; - } else { - freeClient(client); - } - return; + if (addr && strcmp(getClientPeerId(client),addr) != 0) continue; + if (type != -1 && getClientType(client) != type) continue; + if (id != 0 && client->id != id) continue; + + /* Kill it. */ + if (c == client) { + client->flags |= REDIS_CLOSE_AFTER_REPLY; + } else { + freeClient(client); } + killed++; + } + + /* Reply according to old/new format. */ + if (c->argc == 3) { + if (killed == 0) + addReplyError(c,"No such client"); + else + addReply(c,shared.ok); + } else { + addReplyLongLong(c,killed); } - addReplyError(c,"No such client"); } else if (!strcasecmp(c->argv[1]->ptr,"setname") && c->argc == 3) { int j, len = sdslen(c->argv[2]->ptr); char *p = c->argv[2]->ptr; diff --git a/src/redis.h b/src/redis.h index 4c9cf117cba..0fb40c12dfc 100644 --- a/src/redis.h +++ b/src/redis.h @@ -939,6 +939,7 @@ void rewriteClientCommandArgument(redisClient *c, int i, robj *newval); unsigned long getClientOutputBufferMemoryUsage(redisClient *c); void freeClientsInAsyncFreeQueue(void); void asyncCloseClientOnOutputBufferLimitReached(redisClient *c); +int getClientType(redisClient *c); int getClientTypeByName(char *name); char *getClientTypeName(int class); void flushSlavesOutputBuffers(void); From 61d9a73d87f4c536cfe214cf7b11c4a76398fd02 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Jun 2014 14:28:23 +0200 Subject: [PATCH 1336/1752] CLIENT KILL: fix closing link of the current client. --- src/networking.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/networking.c b/src/networking.c index 8ced3d560c7..acf8e65bba5 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1330,6 +1330,7 @@ void clientCommand(redisClient *c) { int type = -1; uint64_t id = 0; int killed = 0; + int close_this_client = 0; /* Parse arguments. */ if (c->argc == 3) { @@ -1369,7 +1370,7 @@ void clientCommand(redisClient *c) { /* Kill it. */ if (c == client) { - client->flags |= REDIS_CLOSE_AFTER_REPLY; + close_this_client = 1; } else { freeClient(client); } @@ -1385,6 +1386,10 @@ void clientCommand(redisClient *c) { } else { addReplyLongLong(c,killed); } + + /* If this client has to be closed, flag it as CLOSE_AFTER_REPLY + * only after we queued the reply to its output buffers. */ + if (close_this_client) c->flags |= REDIS_CLOSE_AFTER_REPLY; } else if (!strcasecmp(c->argv[1]->ptr,"setname") && c->argc == 3) { int j, len = sdslen(c->argv[2]->ptr); char *p = c->argv[2]->ptr; From 674194ad42a8b8960ee8b939da4ba400bb21ac49 Mon Sep 17 00:00:00 2001 From: antirez Date: Mon, 16 Jun 2014 14:50:15 +0200 Subject: [PATCH 1337/1752] CLIENT KILL API modified. Added a new SKIPME option that is true by default, that prevents the client sending the command to be killed, unless SKIPME NO is sent. --- src/networking.c | 60 +++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/src/networking.c b/src/networking.c index acf8e65bba5..63c15314cda 100644 --- a/src/networking.c +++ b/src/networking.c @@ -1325,35 +1325,52 @@ void clientCommand(redisClient *c) { sdsfree(o); } else if (!strcasecmp(c->argv[1]->ptr,"kill")) { /* CLIENT KILL - * CLIENT KILL */ + * CLIENT KILL