/*
**  Overview buffer and index method.
**
**  Originally written in 1999.
**  Various bug fixes, code and documentation improvements since then
**  in 1999-2010, 2013-2017, 2019-2026.
*/

/*
**  Buffindexed using shared memory on ovbuff by Sang-yong Suh.
**
**  During the discussions in inn-workers in 2003, Alex Kiernan found
**  that inn_lock_range() is not working for MMAPed file.  This explains
**  why buffindexed has long outstanding bugs such as "could not MMAP...".
**
**  This version corrects the file locking error by using shared memory.
**  The bitfield of each buffer file is loaded into memory, and is shared
**  by all programs such as innd, expireover, makehistory, and overchan.
**  The locking problem is handled by semaphore.
*/

#include "portable/system.h"

#include "portable/mmap.h"
#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <fcntl.h>
#include <limits.h>
#include <sys/stat.h>
#include <time.h>

#include "inn/fdflag.h"
#include "inn/innconf.h"
#include "inn/libinn.h"
#include "inn/messages.h"
#include "inn/ov.h"
#include "inn/paths.h"
#include "inn/storage.h"
#include "ovinterface.h"

/* Yes. I know that it violates INN coding style. However, this allows
 * me to compile this new version without reconfiguring INN.
 * If all goes well, shmem.c should go to $INN/lib, and shmem.h should
 * go to $INN/include.  (FIXME)
 */
#include "shmem.h"

#include "buffindexed.h"

#define OVBUFF_MAGIC        "ovbuff"
#define OVBUFF_VERSION      2

/*
** Because ovbuff bitfields are residing in memory, we don't have to
** do file write for each update.  Instead we'll do it at every
** OVBUFF_SYNC_COUNT updates.
*/
#define OVBUFF_SYNC_COUNT   (innconf->ovflushcount)

/* ovbuff header */
#define OVBUFFMASIZ         8
#define OVBUFFLASIZ         16
#define OVBUFFPASIZ         64

#define OVMAXCYCBUFFNAME    8

/*
** Default and minimum size of the header.  If the system default page size is
** larger than this, we'll use the system page size instead.  The page size in
** use is stored in the global hdr_pagesize variable.
*/
#define OV_HDR_MIN_PAGESIZE 16384

#define OV_BEFOREBITF       (1 * OV_BLOCKSIZE)
#define OV_BLOCKSIZE        8192
#define OV_FUDGE            1024
#define OV_OFFSET(block)    (block * (off_t) OV_BLOCKSIZE)

/* ovblock pointer */
typedef struct _OV {
    unsigned int blocknum;
    short index;
} OV;

/* ovbuff header */
typedef struct {
    char magic[OVBUFFMASIZ];
    char path[OVBUFFPASIZ];
    char indexa[OVBUFFLASIZ];   /* ASCII version of index */
    char lena[OVBUFFLASIZ];     /* ASCII version of len */
    char totala[OVBUFFLASIZ];   /* ASCII version of total */
    char useda[OVBUFFLASIZ];    /* ASCII version of used */
    char freea[OVBUFFLASIZ];    /* ASCII version of free */
    char updateda[OVBUFFLASIZ]; /* ASCII version of updated */
    /*
     * The following parts will be synced to the bitfield
     */
    int version;          /* magic version number */
    unsigned int freeblk; /* next free block number */
    unsigned int usedblk; /* number of used blocks */
} OVBUFFHEAD;

/* ovbuff info */
typedef struct _OVBUFF {
    unsigned int index;     /* ovbuff (partition or file) */
    char path[OVBUFFPASIZ]; /* Path to file */
    int fd;                 /* file descriptor for this ovbuff */
    off_t len;              /* Length of writable area, in bytes */
    off_t base;             /* Offset (relative to byte
                               0 of file) to base block */
    unsigned int freeblk;   /* next free block number no
                               freeblk left if equals totalblk */
    unsigned int totalblk;  /* number of total blocks */
    unsigned int usedblk;   /* number of used blocks */
    time_t updated;         /* Time of last update to header */
    void *bitfield;         /* Bitfield for ovbuff block in use */
    unsigned long dirty;    /* OVBUFFHEAD dirty count */
    struct _OVBUFF *next;   /* next ovbuff */
    int nextchunk;          /* next chunk */
    smcd_t *smc;            /* shared mem control data */
#ifdef OV_DEBUG
    struct ov_trace_array *trace;
#endif /* OV_DEBUG */
} OVBUFF;

typedef struct _OVINDEXHEAD {
    OV next;     /* next block */
    ARTNUM low;  /* lowest article number in the index */
    ARTNUM high; /* highest article number in the index */
} OVINDEXHEAD;

typedef struct _OVINDEX {
    ARTNUM artnum;         /* article number */
    unsigned int blocknum; /* overview data block number */
    short index;           /* overview data block index */
    TOKEN token;           /* token for this article */
    off_t offset;          /* offset from the top in the block */
    int len;               /* length of the data */
    time_t arrived;        /* arrived time of article */
    time_t expires;        /* expire time of article */
} OVINDEX;

#define OVINDEXMAX ((OV_BLOCKSIZE - sizeof(OVINDEXHEAD)) / sizeof(OVINDEX))

typedef struct _OVBLOCK {
    OVINDEXHEAD ovindexhead;     /* overview index header */
    OVINDEX ovindex[OVINDEXMAX]; /* overview index */
} OVBLOCK;

typedef struct _OVBLKS {
    OVBLOCK *ovblock;
    void *addr;
    int len;
    OV indexov;
} OVBLKS;

/* Data structure for specifying a location in the group index */
typedef struct {
    int recno; /* Record number in group index */
} GROUPLOC;

#ifdef OV_DEBUG
struct ov_trace {
    time_t occupied;
    time_t freed;
    GROUPLOC gloc;
};

#    define OV_TRACENUM 10
struct ov_trace_array {
    int max;
    int cur;
    struct ov_trace *ov_trace;
};

struct ov_name_table {
    char *name;
    int recno;
    struct ov_name_table *next;
};

static struct ov_name_table *name_table = NULL;
#endif /* OV_DEBUG */

#define GROUPHEADERHASHSIZE (16 * 1024)
#define GROUPHEADERMAGIC    (~(0xf1f0f33d))

typedef struct {
    int magic;
    GROUPLOC hash[GROUPHEADERHASHSIZE];
    GROUPLOC freelist;
} GROUPHEADER;

/* The group is matched based on the MD5 of the grouname. This may prove to
   be inadequate in the future, if so, the right thing to do is to is
   probably just to add a SHA1 hash into here also.  We get a really nice
   benefit from this being fixed length, we should try to keep it that way.
*/
typedef struct {
    HASH hash;          /* MD5 hash of the group name */
    HASH alias;         /* If not empty then this is the hash of the
                           group that this group is an alias for */
    ARTNUM high;        /* High water mark in group */
    ARTNUM low;         /* Low water mark in group */
    int count;          /* Number of articles in group */
    int flag;           /* Posting/Moderation Status */
    time_t expired;     /* When last expiry */
    time_t deleted;     /* When this was deleted, 0 otherwise */
    GROUPLOC next;      /* Next block in this chain */
    OV baseindex;       /* base index buff */
    OV curindex;        /* current index buff */
    int curindexoffset; /* current index offset for this ovbuff */
    ARTNUM curhigh;     /* High water mark in group */
    ARTNUM curlow;      /* Low water mark in group */
    OV curdata;         /* current offset for this ovbuff */
    off_t curoffset;    /* current offset for this ovbuff */
} GROUPENTRY;

typedef struct _GIBLIST {
    OV ov;
    struct _GIBLIST *next;
} GIBLIST;

typedef struct _GDB {
    OV datablk;
    void *addr;
    void *data;
    int len;
    bool mmapped;
    struct _GDB *next;
} GROUPDATABLOCK;

typedef struct {
    char *group;
    ARTNUM lo;
    ARTNUM hi;
    int cur;
    bool needov;
    GROUPLOC gloc;
    int count;
    GROUPDATABLOCK gdb; /* used for caching current block */
} OVSEARCH;

#define GROUPDATAHASHSIZE 25

static GROUPDATABLOCK *groupdatablock[GROUPDATAHASHSIZE];

typedef enum {
    PREPEND_BLK,
    APPEND_BLK
} ADDINDEX;
typedef enum {
    SRCH_FRWD,
    SRCH_BKWD
} SRCH;

#define _PATH_OVBUFFCONFIG "buffindexed.conf"

static long pagesize = 0;
static long hdr_pagesize = OV_HDR_MIN_PAGESIZE;
static OVBUFF *ovbufftab = NULL;
static OVBUFF *ovbuffnext = NULL;
static int GROUPfd;
static GROUPHEADER *GROUPheader = NULL;
static GROUPENTRY *GROUPentries = NULL;
static int GROUPcount = 0;
static size_t GROUPmapsize = 0;
static GROUPLOC GROUPemptyloc = {-1};
#define NULLINDEX (-1)
static OV ovnull = {0, NULLINDEX};
typedef unsigned long ULONG;
static ULONG onarray[64], offarray[64];
static int longsize = sizeof(long);
static bool Nospace;
static bool Needunlink;
static bool Cutofflow;
static bool Cache;
static OVSEARCH *Cachesearch;

static int ovbuffmode;

static GROUPLOC GROUPnewnode(void);
static int GROUPmappingprot(int mode);
static bool GROUPremapifneeded(GROUPLOC loc, bool hash_locked);
static void GROUPLOCclear(GROUPLOC *loc);
static bool GROUPLOCempty(GROUPLOC loc);
static bool GROUPlockhash(enum inn_locktype type);
static bool GROUPlock(GROUPLOC gloc, enum inn_locktype type);
static bool GROUPentrycount(off_t size, int *count);
static bool GROUPfilesize(int count, off_t *size);
static bool GROUPexpand(int mode);
static void *ovopensearch(const char *group, ARTNUM low, ARTNUM high,
                          bool needov);
static void ovclosesearch(void *handle, bool freeblock);
static OVINDEX *Gib;
static GIBLIST *Giblist;
static int Gibcount;

#ifdef MMAP_MISSES_WRITES
#    define PWRITE(fd, buf, nbyte, offset) mmapwrite(fd, buf, nbyte, offset)
#else
#    define PWRITE(fd, buf, nbyte, offset) pwrite(fd, buf, nbyte, offset)
#endif

#ifdef MMAP_MISSES_WRITES
/* With HP/UX, you definitely do not want to mix mmap-accesses of
   a file with read()s and write()s of the same file */
static off_t
mmapwrite(int fd, void *buf, off_t nbyte, off_t offset)
{
    int pagefudge, len;
    off_t mmapoffset;
    void *addr;

    pagefudge = offset % pagesize;
    mmapoffset = offset - pagefudge;
    len = pagefudge + nbyte;

    if ((addr = mmap(NULL, len, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
                     mmapoffset))
        == MAP_FAILED) {
        return -1;
    }
    memcpy((char *) addr + pagefudge, buf, nbyte);
    munmap(addr, len);
    return nbyte;
}
#endif /* MMAP_MISSES_WRITES */

static bool
ovparse_part_line(char *l)
{
    char *p;
    struct stat sb;
    off_t len, base;
    int tonextblock;
    OVBUFF *ovbuff, *tmp = ovbufftab;

    /* ovbuff partition name */
    if ((p = strchr(l, ':')) == NULL || p - l <= 0
        || p - l > OVMAXCYCBUFFNAME - 1) {
        warn("buffindexed: bad index in line '%s'", l);
        return false;
    }
    *p = '\0';
    ovbuff = xmalloc(sizeof(OVBUFF));
    ovbuff->index = strtoul(l, NULL, 10);
    for (; tmp != (OVBUFF *) NULL; tmp = tmp->next) {
        if (tmp->index == ovbuff->index) {
            warn("buffindexed: duplicate index in line '%s'", l);
            free(ovbuff);
            return false;
        }
    }
    l = ++p;

    /* Path to ovbuff partition */
    if ((p = strchr(l, ':')) == NULL || p - l <= 0
        || p - l > OVBUFFPASIZ - 1) {
        warn("buffindexed: bad pathname in line '%s'", l);
        free(ovbuff);
        return false;
    }
    *p = '\0';
    memset(ovbuff->path, '\0', OVBUFFPASIZ);
    strlcpy(ovbuff->path, l, OVBUFFPASIZ);
    if (stat(ovbuff->path, &sb) < 0) {
        warn("buffindexed: file '%s' does not exist, ignoring '%u'",
             ovbuff->path, ovbuff->index);
        free(ovbuff);
        return false;
    }
    l = ++p;

    /* Length/size of symbolic partition in KB */
    len = strtoul(l, NULL, 10) * (off_t) 1024;
    /*
    ** The minimum article offset will be the size of the bitfield itself,
    ** len / (blocksize * 8), plus however many additional blocks the
    ** *OVBUFFHEAD external header occupies... then round up to the next block.
    */
    base = len / (OV_BLOCKSIZE * 8) + OV_BEFOREBITF;
    tonextblock = hdr_pagesize - (base & (hdr_pagesize - 1));
    ovbuff->base = base + tonextblock;
    if (S_ISREG(sb.st_mode)
        && (len != sb.st_size || ovbuff->base > sb.st_size)) {
        if (len != sb.st_size)
            notice("buffindexed: length mismatch '%lu' for index '%u'"
                   " (%lu bytes)",
                   (unsigned long) len, ovbuff->index,
                   (unsigned long) sb.st_size);
        if (ovbuff->base > sb.st_size)
            notice("buffindexed: length must be at least '%lu' for index '%u'"
                   " (%lu bytes)",
                   (unsigned long) ovbuff->base, ovbuff->index,
                   (unsigned long) sb.st_size);
        free(ovbuff);
        return false;
    }
    ovbuff->len = len;
    ovbuff->fd = -1;
    ovbuff->next = (OVBUFF *) NULL;
    ovbuff->dirty = 0;
    ovbuff->bitfield = NULL;
    ovbuff->nextchunk = 1;

    if (ovbufftab == (OVBUFF *) NULL)
        ovbufftab = ovbuff;
    else {
        for (tmp = ovbufftab; tmp->next != (OVBUFF *) NULL; tmp = tmp->next)
            ;
        tmp->next = ovbuff;
    }
    return true;
}

/*
** ovbuffread_config() -- Read the overview partition/file configuration file.
*/

static bool
ovbuffread_config(void)
{
    char *path, *config, *from, *to, **ctab = (char **) NULL;
    int ctab_free = 0; /* Index to next free slot in ctab */
    int ctab_i;

    path = concatpath(innconf->pathetc, _PATH_OVBUFFCONFIG);
    config = ReadInFile(path, NULL);
    if (config == NULL) {
        warn("buffindexed: cannot read %s", path);
        free(config);
        free(path);
        return false;
    }
    free(path);
    for (from = to = config; *from;) {
        if (*from == '#') { /* Comment line? */
            while (*from && *from != '\n')
                from++; /* Skip past it */
            from++;
            continue; /* Back to top of loop */
        }
        if (*from == '\n') { /* End or just a blank line? */
            from++;
            continue; /* Back to top of loop */
        }
        if (ctab_free == 0)
            ctab = xmalloc(sizeof(char *));
        else
            ctab = xrealloc(ctab, (ctab_free + 1) * sizeof(char *));
        /* If we're here, we've got the beginning of a real entry */
        ctab[ctab_free++] = to = from;
        while (1) {
            if (*from && *from == '\\' && *(from + 1) == '\n') {
                from += 2; /* Skip past backslash+newline */
                while (*from && isspace((unsigned char) *from))
                    from++;
                continue;
            }
            if (*from && *from != '\n')
                *to++ = *from++;
            if (*from == '\n') {
                *to++ = '\0';
                from++;
                break;
            }
            if (!*from)
                break;
        }
    }
    for (ctab_i = 0; ctab_i < ctab_free; ctab_i++) {
        if (!ovparse_part_line(ctab[ctab_i])) {
            free(config);
            free(ctab);
            return false;
        }
    }
    free(config);
    free(ctab);
    if (ovbufftab == (OVBUFF *) NULL) {
        warn("buffindexed: no buffindexed defined");
        return false;
    }
    return true;
}

static char hextbl[] = {'0', '1', '2', '3', '4', '5', '6', '7',
                        '8', '9', 'a', 'b', 'c', 'd', 'e', 'f'};

static char *
offt2hex(off_t offset, bool leadingzeros)
{
    static char buf[24];
    char *p;

    if (sizeof(off_t) <= sizeof(unsigned long)) {
        snprintf(buf, sizeof(buf), (leadingzeros) ? "%016lx" : "%lx",
                 (unsigned long) offset);
    } else {
        int i;

        for (i = 0; i < OVBUFFLASIZ; i++)
            buf[i] = '0'; /* Pad with zeros to start */
        for (i = OVBUFFLASIZ - 1; i >= 0; i--) {
            buf[i] = hextbl[offset & 0xf];
            offset >>= 4;
        }
    }
    if (!leadingzeros) {
        for (p = buf; *p == '0'; p++)
            ;
        if (*p != '\0')
            return p;
        else
            return p - 1; /* We converted a "0" and then bypassed all the zeros
                           */
    } else
        return buf;
}

static off_t
hex2offt(char *hex)
{
    if (sizeof(off_t) <= 4) {
        unsigned long rpofft;

        sscanf(hex, "%lx", &rpofft);
        return rpofft;
    } else {
        char diff;
        off_t n = 0;

        for (; *hex != '\0'; hex++) {
            if (*hex >= '0' && *hex <= '9')
                diff = '0';
            else if (*hex >= 'a' && *hex <= 'f')
                diff = 'a' - 10;
            else if (*hex >= 'A' && *hex <= 'F')
                diff = 'A' - 10;
            else {
                /*
                ** We used to have a syslog() message here, but the case
                ** where we land here because of a ":" happens, er, often.
                */
                break;
            }
            n += (*hex - diff);
            if (isalnum((unsigned char) *(hex + 1)))
                n <<= 4;
        }
        return n;
    }
}

static void
ovreadhead(OVBUFF *ovbuff)
{
    OVBUFFHEAD *x = (OVBUFFHEAD *) ovbuff->bitfield;
    ovbuff->freeblk = x->freeblk;
    ovbuff->usedblk = x->usedblk;
    return;
}

static void
ovflushhead(OVBUFF *ovbuff)
{
    OVBUFFHEAD rpx;

    /* skip time consuming data conversion and write call */
    if (ovbuff->dirty < OVBUFF_SYNC_COUNT) {
        OVBUFFHEAD *x = (OVBUFFHEAD *) ovbuff->bitfield;
        x->freeblk = ovbuff->freeblk;
        x->usedblk = ovbuff->usedblk;
        return;
    }

    memset(&rpx, 0, sizeof(OVBUFFHEAD));
    ovbuff->updated = time(NULL);
    /* Don't use sprintf() or strlcat() directly...
     * The terminating '\0' causes grief. */
#if __GNUC__ > 7
#    pragma GCC diagnostic ignored "-Wstringop-truncation"
#endif
    strncpy(rpx.magic, OVBUFF_MAGIC, strlen(OVBUFF_MAGIC));
    strncpy(rpx.path, ovbuff->path, OVBUFFPASIZ);
    strncpy(rpx.indexa, offt2hex(ovbuff->index, true), OVBUFFLASIZ);
    strncpy(rpx.lena, offt2hex(ovbuff->len, true), OVBUFFLASIZ);
    strncpy(rpx.totala, offt2hex(ovbuff->totalblk, true), OVBUFFLASIZ);
    strncpy(rpx.useda, offt2hex(ovbuff->usedblk, true), OVBUFFLASIZ);
    strncpy(rpx.freea, offt2hex(ovbuff->freeblk, true), OVBUFFLASIZ);
    strncpy(rpx.updateda, offt2hex(ovbuff->updated, true), OVBUFFLASIZ);
#if __GNUC__ > 7
#    pragma GCC diagnostic warning "-Wstringop-truncation"
#endif
    rpx.version = OVBUFF_VERSION;
    rpx.freeblk = ovbuff->freeblk;
    rpx.usedblk = ovbuff->usedblk;
    memcpy(ovbuff->bitfield, &rpx, sizeof(OVBUFFHEAD));

    if (pwrite(ovbuff->fd, ovbuff->bitfield, ovbuff->base, 0) != ovbuff->base)
        syswarn("buffindexed: ovflushhead: cant flush on %s", ovbuff->path);
    ovbuff->dirty = 0;
    return;
}

static bool
ovlock(OVBUFF *ovbuff, enum inn_locktype type)
{
    int ret;
    smcd_t *smc = ovbuff->smc;

    if (type == INN_LOCK_WRITE) {
        ret = smcGetExclusiveLock(smc);
        smc->locktype = (int) INN_LOCK_WRITE;
    } else if (type == INN_LOCK_READ) {
        ret = smcGetSharedLock(smc);
        smc->locktype = (int) INN_LOCK_READ;
    } else if (smc->locktype == (int) INN_LOCK_WRITE) {
        ret = smcReleaseExclusiveLock(smc);
    } else {
        ret = smcReleaseSharedLock(smc);
    }
    return (ret == 0);
}

static bool
ovbuffinit_disks(void)
{
    OVBUFF *ovbuff = ovbufftab;
    char buf[64];
    OVBUFFHEAD *rpx, dpx;
    int fd;
    unsigned int i;
    off_t tmpo;
    smcd_t *smc;
    static bool atexit_registered = false;

    /*
     * Register the exit callback to sync the bitfield to the disk
     */
    if (!atexit_registered) {
        atexit(buffindexed_close);
        atexit_registered = true;
    }

    /*
    ** Discover the state of our ovbuffs.  If any of them are in icky shape,
    ** duck shamelessly & return false.
    */
    for (; ovbuff != (OVBUFF *) NULL; ovbuff = ovbuff->next) {
        if (ovbuff->fd < 0) {
            if ((fd = open(ovbuff->path,
                           (ovbuffmode & OV_WRITE) ? O_RDWR : O_RDONLY))
                < 0) {
                syswarn("buffindexed: ERROR opening '%s'", ovbuff->path);
                return false;
            } else {
                fdflag_close_exec(fd, true);
                ovbuff->fd = fd;
            }
        }

        /* get shared memory buffer */
        smc = smcGetShmemBuffer(ovbuff->path, ovbuff->base);
        if (!smc) {
            /* No shared memory exists, create one. */
            smc = smcCreateShmemBuffer(ovbuff->path, ovbuff->base);
            if (!smc) {
                syswarn("buffindexed: ovinitdisks: cant create shmem for %s "
                        "len %lu",
                        ovbuff->path, (unsigned long) ovbuff->base);
                return false;
            }
        }

        ovbuff->smc = smc;
        ovbuff->bitfield = smc->addr;
        rpx = (OVBUFFHEAD *) ovbuff->bitfield;

        /* lock the buffer */
        ovlock(ovbuff,
               (ovbuffmode & OV_WRITE) ? INN_LOCK_WRITE : INN_LOCK_READ);

        if (pread(ovbuff->fd, &dpx, sizeof(OVBUFFHEAD), 0) < 0) {
            syswarn("buffindexed: cant read from %s", ovbuff->path);
            ovlock(ovbuff, INN_LOCK_UNLOCK);
            return false;
        }

        /*
         * check validity of the disk data
         */
        if (strncmp(dpx.magic, OVBUFF_MAGIC, strlen(OVBUFF_MAGIC)) == 0
            && strncmp(dpx.path, ovbuff->path, OVBUFFPASIZ) == 0) {
            strncpy(buf, dpx.indexa, OVBUFFLASIZ);
            buf[OVBUFFLASIZ] = '\0';
            i = hex2offt(buf);
            if (i != ovbuff->index) {
                warn("buffindexed: Mismatch: index '%u' for buffindexed %s", i,
                     ovbuff->path);
                ovlock(ovbuff, INN_LOCK_UNLOCK);
                return false;
            }
            strncpy(buf, dpx.lena, OVBUFFLASIZ);
            buf[OVBUFFLASIZ] = '\0';
            tmpo = hex2offt(buf);
            if (tmpo != ovbuff->len) {
                warn("buffindexed: Mismatch: read 0x%s length for buffindexed "
                     "%s",
                     offt2hex(tmpo, false), ovbuff->path);
                ovlock(ovbuff, INN_LOCK_UNLOCK);
                return false;
            }

            /*
             * compare shared memory with disk data.
             */
            if (strncmp(dpx.magic, rpx->magic, strlen(OVBUFF_MAGIC)) != 0
                || strncmp(dpx.path, rpx->path, OVBUFFPASIZ) != 0
                || strncmp(dpx.indexa, rpx->indexa, OVBUFFLASIZ) != 0
                || strncmp(dpx.lena, rpx->lena, OVBUFFLASIZ) != 0) {
                /*
                 * Load shared memory with disk data.
                 */
                if (pread(ovbuff->fd, rpx, ovbuff->base, 0) < 0) {
                    syswarn("buffindexed: cant read from %s", ovbuff->path);
                    ovlock(ovbuff, INN_LOCK_UNLOCK);
                    return false;
                }
            }
            strncpy(buf, dpx.totala, OVBUFFLASIZ);
            buf[OVBUFFLASIZ] = '\0';
            ovbuff->totalblk = hex2offt(buf);
            if (ovbuff->base > ovbuff->len
                || (off_t) ovbuff->totalblk
                       != (ovbuff->len - ovbuff->base) / OV_BLOCKSIZE) {
                warn("buffindexed: invalid block count for %s", ovbuff->path);
                ovlock(ovbuff, INN_LOCK_UNLOCK);
                return false;
            }

            if (rpx->version == 0) {
                /* no binary data available. use character data */
                strncpy(buf, rpx->useda, OVBUFFLASIZ);
                buf[OVBUFFLASIZ] = '\0';
                ovbuff->usedblk = hex2offt(buf);
                strncpy(buf, rpx->freea, OVBUFFLASIZ);
                buf[OVBUFFLASIZ] = '\0';
                ovbuff->freeblk = hex2offt(buf);
            } else {
                /* use binary data. The first reason is the speed.
                   and the second reason is the other partner is not
                   synced.
                 */
                ovbuff->usedblk = rpx->usedblk;
                ovbuff->freeblk = rpx->freeblk;
            }
            if (ovbuff->usedblk > ovbuff->totalblk
                || ovbuff->freeblk > ovbuff->totalblk) {
                warn("buffindexed: invalid block usage for %s", ovbuff->path);
                ovlock(ovbuff, INN_LOCK_UNLOCK);
                return false;
            }
            Needunlink = false;
        } else {
            /*
             * Initialize the contents of the shared memory
             */
            memset(rpx, 0, ovbuff->base);

            ovbuff->totalblk = (ovbuff->len - ovbuff->base) / OV_BLOCKSIZE;
            if (ovbuff->totalblk < 1) {
                warn("buffindexed: too small length '%lu' for buffindexed %s",
                     (unsigned long) ovbuff->len, ovbuff->path);
                ovlock(ovbuff, INN_LOCK_UNLOCK);
                return false;
            }
            ovbuff->usedblk = 0;
            ovbuff->freeblk = 0;
            ovbuff->updated = 0;
            /* Force a flush. */
            ovbuff->dirty = OVBUFF_SYNC_COUNT + 1;
            notice("buffindexed: no magic cookie found for ovbuff %u, "
                   "initializing",
                   ovbuff->index);
            ovflushhead(ovbuff);
        }
#ifdef OV_DEBUG
        ovbuff->trace =
            xcalloc(ovbuff->totalblk, sizeof(struct ov_trace_array));
#endif /* OV_DEBUG */
        ovlock(ovbuff, INN_LOCK_UNLOCK);
    }
    return true;
}

static int
ovusedblock(OVBUFF *ovbuff, int blocknum, bool set_operation, bool setbitvalue)
{
    off_t longoffset;
    int bitoffset; /* From the 'left' side of the long */
    ULONG bitlong, mask;

    longoffset = blocknum / (sizeof(long) * 8);
    bitoffset = blocknum % (sizeof(long) * 8);
    bitlong = *((ULONG *) ovbuff->bitfield + (OV_BEFOREBITF / sizeof(long))
                + longoffset);
    if (set_operation) {
        if (setbitvalue) {
            mask = onarray[bitoffset];
            bitlong |= mask;
        } else {
            mask = offarray[bitoffset];
            bitlong &= mask;
        }
        *((ULONG *) ovbuff->bitfield + (OV_BEFOREBITF / sizeof(long))
          + longoffset) = bitlong;
        return 2; /* XXX Clean up return semantics */
    }
    /* It's a read operation */
    mask = onarray[bitoffset];
    /* return bitlong & mask; doesn't work if sizeof(ulong) > sizeof(int) */
    if (bitlong & mask)
        return 1;
    else
        return 0;
}

static void
ovnextblock(OVBUFF *ovbuff)
{
    int i, j, last, lastbit, left;
    ULONG *table;

    last = ovbuff->totalblk / (sizeof(long) * 8);
    if ((left = ovbuff->totalblk % (sizeof(long) * 8)) != 0) {
        last++;
    }

    table = ((ULONG *) ovbuff->bitfield + (OV_BEFOREBITF / sizeof(long)));

    /* For tighter placement look for unused block from the beginning on
     * every run. */
    for (i = 0; i < last; i++) {
        if ((table[i] ^ ~0) != 0)
            break;
    }
    if (i == last) {
        ovbuff->freeblk = ovbuff->totalblk;
        return;
    }

    if ((i - 1) >= 0 && (last - 1 == i) && left != 0) {
        lastbit = left;
    } else {
        lastbit = sizeof(long) * 8;
    }
    for (j = 0; j < lastbit; j++) {
        if ((table[i] & onarray[j]) == 0)
            break;
    }
    if (j == lastbit) {
        ovbuff->freeblk = ovbuff->totalblk;
        return;
    }
    ovbuff->freeblk = i * sizeof(long) * 8 + j;
    ovbuff->nextchunk = i + 1;
    if (i == last)
        ovbuff->nextchunk = 0;
    return;
}

static OVBUFF *
getovbuff(OV ov)
{
    OVBUFF *ovbuff = ovbufftab;
    for (; ovbuff != NULL; ovbuff = ovbuff->next) {
        if (ovbuff->index == (unsigned int) ov.index)
            return ovbuff;
    }
    return NULL;
}

static bool
ovblockvalid(const OVBUFF *ovbuff, unsigned int blocknum)
{
    off_t blocks;

    if (ovbuff->base > ovbuff->len)
        return false;
    blocks = (ovbuff->len - ovbuff->base) / OV_BLOCKSIZE;
    return blocknum < (unsigned long) blocks;
}

static bool
ovindexspanvalid(const OVINDEX *index)
{
    return index->offset >= 0 && index->len >= 0
           && index->offset <= OV_BLOCKSIZE
           && index->len <= OV_BLOCKSIZE - index->offset;
}

#ifdef OV_DEBUG
static OV
ovblocknew(GROUPENTRY *ge)
{
#else
static OV
ovblocknew(void)
{
#endif /* OV_DEBUG */
    OVBUFF *ovbuff;
    OV ov;
    bool done = false;
#ifdef OV_DEBUG
    int recno;
    struct ov_trace_array *trace;
#endif /* OV_DEBUG */

    if (ovbuffnext == NULL)
        ovbuffnext = ovbufftab;

    /*
     * We will try to recover broken overview possibly due to unsync.
     * The recovering is inactive for OV_DEBUG mode.
     */

retry:
    for (ovbuff = ovbuffnext; ovbuff != (OVBUFF *) NULL;
         ovbuff = ovbuff->next) {
        ovlock(ovbuff, INN_LOCK_WRITE);
        ovreadhead(ovbuff);
        if (ovbuff->totalblk != ovbuff->usedblk
            && ovbuff->freeblk == ovbuff->totalblk) {
            ovnextblock(ovbuff);
        }
        if (ovbuff->totalblk == ovbuff->usedblk
            || ovbuff->freeblk == ovbuff->totalblk) {
            /* no space left for this ovbuff */
            ovlock(ovbuff, INN_LOCK_UNLOCK);
            continue;
        }
        break;
    }
    if (ovbuff == NULL) {
        for (ovbuff = ovbufftab; ovbuff != ovbuffnext; ovbuff = ovbuff->next) {
            ovlock(ovbuff, INN_LOCK_WRITE);
            ovreadhead(ovbuff);
            if (ovbuff->totalblk == ovbuff->usedblk
                || ovbuff->freeblk == ovbuff->totalblk) {
                /* no space left for this ovbuff */
                ovlock(ovbuff, INN_LOCK_UNLOCK);
                continue;
            }
            break;
        }
        if (ovbuff == ovbuffnext) {
            Nospace = true;
            return ovnull;
        }
    }
#ifdef OV_DEBUG
    recno = ((char *) ge - (char *) &GROUPentries[0]) / sizeof(GROUPENTRY);
    if (ovusedblock(ovbuff, ovbuff->freeblk, false, true)) {
        warn("buffindexed: 0x%08x trying to occupy new block(%d, %d), but "
             "already occupied",
             recno, ovbuff->index, ovbuff->freeblk);
        buffindexed_close();
        abort();
    }
    trace = &ovbuff->trace[ovbuff->freeblk];
    if (trace->ov_trace == NULL) {
        trace->ov_trace = xcalloc(OV_TRACENUM, sizeof(struct ov_trace));
        trace->max = OV_TRACENUM;
    } else if (trace->cur + 1 == trace->max) {
        trace->max += OV_TRACENUM;
        trace->ov_trace =
            xrealloc(trace->ov_trace, trace->max * sizeof(struct ov_trace));
        memset(&trace->ov_trace[trace->cur], '\0',
               sizeof(struct ov_trace) * (trace->max - trace->cur));
    }
    if (trace->ov_trace[trace->cur].occupied != 0) {
        trace->cur++;
    }
    trace->ov_trace[trace->cur].gloc.recno = recno;
    trace->ov_trace[trace->cur].occupied = time(NULL);
#endif /* OV_DEBUG */

    ov.index = ovbuff->index;
    ov.blocknum = ovbuff->freeblk;

#ifndef OV_DEBUG
    if (ovusedblock(ovbuff, ovbuff->freeblk, false, true)) {
        notice("buffindexed: fixing invalid free block(%u, %u).",
               ovbuff->index, ovbuff->freeblk);
    } else
        done = true;
#endif /* OV_DEBUG */

    /* mark it as allocated */
    ovusedblock(ovbuff, ov.blocknum, true, true);

    ovnextblock(ovbuff);
    ovbuff->usedblk++;
    ovbuff->dirty++;
    ovflushhead(ovbuff);
    ovlock(ovbuff, INN_LOCK_UNLOCK);
    ovbuffnext = ovbuff->next;
    if (ovbuffnext == NULL)
        ovbuffnext = ovbufftab;

    if (!done)
        goto retry;

    return ov;
}

#ifdef OV_DEBUG
static void
ovblockfree(OV ov, GROUPENTRY *ge)
{
#else
static void
ovblockfree(OV ov)
{
#endif /* OV_DEBUG */
    OVBUFF *ovbuff;
#ifdef OV_DEBUG
    int recno;
    struct ov_trace_array *trace;
#endif /* OV_DEBUG */

    if (ov.index == NULLINDEX)
        return;
    if ((ovbuff = getovbuff(ov)) == NULL)
        return;
    ovlock(ovbuff, INN_LOCK_WRITE);
#ifdef OV_DEBUG
    recno = ((char *) ge - (char *) &GROUPentries[0]) / sizeof(GROUPENTRY);
    if (!ovusedblock(ovbuff, ov.blocknum, false, false)) {
        warn("buffindexed: 0x%08x trying to free block(%d, %d), but already "
             "freed",
             recno, ov.index, ov.blocknum);
        buffindexed_close();
        abort();
    }
    trace = &ovbuff->trace[ov.blocknum];
    if (trace->ov_trace == NULL) {
        trace->ov_trace = xcalloc(OV_TRACENUM, sizeof(struct ov_trace));
        trace->max = OV_TRACENUM;
    } else if (trace->cur + 1 == trace->max) {
        trace->max += OV_TRACENUM;
        trace->ov_trace =
            xrealloc(trace->ov_trace, trace->max * sizeof(struct ov_trace));
        memset(&trace->ov_trace[trace->cur], '\0',
               sizeof(struct ov_trace) * (trace->max - trace->cur));
    }
    if (trace->ov_trace[trace->cur].freed != 0) {
        trace->cur++;
    }
    trace->ov_trace[trace->cur].freed = time(NULL);
    trace->ov_trace[trace->cur].gloc.recno = recno;
    trace->cur++;
#endif /* OV_DEBUG */

#ifndef OV_DEBUG
    if (!ovusedblock(ovbuff, ov.blocknum, false, false)) {
        notice("buffindexed: trying to free block(%d, %u), but already freed.",
               ov.index, ov.blocknum);
    }
#endif

    ovusedblock(ovbuff, ov.blocknum, true, false);
    ovreadhead(ovbuff);
    if (ovbuff->freeblk == ovbuff->totalblk)
        ovbuff->freeblk = ov.blocknum;
    ovbuff->usedblk--;
    ovbuff->dirty++;
    ovflushhead(ovbuff);
    ovlock(ovbuff, INN_LOCK_UNLOCK);
    return;
}

bool
buffindexed_open(int mode)
{
    char *groupfn;
    struct stat sb;
    off_t groupsize;
    enum inn_locktype locktype;
    int i, flag;
    static int uninitialized = 1;
    ULONG on, off;

    if (uninitialized) {
        on = 1;
        off = on;
        off ^= ULONG_MAX;
        for (i = (longsize * 8) - 1; i >= 0; i--) {
            onarray[i] = on;
            offarray[i] = off;
            on <<= 1;
            off = on;
            off ^= ULONG_MAX;
        }
        uninitialized = 0;
    }
    ovbuffmode = mode;
    if (pagesize == 0) {
        pagesize = sysconf(_SC_PAGESIZE);
        if (pagesize == -1) {
            syswarn("buffindexed: sysconf(_SC_PAGESIZE) failed");
            pagesize = 0;
            return false;
        }
        if (pagesize > OV_HDR_MIN_PAGESIZE)
            hdr_pagesize = pagesize;
        if ((hdr_pagesize % pagesize) != 0) {
            warn(
                "buffindexed: hdr_pagesize (%ld) is not a multiple of pagesize"
                " (%ld)",
                hdr_pagesize, pagesize);
            return false;
        }
    }
    memset(&groupdatablock, '\0', sizeof(groupdatablock));
    if (!ovbuffread_config()) {
        return false;
    }
    Needunlink = true;
    if (!ovbuffinit_disks()) {
        return false;
    }

    groupfn = concatpath(innconf->pathdb, "group.index");
    if (Needunlink && unlink(groupfn) == 0) {
        notice("buffindexed: all buffers are brandnew, unlink '%s'", groupfn);
    }
    GROUPfd = open(
        groupfn, (ovbuffmode & OV_WRITE) ? O_RDWR | O_CREAT : O_RDONLY, 0660);
    if (GROUPfd < 0) {
        syswarn("buffindexed: Could not create %s", groupfn);
        free(groupfn);
        return false;
    }

    locktype = (mode & OV_WRITE) ? INN_LOCK_WRITE : INN_LOCK_READ;
    if (!GROUPlockhash(locktype)) {
        syswarn("buffindexed: Could not lock %s", groupfn);
        free(groupfn);
        close(GROUPfd);
        return false;
    }

    if (fstat(GROUPfd, &sb) < 0) {
        syswarn("buffindexed: Could not fstat %s", groupfn);
        GROUPlockhash(INN_LOCK_UNLOCK);
        free(groupfn);
        close(GROUPfd);
        return false;
    }
    if (sb.st_size < 0) {
        errno = EOVERFLOW;
        syswarn("buffindexed: invalid size for %s", groupfn);
        GROUPlockhash(INN_LOCK_UNLOCK);
        free(groupfn);
        close(GROUPfd);
        return false;
    }
    if (sb.st_size > (off_t) sizeof(GROUPHEADER)) {
        flag = GROUPmappingprot(mode);
        if (!GROUPentrycount(sb.st_size, &GROUPcount)
            || !GROUPfilesize(GROUPcount, &groupsize)) {
            syswarn("buffindexed: invalid size for %s", groupfn);
            GROUPlockhash(INN_LOCK_UNLOCK);
            free(groupfn);
            close(GROUPfd);
            return false;
        }
        GROUPmapsize = (size_t) groupsize;
        GROUPheader = mmap(0, GROUPmapsize, flag, MAP_SHARED, GROUPfd, 0);
        if (GROUPheader == MAP_FAILED) {
            syswarn("buffindexed: Could not mmap %s in buffindexed_open",
                    groupfn);
            GROUPheader = NULL;
            GROUPmapsize = 0;
            GROUPlockhash(INN_LOCK_UNLOCK);
            free(groupfn);
            close(GROUPfd);
            return false;
        }
        GROUPentries = (void *) &GROUPheader[1];
    } else {
        GROUPcount = 0;
        GROUPmapsize = 0;
        if (!GROUPexpand(mode)) {
            GROUPlockhash(INN_LOCK_UNLOCK);
            free(groupfn);
            close(GROUPfd);
            return false;
        }
    }
    GROUPlockhash(INN_LOCK_UNLOCK);
    fdflag_close_exec(GROUPfd, true);

    free(groupfn);
    Cutofflow = false;

    return true;
}

static GROUPLOC
GROUPfind(const char *group, bool Ignoredeleted, bool *failed)
{
    HASH grouphash;
    unsigned int i;
    int steps = 0;
    GROUPLOC loc;

    grouphash = Hash(group, strlen(group));
    memcpy(&i, &grouphash, sizeof(i));
    if (failed != NULL)
        *failed = false;
    if (GROUPheader == NULL || GROUPentries == NULL) {
        if (failed != NULL)
            *failed = true;
        return GROUPemptyloc;
    }

    loc = GROUPheader->hash[i % GROUPHEADERHASHSIZE];
    if (!GROUPremapifneeded(loc, false)) {
        if (failed != NULL)
            *failed = true;
        return GROUPemptyloc;
    }
    loc = GROUPheader->hash[i % GROUPHEADERHASHSIZE];

    while (!GROUPLOCempty(loc)) {
        if (loc.recno >= GROUPcount && !GROUPremapifneeded(loc, false)) {
            if (failed != NULL)
                *failed = true;
            return GROUPemptyloc;
        }
        if (loc.recno >= GROUPcount) {
            warn("buffindexed: group index entry %d out of range", loc.recno);
            if (failed != NULL)
                *failed = true;
            return GROUPemptyloc;
        }
        if (steps++ >= GROUPcount) {
            warn("buffindexed: cycle in group index chain");
            if (failed != NULL)
                *failed = true;
            return GROUPemptyloc;
        }
        if (GROUPentries[loc.recno].deleted == 0 || Ignoredeleted) {
            if (memcmp(&grouphash, &GROUPentries[loc.recno].hash, sizeof(HASH))
                == 0) {
                return loc;
            }
        }
        loc = GROUPentries[loc.recno].next;
    }
    return GROUPemptyloc;
}

bool
buffindexed_groupstats(const char *group, int *lo, int *hi, int *count,
                       int *flag)
{
    GROUPLOC gloc;

    gloc = GROUPfind(group, false, NULL);
    if (GROUPLOCempty(gloc)) {
        return false;
    }
    GROUPlock(gloc, INN_LOCK_READ);
    if (lo != NULL)
        *lo = GROUPentries[gloc.recno].low;
    if (hi != NULL)
        *hi = GROUPentries[gloc.recno].high;
    if (count != NULL)
        *count = GROUPentries[gloc.recno].count;
    if (flag != NULL)
        *flag = GROUPentries[gloc.recno].flag;
    GROUPlock(gloc, INN_LOCK_UNLOCK);
    return true;
}

static void
setinitialge(GROUPENTRY *ge, HASH grouphash, char *flag, GROUPLOC next,
             ARTNUM lo, ARTNUM hi)
{
    ge->hash = grouphash;
    if (lo != 0)
        ge->low = lo;
    ge->high = hi;
    ge->expired = ge->deleted = ge->count = 0;
    ge->flag = *flag;
    ge->baseindex = ge->curindex = ge->curdata = ovnull;
    ge->curindexoffset = ge->curoffset = 0;
    ge->next = next;
}

bool
buffindexed_groupadd(const char *group, ARTNUM lo, ARTNUM hi, char *flag)
{
    bool failed;
    unsigned int i;
    HASH grouphash;
    GROUPLOC gloc;
    GROUPENTRY *ge;
#ifdef OV_DEBUG
    struct ov_name_table *ntp;
#endif /* OV_DEBUG */

    gloc = GROUPfind(group, true, &failed);
    if (failed)
        return false;
    if (!GROUPLOCempty(gloc)) {
        ge = &GROUPentries[gloc.recno];
        if (GROUPentries[gloc.recno].deleted != 0) {
            grouphash = Hash(group, strlen(group));
            setinitialge(ge, grouphash, flag, ge->next, lo, hi);
        } else {
            ge->flag = *flag;
        }
        return true;
    }
    grouphash = Hash(group, strlen(group));
    memcpy(&i, &grouphash, sizeof(i));
    i = i % GROUPHEADERHASHSIZE;
    if (!GROUPlockhash(INN_LOCK_WRITE))
        return false;
    gloc = GROUPnewnode();
    if (GROUPLOCempty(gloc)) {
        GROUPlockhash(INN_LOCK_UNLOCK);
        return false;
    }
    ge = &GROUPentries[gloc.recno];
    setinitialge(ge, grouphash, flag, GROUPheader->hash[i], lo, hi);
    GROUPheader->hash[i] = gloc;
#ifdef OV_DEBUG
    ntp = xmalloc(sizeof(struct ov_name_table));
    memset(ntp, '\0', sizeof(struct ov_name_table));
    ntp->name = xstrdup(group);
    ntp->recno = gloc.recno;
    if (name_table == NULL)
        name_table = ntp;
    else {
        ntp->next = name_table;
        name_table = ntp;
    }
#endif /* OV_DEBUG */
    GROUPlockhash(INN_LOCK_UNLOCK);
    return true;
}

static bool
GROUPentrycount(off_t size, int *count)
{
    uintmax_t payload;
    uintmax_t entries;

    if (size < (off_t) sizeof(GROUPHEADER)) {
        errno = EINVAL;
        return false;
    }
    if ((uintmax_t) size > SIZE_MAX) {
        errno = EOVERFLOW;
        return false;
    }
    payload = (uintmax_t) size - sizeof(GROUPHEADER);
    if (payload % sizeof(GROUPENTRY) != 0) {
        errno = EINVAL;
        return false;
    }
    entries = payload / sizeof(GROUPENTRY);
    if (entries > INT_MAX) {
        errno = EOVERFLOW;
        return false;
    }
    *count = (int) entries;
    return true;
}

static bool
GROUPfilesize(int count, off_t *size)
{
    uintmax_t bytes;

    if (count < 0
        || (uintmax_t) count
               > (UINTMAX_MAX - sizeof(GROUPHEADER)) / sizeof(GROUPENTRY)) {
        errno = EOVERFLOW;
        return false;
    }
    bytes = sizeof(GROUPHEADER) + (uintmax_t) count * sizeof(GROUPENTRY);
    if (bytes > SIZE_MAX) {
        errno = EOVERFLOW;
        return false;
    }
    *size = (off_t) bytes;
    if (*size < 0 || (uintmax_t) *size != bytes) {
        errno = EOVERFLOW;
        return false;
    }
    return true;
}

/* Return the mmap protection required by an overview open mode.  Writers
 * also need read permission because the mapped header is inspected. */
static int
GROUPmappingprot(int mode)
{
    int prot = 0;

    if (mode & OV_READ)
        prot |= PROT_READ;
    if (mode & OV_WRITE)
        prot |= PROT_READ | PROT_WRITE;
    return prot;
}

/* Check if the given GROUPLOC refers to GROUPENTRY that we don't have mmap'ed,
** if so then see if the file has been grown by another writer and remmap
*/
static bool
GROUPremapifneeded(GROUPLOC loc, bool hash_locked)
{
    GROUPHEADER *newheader;
    struct stat sb;
    off_t groupsize;
    bool locked = false;
    int count;

    if (GROUPheader != NULL && GROUPentries != NULL && loc.recno < GROUPcount)
        return true;

    /* GROUPexpand publishes the new file size while holding this lock.  A
       remapper must not map that size until expansion has committed the new
       header or left a stable zero-filled extension after failure. */
    if (!hash_locked) {
        if (!GROUPlockhash(INN_LOCK_READ))
            return false;
        locked = true;
    }

    if (fstat(GROUPfd, &sb) < 0) {
        if (locked)
            GROUPlockhash(INN_LOCK_UNLOCK);
        return false;
    }

    if (GROUPheader != NULL && GROUPentries != NULL
        && (uintmax_t) GROUPmapsize >= (uintmax_t) sb.st_size) {
        if (locked)
            GROUPlockhash(INN_LOCK_UNLOCK);
        return true;
    }

    if (!GROUPentrycount(sb.st_size, &count)
        || !GROUPfilesize(count, &groupsize)) {
        syswarn("buffindexed: invalid group.index size");
        if (locked)
            GROUPlockhash(INN_LOCK_UNLOCK);
        return false;
    }

    newheader = mmap(0, (size_t) groupsize, GROUPmappingprot(ovbuffmode),
                     MAP_SHARED, GROUPfd, 0);
    if (newheader == MAP_FAILED) {
        syswarn(
            "buffindexed: Could not mmap group.index in GROUPremapifneeded");
        if (locked)
            GROUPlockhash(INN_LOCK_UNLOCK);
        return false;
    }

    if (GROUPheader != NULL) {
        if (munmap((void *) GROUPheader, GROUPmapsize) < 0) {
            syswarn("buffindexed: Could not munmap group.index in "
                    "GROUPremapifneeded");
            munmap(newheader, (size_t) groupsize);
            if (locked)
                GROUPlockhash(INN_LOCK_UNLOCK);
            return false;
        }
    }

    GROUPcount = count;
    GROUPmapsize = (size_t) groupsize;
    GROUPheader = newheader;
    GROUPentries = (void *) &GROUPheader[1];
    if (locked)
        GROUPlockhash(INN_LOCK_UNLOCK);
    return true;
}

/* This function does not need to lock because it's callers are expected to do
 * so */
static bool
GROUPexpand(int mode)
{
    GROUPHEADER *newheader;
    struct stat st;
    off_t groupsize, oldsize;
    int i;
    int newcount;

    if (GROUPcount > INT_MAX - 1024) {
        errno = EOVERFLOW;
        syswarn("buffindexed: group.index has too many entries");
        return false;
    }
    newcount = GROUPcount + 1024;
    if (!GROUPfilesize(newcount, &groupsize)) {
        syswarn("buffindexed: invalid expanded group.index size");
        return false;
    }
    if (fstat(GROUPfd, &st) < 0) {
        syswarn("buffindexed: Could not stat group.index before expansion");
        return false;
    }
    oldsize = st.st_size;
    if (ftruncate(GROUPfd, groupsize) < 0) {
        syswarn("buffindexed: Could not extend group.index");
        return false;
    }
    newheader = mmap(0, (size_t) groupsize, GROUPmappingprot(mode), MAP_SHARED,
                     GROUPfd, 0);
    if (newheader == MAP_FAILED) {
        syswarn("buffindexed: Could not mmap group.index in GROUPexpand");
        /* If this is initial creation, restore the header-only file.  For an
           established mapping, keep the harmless zero-filled extension so
           that a reader from an older process cannot be left mapped past EOF.
           A later expansion attempt will initialize the same entries. */
        if (GROUPheader == NULL && ftruncate(GROUPfd, oldsize) < 0)
            syswarn("buffindexed: Could not undo group.index expansion");
        return false;
    }
    if (GROUPheader != NULL
        && munmap((void *) GROUPheader, GROUPmapsize) < 0) {
        syswarn("buffindexed: Could not munmap group.index in GROUPexpand");
        munmap(newheader, (size_t) groupsize);
        return false;
    }
    GROUPcount = newcount;
    GROUPmapsize = (size_t) groupsize;
    GROUPheader = newheader;
    GROUPentries = (void *) &GROUPheader[1];
    if (GROUPheader->magic != GROUPHEADERMAGIC) {
        GROUPheader->magic = GROUPHEADERMAGIC;
        GROUPLOCclear(&GROUPheader->freelist);
        for (i = 0; i < GROUPHEADERHASHSIZE; i++)
            GROUPLOCclear(&GROUPheader->hash[i]);
    }
    /* Walk the new entries from the back to the front, adding them to the
     * freelist */
    for (i = GROUPcount - 1; (GROUPcount - 1024) <= i; i--) {
        GROUPentries[i].next = GROUPheader->freelist;
        GROUPheader->freelist.recno = i;
    }
    return true;
}

static GROUPLOC
GROUPnewnode(void)
{
    GROUPLOC loc;

    /* If we didn't find any free space, first pick up an expansion performed
       by another writer, and then make more space if necessary. */
    if (GROUPLOCempty(GROUPheader->freelist)) {
        loc.recno = INT_MAX;
        if (!GROUPremapifneeded(loc, true))
            return GROUPemptyloc;
    }
    if (GROUPLOCempty(GROUPheader->freelist)) {
        if (!GROUPexpand(ovbuffmode)) {
            return GROUPemptyloc;
        }
    }

    loc = GROUPheader->freelist;
    if (!GROUPremapifneeded(loc, true))
        return GROUPemptyloc;
    loc = GROUPheader->freelist;
    if (GROUPLOCempty(loc) || loc.recno >= GROUPcount) {
        warn("buffindexed: freelist entry %d out of range", loc.recno);
        return GROUPemptyloc;
    }
    GROUPheader->freelist = GROUPentries[loc.recno].next;
    return loc;
}

bool
buffindexed_groupdel(const char *group)
{
    bool failed;
    GROUPLOC gloc;
    GROUPENTRY *ge;

    gloc = GROUPfind(group, false, &failed);
    if (failed)
        return false;
    if (GROUPLOCempty(gloc)) {
        return true;
    }
    GROUPlock(gloc, INN_LOCK_WRITE);
    ge = &GROUPentries[gloc.recno];
    ge->deleted = time(NULL);
    HashClear(&ge->hash);
    GROUPlock(gloc, INN_LOCK_UNLOCK);
    return true;
}

static void
GROUPLOCclear(GROUPLOC *loc)
{
    loc->recno = -1;
}

static bool
GROUPLOCempty(GROUPLOC loc)
{
    return (loc.recno < 0);
}

static bool
GROUPlockhash(enum inn_locktype type)
{
    return inn_lock_range(GROUPfd, type, true, 0, sizeof(GROUPHEADER));
}

static bool
GROUPlock(GROUPLOC gloc, enum inn_locktype type)
{
    return inn_lock_range(GROUPfd, type, true,
                          sizeof(GROUPHEADER)
                              + (sizeof(GROUPENTRY) * gloc.recno),
                          sizeof(GROUPENTRY));
}

#ifdef OV_DEBUG
static bool
ovsetcurindexblock(GROUPENTRY *ge, GROUPENTRY *georig)
{
#else
static bool
ovsetcurindexblock(GROUPENTRY *ge)
{
#endif /* OV_DEBUG */
    OVBUFF *ovbuff;
    OV ov;
    OVINDEXHEAD ovindexhead;

    /* there is no index */
#ifdef OV_DEBUG
    ov = ovblocknew(georig ? georig : ge);
#else
    ov = ovblocknew();
#endif /* OV_DEBUG */
    if (ov.index == NULLINDEX) {
        warn("buffindexed: ovsetcurindexblock could not get new block");
        return false;
    }
    if ((ovbuff = getovbuff(ov)) == NULL) {
        warn("buffindexed: ovsetcurindexblock could not get ovbuff block for"
             " new, %d, %u",
             ov.index, ov.blocknum);
        return false;
    }
    ovindexhead.next = ovnull;
    ovindexhead.low = 0;
    ovindexhead.high = 0;
    if (PWRITE(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD),
               ovbuff->base + OV_OFFSET(ov.blocknum))
        != sizeof(OVINDEXHEAD)) {
        syswarn("buffindexed: could not write index record index '%d',"
                " blocknum '%u'",
                ge->curindex.index, ge->curindex.blocknum);
        return true;
    }
    if (ge->baseindex.index == NULLINDEX) {
        ge->baseindex = ov;
    } else {
        if ((ovbuff = getovbuff(ge->curindex)) == NULL)
            return false;
        if (!ovusedblock(ovbuff, ge->curindex.blocknum, false, false)) {
            warn("buffindexed: block(%u, %u) not occupied (index)",
                 ovbuff->index, ge->curindex.blocknum);
#ifdef OV_DEBUG
            abort();
#else  /* OV_DEBUG */
            /* fix it */
            ovusedblock(ovbuff, ge->curindex.blocknum, true, true);
#endif /* OV_DEBUG */
        }
        ovindexhead.next = ov;
        ovindexhead.low = ge->curlow;
        ovindexhead.high = ge->curhigh;
        if (PWRITE(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD),
                   ovbuff->base + OV_OFFSET(ge->curindex.blocknum))
            != sizeof(OVINDEXHEAD)) {
            syswarn("buffindexed: could not write index record index '%d',"
                    " blocknum '%u'",
                    ge->curindex.index, ge->curindex.blocknum);
            return false;
        }
    }
    ge->curindex = ov;
    ge->curindexoffset = 0;
    ge->curlow = 0;
    ge->curhigh = 0;
    return true;
}

#ifdef OV_DEBUG
static bool
ovaddrec(GROUPENTRY *ge, ARTNUM artnum, TOKEN token, char *data, int len,
         time_t arrived, time_t expires, GROUPENTRY *georig)
{
#else
static bool
ovaddrec(GROUPENTRY *ge, ARTNUM artnum, TOKEN token, char *data, int len,
         time_t arrived, time_t expires)
{
#endif /* OV_DEBUG */
    OV ov;
    OVINDEX ie;
    OVBUFF *ovbuff;
    OVINDEXHEAD ovindexhead;
    bool needupdate = false;

    Nospace = false;
    if (OV_BLOCKSIZE < len) {
        warn("buffindexed: overview data must be under %d (%d)", OV_BLOCKSIZE,
             len);
        return false;
    }
    if (ge->curdata.index == NULLINDEX) {
        /* no data block allocated */
#ifdef OV_DEBUG
        ov = ovblocknew(georig ? georig : ge);
#else
        ov = ovblocknew();
#endif /* OV_DEBUG */
        if (ov.index == NULLINDEX) {
            warn("buffindexed: ovaddrec could not get new block");
            return false;
        }
        if ((ovbuff = getovbuff(ov)) == NULL) {
            warn("buffindexed: ovaddrec could not get ovbuff block for"
                 " new, %d, %u, %lu",
                 ov.index, ov.blocknum, artnum);
            return false;
        }
        ge->curdata = ov;
        ge->curoffset = 0;
    } else if ((ovbuff = getovbuff(ge->curdata)) == NULL)
        return false;
    else if (OV_BLOCKSIZE - ge->curoffset < len) {
        /* too short to store data, allocate new block */
#ifdef OV_DEBUG
        ov = ovblocknew(georig ? georig : ge);
#else
        ov = ovblocknew();
#endif /* OV_DEBUG */
        if (ov.index == NULLINDEX) {
            warn("buffindexed: ovaddrec could not get new block");
            return false;
        }
        if ((ovbuff = getovbuff(ov)) == NULL) {
            warn("buffindexed: ovaddrec could not get ovbuff block for new, "
                 "%d, %u, %lu",
                 ov.index, ov.blocknum, artnum);
            return false;
        }
        ge->curdata = ov;
        ge->curoffset = 0;
    }
    if (!ovusedblock(ovbuff, ge->curdata.blocknum, false, false)) {
        warn("buffindexed: block(%u, %u) not occupied", ovbuff->index,
             ge->curdata.blocknum);
#ifdef OV_DEBUG
        buffindexed_close();
        abort();
#else  /* OV_DEBUG */
        /* fix it */
        ovusedblock(ovbuff, ge->curdata.blocknum, true, true);
#endif /* OV_DEBUG */
    }

    if (PWRITE(ovbuff->fd, data, len,
               ovbuff->base + OV_OFFSET(ge->curdata.blocknum) + ge->curoffset)
        != len) {
        syswarn("buffindexed: could not append overview record index '%d',"
                " blocknum '%u'",
                ge->curdata.index, ge->curdata.blocknum);
        return false;
    }
    memset(&ie, '\0', sizeof(ie));
    ie.artnum = artnum;
    ie.len = len;
    ie.index = ge->curdata.index;
    ie.blocknum = ge->curdata.blocknum;
    ie.offset = ge->curoffset;
    ie.token = token;
    ie.arrived = arrived;
    ie.expires = expires;

    if (ge->baseindex.index == NULLINDEX || ge->curindexoffset == OVINDEXMAX) {
#ifdef OV_DEBUG
        if (!ovsetcurindexblock(ge, georig)) {
#else
        if (!ovsetcurindexblock(ge)) {
#endif /* OV_DEBUG */
            warn("buffindexed: could not set current index");
            return false;
        }
    }
    if ((ovbuff = getovbuff(ge->curindex)) == NULL)
        return false;
    if (!ovusedblock(ovbuff, ge->curindex.blocknum, false, false)) {
        warn("buffindexed: block(%u, %u) not occupied (index)", ovbuff->index,
             ge->curindex.blocknum);
#ifdef OV_DEBUG
        buffindexed_close();
        abort();
#else  /* OV_DEBUG */
        /* fix this */
        ovusedblock(ovbuff, ge->curindex.blocknum, true, true);
#endif /* OV_DEBUG */
    }
    if (PWRITE(ovbuff->fd, &ie, sizeof(ie),
               ovbuff->base + OV_OFFSET(ge->curindex.blocknum)
                   + sizeof(OVINDEXHEAD) + sizeof(ie) * ge->curindexoffset)
        != sizeof(ie)) {
        syswarn(
            "buffindexed: could not write index record index '%d', blocknum"
            " '%u'",
            ge->curindex.index, ge->curindex.blocknum);
        return true;
    }
    if ((ge->curlow <= 0) || (ge->curlow > artnum)) {
        ge->curlow = artnum;
        needupdate = true;
    }
    if ((ge->curhigh <= 0) || (ge->curhigh < artnum)) {
        ge->curhigh = artnum;
        needupdate = true;
    }
    if (needupdate) {
        ovindexhead.next = ovnull;
        ovindexhead.low = ge->curlow;
        ovindexhead.high = ge->curhigh;
        if (PWRITE(ovbuff->fd, &ovindexhead, sizeof(OVINDEXHEAD),
                   ovbuff->base + OV_OFFSET(ge->curindex.blocknum))
            != sizeof(OVINDEXHEAD)) {
            syswarn("buffindexed: could not write index record index '%d', "
                    "blocknum '%u'",
                    ge->curindex.index, ge->curindex.blocknum);
            return true;
        }
    }
    if ((ge->low <= 0) || (ge->low > artnum))
        ge->low = artnum;
    if ((ge->high <= 0) || (ge->high < artnum))
        ge->high = artnum;
    ge->curindexoffset++;
    ge->curoffset += len;
    ge->count++;
    return true;
}

bool
buffindexed_add(const char *group, ARTNUM artnum, TOKEN token, char *data,
                int len, time_t arrived, time_t expires)
{
    bool failed;
    GROUPLOC gloc;
    GROUPENTRY *ge;

    if (len > OV_BLOCKSIZE) {
        warn("buffindexed: overview data is too large %d", len);
        return true;
    }

    gloc = GROUPfind(group, false, &failed);
    if (failed)
        return false;
    if (GROUPLOCempty(gloc)) {
        return true;
    }
    GROUPlock(gloc, INN_LOCK_WRITE);
    /* prepend block(s) if needed. */
    ge = &GROUPentries[gloc.recno];
    if (Cutofflow && ge->low > artnum) {
        GROUPlock(gloc, INN_LOCK_UNLOCK);
        return true;
    }
#ifdef OV_DEBUG
    if (!ovaddrec(ge, artnum, token, data, len, arrived, expires, NULL)) {
#else
    if (!ovaddrec(ge, artnum, token, data, len, arrived, expires)) {
#endif /* OV_DEBUG */
        if (Nospace) {
            GROUPlock(gloc, INN_LOCK_UNLOCK);
            warn("buffindexed: no space left for buffer, adding '%s'", group);
            return false;
        }
        warn("buffindexed: could not add overview for '%s'", group);
    }
    GROUPlock(gloc, INN_LOCK_UNLOCK);

    return true;
}

bool
buffindexed_cancel(const char *group UNUSED, ARTNUM artnum UNUSED)
{
    return true;
}

#ifdef OV_DEBUG
static void
freegroupblock(GROUPENTRY *ge)
{
#else
static void
freegroupblock(void)
{
#endif /* OV_DEBUG */
    GROUPDATABLOCK *gdb;
    int i;
    GIBLIST *giblist;

    for (giblist = Giblist; giblist != NULL; giblist = giblist->next) {
#ifdef OV_DEBUG
        ovblockfree(giblist->ov, ge);
#else
        ovblockfree(giblist->ov);
#endif /* OV_DEBUG */
    }
    for (i = 0; i < GROUPDATAHASHSIZE; i++) {
        for (gdb = groupdatablock[i]; gdb != NULL; gdb = gdb->next) {
#ifdef OV_DEBUG
            ovblockfree(gdb->datablk, ge);
#else
            ovblockfree(gdb->datablk);
#endif /* OV_DEBUG */
        }
    }
}

static void
ovgroupunmap(void)
{
    GROUPDATABLOCK *gdb, *gdbnext;
    int i;
    GIBLIST *giblist, *giblistnext;

    for (i = 0; i < GROUPDATAHASHSIZE; i++) {
        for (gdb = groupdatablock[i]; gdb != NULL; gdb = gdbnext) {
            gdbnext = gdb->next;
            if (gdb->mmapped)
                munmap(gdb->addr, gdb->len);
            free(gdb);
        }
        groupdatablock[i] = NULL;
    }
    for (giblist = Giblist; giblist != NULL; giblist = giblistnext) {
        giblistnext = giblist->next;
        free(giblist);
    }
    Giblist = NULL;
    if (!Cache && (Gib != NULL)) {
        free(Gib);
        Gib = NULL;
        if (Cachesearch != NULL) {
            free(Cachesearch->group);
            free(Cachesearch);
            Cachesearch = NULL;
        }
    }
}

/* Failure must invalidate cached index state as well as temporary mappings.
   A zero Gibcount with a non-NULL, partially filled Gib is not a valid cache
   entry (gettoken expects at least one element whenever Gib is non-NULL). */
static void
ovgroupunmap_failed(void)
{
    ovgroupunmap();
    free(Gib);
    Gib = NULL;
    Gibcount = 0;
    if (Cachesearch != NULL) {
        free(Cachesearch->group);
        free(Cachesearch);
        Cachesearch = NULL;
    }
}

static void
insertgdb(OV *ov, GROUPDATABLOCK *gdb)
{
    gdb->next = groupdatablock[(ov->index + ov->blocknum) % GROUPDATAHASHSIZE];
    groupdatablock[(ov->index + ov->blocknum) % GROUPDATAHASHSIZE] = gdb;
    return;
}

static GROUPDATABLOCK *
searchgdb(OV *ov)
{
    GROUPDATABLOCK *gdb;

    gdb = groupdatablock[(ov->index + ov->blocknum) % GROUPDATAHASHSIZE];
    for (; gdb != NULL; gdb = gdb->next) {
        if (ov->index == gdb->datablk.index
            && ov->blocknum == gdb->datablk.blocknum)
            break;
    }
    return gdb;
}

static int
INDEXcompare(const void *p1, const void *p2)
{
    const OVINDEX *oi1 = p1;
    const OVINDEX *oi2 = p2;

    return oi1->artnum - oi2->artnum;
}

static bool
ovgroupmmap(GROUPENTRY *ge, ARTNUM low, ARTNUM high, bool needov)
{
    OV ov = ge->baseindex;
    OVBUFF *ovbuff;
    GROUPDATABLOCK *gdb;
    int pagefudge, limit, i, capacity, count, expected, len;
    off_t offset, mmapoffset;
    OVBLOCK *ovblock;
    void *addr;
    GIBLIST *giblist;

    if (low > high) {
        Gibcount = 0;
        return true;
    }
    expected = ge->count;
    Gibcount = 0;
    if (expected == 0)
        return true;
    if (expected < 0) {
        warn("buffindexed: negative overview count");
        return false;
    }
    capacity = expected > OV_FUDGE ? OV_FUDGE : expected;
    Gib = xmalloc(capacity * sizeof(OVINDEX));
    count = 0;
    while (ov.index != NULLINDEX) {
        for (giblist = Giblist; giblist != NULL; giblist = giblist->next)
            if (giblist->ov.index == ov.index
                && giblist->ov.blocknum == ov.blocknum) {
                warn("buffindexed: loop in overview index blocks");
                ovgroupunmap_failed();
                return false;
            }
        ovbuff = getovbuff(ov);
        if (ovbuff == NULL || !ovblockvalid(ovbuff, ov.blocknum)) {
            warn("buffindexed: invalid index block %d:%u", ov.index,
                 ov.blocknum);
            ovgroupunmap_failed();
            return false;
        }
        offset = ovbuff->base + OV_OFFSET(ov.blocknum);
        pagefudge = offset % pagesize;
        mmapoffset = offset - pagefudge;
        len = pagefudge + OV_BLOCKSIZE;
        if ((addr = mmap(NULL, len, PROT_READ, MAP_SHARED, ovbuff->fd,
                         mmapoffset))
            == MAP_FAILED) {
            syswarn("buffindexed: ovgroupmmap could not mmap index block");
            ovgroupunmap_failed();
            return false;
        }
        ovblock = (void *) ((char *) addr + pagefudge);
        if (ov.index == ge->curindex.index
            && ov.blocknum == ge->curindex.blocknum) {
            limit = ge->curindexoffset;
            if (limit < 0 || (size_t) limit > OVINDEXMAX) {
                warn("buffindexed: invalid index entry count %d", limit);
                munmap(addr, len);
                ovgroupunmap_failed();
                return false;
            }
        } else {
            limit = OVINDEXMAX;
        }
        for (i = 0; i < limit; i++) {
            if (capacity == count) {
                if (capacity > INT_MAX - OV_FUDGE) {
                    warn("buffindexed: too many overview index entries");
                    munmap(addr, len);
                    ovgroupunmap_failed();
                    return false;
                }
                capacity += OV_FUDGE;
                Gib = xreallocarray(Gib, capacity, sizeof(OVINDEX));
            }
            Gib[count++] = ovblock->ovindex[i];
        }
        giblist = xmalloc(sizeof(GIBLIST));
        giblist->ov = ov;
        giblist->next = Giblist;
        Giblist = giblist;
        ov = ovblock->ovindexhead.next;
        munmap(addr, len);
    }
    Gibcount = count;
    qsort(Gib, Gibcount, sizeof(OVINDEX), INDEXcompare);
    /* Remove duplicates. */
    for (i = 0; i < Gibcount - 1; i++) {
        if (Gib[i].artnum == Gib[i + 1].artnum) {
            /* lower position is removed */
            Gib[i].artnum = 0;
        }
    }
    if (!needov)
        return true;
    count = 0;
    for (i = 0; i < Gibcount; i++) {
        if (Gib[i].artnum == 0 || Gib[i].artnum < low || Gib[i].artnum > high)
            continue;
        ov.index = Gib[i].index;
        ov.blocknum = Gib[i].blocknum;
        gdb = searchgdb(&ov);
        if (gdb != NULL)
            continue;
        ovbuff = getovbuff(ov);
        if (ovbuff == NULL || !ovblockvalid(ovbuff, ov.blocknum))
            continue;
        gdb = xmalloc(sizeof(GROUPDATABLOCK));
        gdb->datablk = ov;
        gdb->next = NULL;
        gdb->mmapped = false;
        insertgdb(&ov, gdb);
        count++;
    }
    if (count == 0)
        return true;
    if ((unsigned) count * OV_BLOCKSIZE > innconf->keepmmappedthreshold * 1024)
        /* large retrieval, mmap is done in ovsearch() */
        return true;
    /* Data blocks are being mmapped, not copied. */
    for (i = 0; i < GROUPDATAHASHSIZE; i++) {
        for (gdb = groupdatablock[i]; gdb != NULL; gdb = gdb->next) {
            ov = gdb->datablk;
            ovbuff = getovbuff(ov);
            if (ovbuff == NULL || !ovblockvalid(ovbuff, ov.blocknum)) {
                warn("buffindexed: invalid overview data block %d:%u",
                     ov.index, ov.blocknum);
                ovgroupunmap_failed();
                return false;
            }
            offset = ovbuff->base + OV_OFFSET(ov.blocknum);
            pagefudge = offset % pagesize;
            mmapoffset = offset - pagefudge;
            gdb->len = pagefudge + OV_BLOCKSIZE;
            if ((gdb->addr = mmap(NULL, gdb->len, PROT_READ, MAP_SHARED,
                                  ovbuff->fd, mmapoffset))
                == MAP_FAILED) {
                syswarn("buffindexed: ovgroupmmap could not mmap data block");
                ovgroupunmap_failed();
                return false;
            }
            gdb->data = (char *) gdb->addr + pagefudge;
            gdb->mmapped = true;
        }
    }
    return true;
}

static void *
ovopensearch(const char *group, ARTNUM low, ARTNUM high, bool needov)
{
    GROUPLOC gloc;
    GROUPENTRY *ge;
    OVSEARCH *search;

    gloc = GROUPfind(group, false, NULL);
    if (GROUPLOCempty(gloc))
        return NULL;

    ge = &GROUPentries[gloc.recno];
    if (low < ge->low)
        low = ge->low;
    if (high > ge->high)
        high = ge->high;

    if (!ovgroupmmap(ge, low, high, needov)) {
        return NULL;
    }

    search = xmalloc(sizeof(OVSEARCH));
    search->hi = high;
    search->lo = low;
    search->cur = 0;
    search->group = xstrdup(group);
    search->needov = needov;
    search->gloc = gloc;
    search->count = ge->count;
    search->gdb.mmapped = false;
    return (void *) search;
}

void *
buffindexed_opensearch(const char *group, int low, int high)
{
    GROUPLOC gloc;
    void *handle;

    if (Gib != NULL) {
        free(Gib);
        Gib = NULL;
        if (Cachesearch != NULL) {
            free(Cachesearch->group);
            free(Cachesearch);
            Cachesearch = NULL;
        }
    }
    gloc = GROUPfind(group, false, NULL);
    if (GROUPLOCempty(gloc)) {
        return NULL;
    }
    GROUPlock(gloc, INN_LOCK_WRITE);
    if ((handle = ovopensearch(group, low, high, true)) == NULL)
        GROUPlock(gloc, INN_LOCK_UNLOCK);
    return (handle);
}

static bool
ovsearch(void *handle, ARTNUM *artnum, char **data, int *len, TOKEN *token,
         time_t *arrived, time_t *expires)
{
    OVSEARCH *search = (OVSEARCH *) handle;
    OV srchov;
    GROUPDATABLOCK *gdb;
    off_t offset, mmapoffset;
    OVBUFF *ovbuff;
    int pagefudge;
    bool newblock;

    if (search->cur == Gibcount) {
        return false;
    }
    while (Gib[search->cur].artnum == 0
           || Gib[search->cur].artnum < search->lo) {
        search->cur++;
        if (search->cur == Gibcount)
            return false;
    }
    if (Gib[search->cur].artnum > search->hi)
        return false;

    if (search->needov) {
        if (Gib[search->cur].index == NULLINDEX) {
            if (len)
                *len = 0;
            if (artnum)
                *artnum = Gib[search->cur].artnum;
        } else {
            srchov.index = Gib[search->cur].index;
            srchov.blocknum = Gib[search->cur].blocknum;
            ovbuff = getovbuff(srchov);
            if (ovbuff == NULL || !ovblockvalid(ovbuff, srchov.blocknum)
                || !ovindexspanvalid(&Gib[search->cur])) {
                warn("buffindexed: invalid overview entry for article %lu",
                     (unsigned long) Gib[search->cur].artnum);
                search->cur++;
                return false;
            }
            if (artnum)
                *artnum = Gib[search->cur].artnum;
            if (len)
                *len = Gib[search->cur].len;
            if (arrived)
                *arrived = Gib[search->cur].arrived;
            if (expires)
                *expires = Gib[search->cur].expires;
            if (data) {
                gdb = searchgdb(&srchov);
                if (gdb == NULL) {
                    if (len)
                        *len = 0;
                    search->cur++;
                    return true;
                }
                if (!gdb->mmapped) {
                    /* block needs to be mmapped */
                    if (search->gdb.mmapped) {
                        /* check previous mmapped area */
                        if (search->gdb.datablk.blocknum != srchov.blocknum
                            || search->gdb.datablk.index != srchov.index) {
                            /* different one, release previous one */
                            munmap(search->gdb.addr, search->gdb.len);
                            newblock = true;
                        } else
                            newblock = false;
                    } else
                        newblock = true;
                    if (newblock) {
                        search->gdb.datablk.blocknum = srchov.blocknum;
                        search->gdb.datablk.index = srchov.index;
                        if (ovbuff == NULL
                            || !ovblockvalid(ovbuff, srchov.blocknum)) {
                            warn("buffindexed: ovsearch could not get ovbuff"
                                 " block for new, %d, %u",
                                 srchov.index, srchov.blocknum);
                            return false;
                        }
                        offset = ovbuff->base + OV_OFFSET(srchov.blocknum);
                        pagefudge = offset % pagesize;
                        mmapoffset = offset - pagefudge;
                        search->gdb.len = pagefudge + OV_BLOCKSIZE;
                        if ((search->gdb.addr =
                                 mmap(NULL, search->gdb.len, PROT_READ,
                                      MAP_SHARED, ovbuff->fd, mmapoffset))
                            == MAP_FAILED) {
                            syswarn("buffindexed: ovsearch could not mmap "
                                    "data block");
                            return false;
                        }
                        gdb->data = search->gdb.data =
                            (char *) search->gdb.addr + pagefudge;
                        search->gdb.mmapped = true;
                    }
                }
                *data = (char *) gdb->data + Gib[search->cur].offset;
            }
        }
    }
    if (token) {
        if (Gib[search->cur].index == NULLINDEX && !search->needov) {
            search->cur++;
            return false;
        }
        *token = Gib[search->cur].token;
    }
    search->cur++;
    return true;
}

bool
buffindexed_search(void *handle, ARTNUM *artnum, char **data, int *len,
                   TOKEN *token, time_t *arrived)
{
    return (ovsearch(handle, artnum, data, len, token, arrived, NULL));
}

static void
ovclosesearch(void *handle, bool freeblock)
{
    OVSEARCH *search = (OVSEARCH *) handle;
#ifdef OV_DEBUG
    GROUPENTRY *ge;
    GROUPLOC gloc;
#endif /* OV_DEBUG */

    if (search->gdb.mmapped)
        munmap(search->gdb.addr, search->gdb.len);
    if (freeblock) {
#ifdef OV_DEBUG
        gloc = GROUPfind(search->group, false, NULL);
        if (!GROUPLOCempty(gloc)) {
            ge = &GROUPentries[gloc.recno];
            freegroupblock(ge);
        }
#else
        freegroupblock();
#endif /* OV_DEBUG */
    }
    ovgroupunmap();
    if (Cache) {
        Cachesearch = search;
    } else {
        free(search->group);
        free(search);
    }
    return;
}

void
buffindexed_closesearch(void *handle)
{
    OVSEARCH *search = (OVSEARCH *) handle;
    GROUPLOC gloc;

    gloc = search->gloc;
    ovclosesearch(handle, false);
    GROUPlock(gloc, INN_LOCK_UNLOCK);
}

/* get token from sorted index */
static bool
gettoken(ARTNUM artnum, TOKEN *token)
{
    int i, j, offset, limit;

    if (Gib == NULL || Gibcount <= 0)
        return false;
    offset = 0;
    limit = Gibcount;
    for (i = (limit - offset) / 2; i > 0; i = (limit - offset) / 2) {
        if (Gib[offset + i].artnum == artnum) {
            *token = Gib[offset + i].token;
            return true;
        } else if (Gib[offset + i].artnum == 0) {
            /* case for duplicated index */
            for (j = offset + i - 1; j >= offset; j--) {
                if (Gib[j].artnum != 0)
                    break;
            }
            if (j < offset) {
                /* article not found */
                return false;
            }
            if (Gib[j].artnum == artnum) {
                *token = Gib[j].token;
                return true;
            } else if (Gib[j].artnum < artnum) {
                /* limit is not changed */
                offset += i + 1;
            } else {
                /* offset is not changed */
                limit = j;
            }
        } else if (Gib[offset + i].artnum < artnum) {
            /* limit is unchanged */
            offset += i + 1;
        } else {
            /* offset is unchanged */
            limit = offset + i;
        }
    }
    /* i == 0 */
    if (Gib[offset].artnum != artnum) {
        /* article not found */
        return false;
    }
    *token = Gib[offset].token;
    return true;
}

bool
buffindexed_getartinfo(const char *group, ARTNUM artnum, TOKEN *token)
{
    GROUPLOC gloc;
    void *handle;
    bool retval, grouplocked = false;

    if (Gib != NULL) {
        if (Cachesearch != NULL && strcmp(Cachesearch->group, group) != 0) {
            free(Gib);
            Gib = NULL;
            free(Cachesearch->group);
            free(Cachesearch);
            Cachesearch = NULL;
        } else {
            if (gettoken(artnum, token))
                return true;
            else {
                /* examine to see if overview index are increased */
                gloc = GROUPfind(group, false, NULL);
                if (GROUPLOCempty(gloc)) {
                    return false;
                }
                GROUPlock(gloc, INN_LOCK_WRITE);
                if ((Cachesearch != NULL)
                    && (GROUPentries[gloc.recno].count
                        == Cachesearch->count)) {
                    /* no new overview data is stored */
                    GROUPlock(gloc, INN_LOCK_UNLOCK);
                    return false;
                } else {
                    grouplocked = true;
                    free(Gib);
                    Gib = NULL;
                    if (Cachesearch != NULL) {
                        free(Cachesearch->group);
                        free(Cachesearch);
                        Cachesearch = NULL;
                    }
                }
            }
        }
    }
    gloc = GROUPfind(group, false, NULL);
    if (GROUPLOCempty(gloc)) {
        return false;
    }
    if (!grouplocked) {
        GROUPlock(gloc, INN_LOCK_WRITE);
    }
    if (!(handle = ovopensearch(group, artnum, artnum, false))) {
        GROUPlock(gloc, INN_LOCK_UNLOCK);
        return false;
    }
    retval = buffindexed_search(handle, NULL, NULL, NULL, token, NULL);
    ovclosesearch(handle, false);
    GROUPlock(gloc, INN_LOCK_UNLOCK);
    return retval;
}

bool
buffindexed_expiregroup(const char *group, int *lo, struct history *h)
{
    void *handle;
    GROUPENTRY newge, *ge;
    GROUPLOC gloc, next;
    char *data = NULL;
    int i;
    int len = 0;
    TOKEN token;
    ARTNUM artnum = 0;
    ARTNUM low, high;
    ARTHANDLE *ah;
    char flag;
    HASH hash;
    time_t arrived = 0;
    time_t expires = 0;
    OVSEARCH search = {0};

    if (group == NULL) {
        /* Refresh a mapping that may have been extended by another writer,
           then inspect every slot.  Empty slots have count zero, while -1 in
           the freelist means all slots have been allocated. */
        gloc.recno = INT_MAX;
        if (!GROUPremapifneeded(gloc, false))
            return false;
        for (i = 0; i < GROUPcount; i++) {
            gloc.recno = i;
            GROUPlock(gloc, INN_LOCK_WRITE);
            ge = &GROUPentries[gloc.recno];
            if (ge->expired >= OVrealnow || ge->count == 0) {
                GROUPlock(gloc, INN_LOCK_UNLOCK);
                continue;
            }
            if (!ovgroupmmap(ge, ge->low, ge->high, true)) {
                GROUPlock(gloc, INN_LOCK_UNLOCK);
                warn("buffindexed: could not mmap overview for hidden "
                     "groups(%d)",
                     i);
                continue;
            }
            search.hi = ge->high;
            search.lo = ge->low;
            search.cur = 0;
            search.needov = true;
            while (ovsearch((void *) &search, NULL, &data, &len, &token,
                            &arrived, &expires)) {
                if (innconf->groupbaseexpiry)
                    /* assuming "." is not real newsgroup */
                    OVgroupbasedexpire(token, ".", data, len, arrived,
                                       expires);
            }
#ifdef OV_DEBUG
            freegroupblock(ge);
#else
            freegroupblock();
#endif
            ovgroupunmap();
            ge->expired = time(NULL);
            ge->count = 0;
            ge->baseindex = ge->curindex = ge->curdata = ovnull;
            GROUPlock(gloc, INN_LOCK_UNLOCK);
        }
        return true;
    }
    gloc = GROUPfind(group, false, NULL);
    if (GROUPLOCempty(gloc)) {
        return false;
    }
    GROUPlock(gloc, INN_LOCK_WRITE);
    ge = &GROUPentries[gloc.recno];
    if (ge->count == 0) {
        ge->low = ge->low > ge->high ? ge->low : ge->high;
        if (ge->low == 0)
            ge->low = 1;
        ge->high = ge->low - 1;
        if (lo != NULL)
            *lo = ge->low;
        ge->expired = time(NULL);
        GROUPlock(gloc, INN_LOCK_UNLOCK);
        return true;
    }
    flag = ge->flag;
    hash = ge->hash;
    next = ge->next;
    low = ge->low;
    high = ge->high;

    newge.low = 0;
    setinitialge(&newge, hash, &flag, next, 0, high);
    if ((handle = ovopensearch(group, low, high, true)) == NULL) {
        ge->expired = time(NULL);
        GROUPlock(gloc, INN_LOCK_UNLOCK);
        warn("buffindexed: could not open overview for '%s'", group);
        return false;
    }
    while (
        ovsearch(handle, &artnum, &data, &len, &token, &arrived, &expires)) {
        ah = NULL;
        if (len == 0)
            continue;
        if (!SMprobe(EXPENSIVESTAT, &token, NULL) || OVstatall) {
            if ((ah = SMretrieve(token, RETR_STAT)) == NULL)
                continue;
            SMfreearticle(ah);
        } else {
            if (!OVhisthasmsgid(h, data))
                continue;
        }
        if (innconf->groupbaseexpiry
            && OVgroupbasedexpire(token, group, data, len, arrived, expires))
            continue;
#ifdef OV_DEBUG
        if (!ovaddrec(&newge, artnum, token, data, len, arrived, expires,
                      ge)) {
#else
        if (!ovaddrec(&newge, artnum, token, data, len, arrived, expires)) {
#endif /* OV_DEBUG */
            /* Old group cannot be freed. */
            ovclosesearch(handle, false);
            ge->expired = time(NULL);
            GROUPlock(gloc, INN_LOCK_UNLOCK);
            warn("buffindexed: not enough room to expire overview for group "
                 "'%s'",
                 group);
            /* Clean just reserved overview entries. */
            if (!ovgroupmmap(&newge, newge.low, newge.high, true)) {
                warn("buffindexed: cannot prepare free operation");
                return false;
            }
#ifdef OV_DEBUG
            freegroupblock(ge);
#else
            freegroupblock();
#endif /* OV_DEBUG */
            ovgroupunmap();

            return false;
        }
    }
    if (newge.low == 0) {
        /* No article for the group. */
        newge.low = newge.high;
    }
    if (newge.count == 0) {
        /* The high water mark should be one less than the low water mark when
         * there is no article in the group. */
        newge.low = newge.low > newge.high ? newge.low : newge.high;
        if (newge.low == 0)
            newge.low = 1;
        newge.high = newge.low - 1;
    }
    *ge = newge;
    if (lo != NULL) {
        *lo = ge->low;
    }
    ovclosesearch(handle, true);
    ge->expired = time(NULL);
    GROUPlock(gloc, INN_LOCK_UNLOCK);
    return true;
}

bool
buffindexed_ctl(OVCTLTYPE type, void *val)
{
    int total, used, *i, j;
    float *f;
    OVBUFF *ovbuff = ovbufftab;
    OVSORTTYPE *sorttype;
    bool *boolval;
    GROUPDATABLOCK *gdb;

    switch (type) {
    case OVSPACE:
        for (total = used = 0; ovbuff != (OVBUFF *) NULL;
             ovbuff = ovbuff->next) {
            ovlock(ovbuff, INN_LOCK_READ);
            ovreadhead(ovbuff);
            total += ovbuff->totalblk;
            used += ovbuff->usedblk;
            ovlock(ovbuff, INN_LOCK_UNLOCK);
        }
        f = (float *) val;
        *f = ((float) used / (float) total) * 100;
        return true;
    case OVSORT:
        sorttype = (OVSORTTYPE *) val;
        *sorttype = OVNOSORT;
        return true;
    case OVCUTOFFLOW:
        Cutofflow = *(bool *) val;
        return true;
    case OVSTATICSEARCH:
        i = (int *) val;
        *i = true;
        for (j = 0; j < GROUPDATAHASHSIZE; j++) {
            for (gdb = groupdatablock[j]; gdb != NULL; gdb = gdb->next) {
                if (gdb->mmapped) {
                    *i = false;
                    return true;
                }
            }
        }
        return true;
    case OVCACHEKEEP:
        Cache = *(bool *) val;
        return true;
    case OVCACHEFREE:
        boolval = (bool *) val;
        *boolval = true;
        if (Gib != NULL) {
            free(Gib);
            Gib = NULL;
            if (Cachesearch != NULL) {
                free(Cachesearch->group);
                free(Cachesearch);
                Cachesearch = NULL;
            }
        }
        return true;
    default:
        return false;
    }
}

void
buffindexed_close(void)
{
    struct stat sb;
    OVBUFF *ovbuff = NULL;
#ifdef OV_DEBUG
    FILE *F = NULL;
    pid_t pid;
    char *path = NULL;
    size_t i;
    int j;
    struct ov_trace_array *trace;
    struct ov_name_table *ntp;
    size_t length;
#endif /* OV_DEBUG */

#ifdef OV_DEBUG
    ovbuff = ovbufftab;
    for (; ovbuff != (OVBUFF *) NULL; ovbuff = ovbuff->next) {
        for (i = 0; i < ovbuff->totalblk; i++) {
            trace = &ovbuff->trace[i];
            if (trace->ov_trace == NULL)
                continue;
            for (j = 0; j <= trace->cur && j < trace->max; j++) {
                if (trace->ov_trace[j].occupied != 0
                    || trace->ov_trace[j].freed != 0) {
                    if (F == NULL) {
                        length = strlen(innconf->pathtmp) + 11;
                        path = xmalloc(length);
                        pid = getpid();
                        snprintf(path, length, "%s/%d", innconf->pathtmp, pid);
                        if ((F = fopen(path, "w")) == NULL) {
                            syswarn("buffindexed: could not open %s", path);
                            break;
                        }
                    }
                    fprintf(F, "%d: % 6ld, % 2d: 0x%08x, % 10lu, % 10lu\n",
                            ovbuff->index, i, j, trace->ov_trace[j].gloc.recno,
                            (unsigned long) trace->ov_trace[j].occupied,
                            (unsigned long) trace->ov_trace[j].freed);
                }
            }
        }
    }
    if ((ntp = name_table) != NULL) {
        if (F == NULL) {
            length = strlen(innconf->pathtmp) + 11;
            path = xmalloc(length);
            pid = getpid();
            snprintf(path, length, "%s/%d", innconf->pathtmp, pid);
            if ((F = fopen(path, "w")) == NULL) {
                syswarn("buffindexed: could not open %s", path);
            }
        }
        if (F != NULL) {
            while (ntp) {
                fprintf(F, "0x%08x: %s\n", ntp->recno, ntp->name);
                ntp = ntp->next;
            }
        }
    }
    if (F != NULL)
        fclose(F);
    if (path != NULL)
        free(path);
#endif /* OV_DEBUG */
    if (Gib != NULL) {
        free(Gib);
        Gib = NULL;
        if (Cachesearch != NULL) {
            free(Cachesearch->group);
            free(Cachesearch);
            Cachesearch = NULL;
        }
    }
    if (fstat(GROUPfd, &sb) < 0)
        return;
    close(GROUPfd);

    if (GROUPheader) {
        if (munmap((void *) GROUPheader, GROUPmapsize) < 0) {
            syswarn("buffindexed: could not munmap group.index in "
                    "buffindexed_close");
            return;
        }
        GROUPheader = NULL;
        GROUPentries = NULL;
        GROUPmapsize = 0;
    }

    /* sync the bit field */
    ovbuff = ovbufftab;
    for (; ovbuff != (OVBUFF *) NULL; ovbuff = ovbuffnext) {
        if (ovbuff->dirty != 0) {
            ovbuff->dirty = OVBUFF_SYNC_COUNT + 1;
            ovflushhead(ovbuff);
        }
        ovbuffnext = ovbuff->next;
        if (ovbuff->smc != NULL)
            smcClose(ovbuff->smc);
        if (ovbuff->fd >= 0)
            close(ovbuff->fd);
        free(ovbuff);
    }
    ovbufftab = NULL;
    ovbuffnext = NULL;
}

#ifdef BUFF_DEBUG
static int
countgdb(void)
{
    int i, count = 0;
    GROUPDATABLOCK *gdb;

    for (i = 0; i < GROUPDATAHASHSIZE; i++) {
        for (gdb = groupdatablock[i]; gdb != NULL; gdb = gdb->next)
            count++;
    }
    return count;
}

int
main(int argc, char **argv)
{
    char *group, flag[2], buff[OV_BLOCKSIZE];
    int lo, hi, count, flags, i;
    OVSEARCH *search;
    GROUPENTRY *ge;
    GROUPLOC gloc;
    GIBLIST *giblist;

    if (argc != 2) {
        fprintf(stderr, "only one argument can be specified\n");
        exit(1);
    }
    /* if innconf isn't already read in, do so. */
    if (innconf == NULL) {
        if (!innconf_read(NULL)) {
            fprintf(stderr, "reading inn.conf failed\n");
            exit(1);
        }
    }
    if (!buffindexed_open(OV_READ)) {
        fprintf(stderr, "buffindexed_open failed\n");
        exit(1);
    }
    fprintf(stdout, "GROUPheader->freelist.recno is %d(0x%08x)\n",
            GROUPheader->freelist.recno,
            (unsigned int) GROUPheader->freelist.recno);
    group = argv[1];
    if (isdigit((unsigned char) *group)) {
        gloc.recno = atoi(group);
        ge = &GROUPentries[gloc.recno];
        fprintf(stdout, "left articles are %d for %d, last expiry is %lu\n",
                ge->count, gloc.recno, (unsigned long) ge->expired);
        if (ge->count == 0) {
            GROUPlock(gloc, INN_LOCK_UNLOCK);
            exit(0);
        }
        if (!ovgroupmmap(ge, ge->low, ge->high, true)) {
            fprintf(stderr, "ovgroupmmap failed\n");
            GROUPlock(gloc, INN_LOCK_UNLOCK);
        }
        for (giblist = Giblist, i = 0; giblist != NULL;
             giblist = giblist->next, i++)
            ;
        fprintf(stdout, "%d index block(s)\n", i);
        fprintf(stdout, "%d data block(s)\n", countgdb());
        for (giblist = Giblist; giblist != NULL; giblist = giblist->next) {
            fprintf(stdout, "  % 8d(% 5d)\n", (int) giblist->ov.blocknum,
                    giblist->ov.index);
        }
        for (i = 0; i < Gibcount; i++) {
            if (Gib[i].artnum == 0)
                continue;
            if (Gib[i].index == NULLINDEX)
                fprintf(stdout, "    %lu empty\n",
                        (unsigned long) Gib[i].offset);
            else {
                fprintf(stdout, "    %lu %d\n", (unsigned long) Gib[i].offset,
                        Gib[i].len);
            }
        }
        ovgroupunmap();
        GROUPlock(gloc, INN_LOCK_UNLOCK);
        exit(0);
    }
    gloc = GROUPfind(group, false, NULL);
    if (GROUPLOCempty(gloc)) {
        fprintf(stderr, "gloc is null\n");
    }
    GROUPlock(gloc, INN_LOCK_READ);
    ge = &GROUPentries[gloc.recno];
    fprintf(stdout, "base %u(%d), cur %u(%d), expired at %s\n",
            ge->baseindex.blocknum, ge->baseindex.index, ge->curindex.blocknum,
            ge->curindex.index,
            ge->expired == 0 ? "none\n" : ctime(&ge->expired));
    if (!buffindexed_groupstats(group, &lo, &hi, &count, &flags)) {
        fprintf(stderr, "buffindexed_groupstats failed for group %s\n", group);
        exit(1);
    }
    flag[0] = (char) flags;
    flag[1] = '\0';
    fprintf(stdout, "%s: low is %d, high is %d, count is %d, flag is '%s'\n",
            group, lo, hi, count, flag);
    if ((search = (OVSEARCH *) ovopensearch(group, lo, hi, true)) == NULL) {
        fprintf(stderr, "ovopensearch failed for group %s\n", group);
        exit(1);
    }
    fprintf(stdout, "  gloc is %d(0x%08x)\n", search->gloc.recno,
            (unsigned int) search->gloc.recno);
    for (giblist = Giblist, i = 0; giblist != NULL;
         giblist = giblist->next, i++)
        ;
    fprintf(stdout, "%d index block(s)\n", i);
    fprintf(stdout, "%d data block(s)\n", countgdb());
    for (giblist = Giblist; giblist != NULL; giblist = giblist->next) {
        fprintf(stdout, "  % 8d(% 5d)\n", (int) giblist->ov.blocknum,
                giblist->ov.index);
    }
    for (i = 0; i < Gibcount; i++) {
        if (Gib[i].artnum == 0)
            continue;
        if (Gib[i].index == NULLINDEX)
            fprintf(stdout, "    %lu empty\n", (unsigned long) Gib[i].offset);
        else {
            fprintf(stdout, "    %lu %d\n", (unsigned long) Gib[i].offset,
                    Gib[i].len);
        }
    }
    {
        ARTNUM artnum = 0;
        char *data = NULL;
        int len = 0;
        TOKEN token;
        while (buffindexed_search((void *) search, &artnum, &data, &len,
                                  &token, NULL)) {
            if (len == 0)
                fprintf(stdout, "%lu: len is 0\n", artnum);
            else {
                memcpy(buff, data, len);
                buff[len] = '\0';
                fprintf(stdout, "%lu: %s\n", artnum, buff);
            }
        }
    }
    return 0;
}
#endif /* BUFF_DEBUG */
